diff options
Diffstat (limited to 'src/lib-storage/index')
168 files changed, 64379 insertions, 0 deletions
diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am new file mode 100644 index 0000000..23461e2 --- /dev/null +++ b/src/lib-storage/index/Makefile.am @@ -0,0 +1,58 @@ +SUBDIRS = maildir mbox dbox-common dbox-multi dbox-single imapc pop3c raw shared + +noinst_LTLIBRARIES = libstorage_index.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +libstorage_index_la_SOURCES = \ + istream-mail.c \ + index-attachment.c \ + index-attribute.c \ + index-mail.c \ + index-mail-binary.c \ + index-mail-headers.c \ + index-mailbox-size.c \ + index-pop3-uidl.c \ + index-rebuild.c \ + index-search.c \ + index-search-mime.c \ + index-search-result.c \ + index-sort.c \ + index-sort-string.c \ + index-status.c \ + index-storage.c \ + index-sync.c \ + index-sync-changes.c \ + index-sync-pvt.c \ + index-sync-search.c \ + index-thread.c \ + index-thread-finish.c \ + index-thread-links.c \ + index-transaction.c + +headers = \ + istream-mail.h \ + index-attachment.h \ + index-mail.h \ + index-mailbox-size.h \ + index-pop3-uidl.h \ + index-rebuild.h \ + index-search-private.h \ + index-search-result.h \ + index-sort.h \ + index-sort-private.h \ + index-storage.h \ + index-sync-changes.h \ + index-sync-private.h \ + index-thread-private.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/Makefile.in b/src/lib-storage/index/Makefile.in new file mode 100644 index 0000000..09bbdfa --- /dev/null +++ b/src/lib-storage/index/Makefile.in @@ -0,0 +1,1061 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_index_la_LIBADD = +am_libstorage_index_la_OBJECTS = istream-mail.lo index-attachment.lo \ + index-attribute.lo index-mail.lo index-mail-binary.lo \ + index-mail-headers.lo index-mailbox-size.lo index-pop3-uidl.lo \ + index-rebuild.lo index-search.lo index-search-mime.lo \ + index-search-result.lo index-sort.lo index-sort-string.lo \ + index-status.lo index-storage.lo index-sync.lo \ + index-sync-changes.lo index-sync-pvt.lo index-sync-search.lo \ + index-thread.lo index-thread-finish.lo index-thread-links.lo \ + index-transaction.lo +libstorage_index_la_OBJECTS = $(am_libstorage_index_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/index-attachment.Plo \ + ./$(DEPDIR)/index-attribute.Plo \ + ./$(DEPDIR)/index-mail-binary.Plo \ + ./$(DEPDIR)/index-mail-headers.Plo ./$(DEPDIR)/index-mail.Plo \ + ./$(DEPDIR)/index-mailbox-size.Plo \ + ./$(DEPDIR)/index-pop3-uidl.Plo ./$(DEPDIR)/index-rebuild.Plo \ + ./$(DEPDIR)/index-search-mime.Plo \ + ./$(DEPDIR)/index-search-result.Plo \ + ./$(DEPDIR)/index-search.Plo ./$(DEPDIR)/index-sort-string.Plo \ + ./$(DEPDIR)/index-sort.Plo ./$(DEPDIR)/index-status.Plo \ + ./$(DEPDIR)/index-storage.Plo \ + ./$(DEPDIR)/index-sync-changes.Plo \ + ./$(DEPDIR)/index-sync-pvt.Plo \ + ./$(DEPDIR)/index-sync-search.Plo ./$(DEPDIR)/index-sync.Plo \ + ./$(DEPDIR)/index-thread-finish.Plo \ + ./$(DEPDIR)/index-thread-links.Plo \ + ./$(DEPDIR)/index-thread.Plo ./$(DEPDIR)/index-transaction.Plo \ + ./$(DEPDIR)/istream-mail.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_index_la_SOURCES) +DIST_SOURCES = $(libstorage_index_la_SOURCES) +RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \ + ctags-recursive dvi-recursive html-recursive info-recursive \ + install-data-recursive install-dvi-recursive \ + install-exec-recursive install-html-recursive \ + install-info-recursive install-pdf-recursive \ + install-ps-recursive install-recursive installcheck-recursive \ + installdirs-recursive pdf-recursive ps-recursive \ + tags-recursive uninstall-recursive +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ + distclean-recursive maintainer-clean-recursive +am__recursive_targets = \ + $(RECURSIVE_TARGETS) \ + $(RECURSIVE_CLEAN_TARGETS) \ + $(am__extra_recursive_targets) +AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \ + distdir distdir-am +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +DIST_SUBDIRS = $(SUBDIRS) +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +am__relativize = \ + dir0=`pwd`; \ + sed_first='s,^\([^/]*\)/.*$$,\1,'; \ + sed_rest='s,^[^/]*/*,,'; \ + sed_last='s,^.*/\([^/]*\)$$,\1,'; \ + sed_butlast='s,/*[^/]*$$,,'; \ + while test -n "$$dir1"; do \ + first=`echo "$$dir1" | sed -e "$$sed_first"`; \ + if test "$$first" != "."; then \ + if test "$$first" = ".."; then \ + dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \ + dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \ + else \ + first2=`echo "$$dir2" | sed -e "$$sed_first"`; \ + if test "$$first2" = "$$first"; then \ + dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \ + else \ + dir2="../$$dir2"; \ + fi; \ + dir0="$$dir0"/"$$first"; \ + fi; \ + fi; \ + dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \ + done; \ + reldir="$$dir2" +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +SUBDIRS = maildir mbox dbox-common dbox-multi dbox-single imapc pop3c raw shared +noinst_LTLIBRARIES = libstorage_index.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +libstorage_index_la_SOURCES = \ + istream-mail.c \ + index-attachment.c \ + index-attribute.c \ + index-mail.c \ + index-mail-binary.c \ + index-mail-headers.c \ + index-mailbox-size.c \ + index-pop3-uidl.c \ + index-rebuild.c \ + index-search.c \ + index-search-mime.c \ + index-search-result.c \ + index-sort.c \ + index-sort-string.c \ + index-status.c \ + index-storage.c \ + index-sync.c \ + index-sync-changes.c \ + index-sync-pvt.c \ + index-sync-search.c \ + index-thread.c \ + index-thread-finish.c \ + index-thread-links.c \ + index-transaction.c + +headers = \ + istream-mail.h \ + index-attachment.h \ + index-mail.h \ + index-mailbox-size.h \ + index-pop3-uidl.h \ + index-rebuild.h \ + index-search-private.h \ + index-search-result.h \ + index-sort.h \ + index-sort-private.h \ + index-storage.h \ + index-sync-changes.h \ + index-sync-private.h \ + index-thread-private.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-recursive + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_index.la: $(libstorage_index_la_OBJECTS) $(libstorage_index_la_DEPENDENCIES) $(EXTRA_libstorage_index_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_index_la_OBJECTS) $(libstorage_index_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-attachment.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-attribute.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mail-binary.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mail-headers.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mailbox-size.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-pop3-uidl.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-rebuild.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-search-mime.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-search-result.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-search.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sort-string.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sort.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-status.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync-changes.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync-pvt.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync-search.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-thread-finish.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-thread-links.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-thread.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-transaction.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-mail.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +# This directory's subdirectories are mostly independent; you can cd +# into them and run 'make' without going through this Makefile. +# To change the values of 'make' variables: instead of editing Makefiles, +# (1) if the variable is set in 'config.status', edit 'config.status' +# (which will cause the Makefiles to be regenerated when you run 'make'); +# (2) otherwise, pass the desired values on the 'make' command line. +$(am__recursive_targets): + @fail=; \ + if $(am__make_keepgoing); then \ + failcom='fail=yes'; \ + else \ + failcom='exit 1'; \ + fi; \ + dot_seen=no; \ + target=`echo $@ | sed s/-recursive//`; \ + case "$@" in \ + distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \ + *) list='$(SUBDIRS)' ;; \ + esac; \ + for subdir in $$list; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + dot_seen=yes; \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done; \ + if test "$$dot_seen" = "no"; then \ + $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \ + fi; test -z "$$fail" + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-recursive +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \ + include_option=--etags-include; \ + empty_fix=.; \ + else \ + include_option=--include; \ + empty_fix=; \ + fi; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test ! -f $$subdir/TAGS || \ + set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \ + fi; \ + done; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-recursive + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-recursive + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + $(am__make_dryrun) \ + || test -d "$(distdir)/$$subdir" \ + || $(MKDIR_P) "$(distdir)/$$subdir" \ + || exit 1; \ + dir1=$$subdir; dir2="$(distdir)/$$subdir"; \ + $(am__relativize); \ + new_distdir=$$reldir; \ + dir1=$$subdir; dir2="$(top_distdir)"; \ + $(am__relativize); \ + new_top_distdir=$$reldir; \ + echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \ + echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \ + ($(am__cd) $$subdir && \ + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$$new_top_distdir" \ + distdir="$$new_distdir" \ + am__remove_distdir=: \ + am__skip_length_check=: \ + am__skip_mode_fix=: \ + distdir) \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-recursive +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: installdirs-recursive +installdirs-am: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-recursive +install-exec: install-exec-recursive +install-data: install-data-recursive +uninstall: uninstall-recursive + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-recursive +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-recursive + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-recursive + -rm -f ./$(DEPDIR)/index-attachment.Plo + -rm -f ./$(DEPDIR)/index-attribute.Plo + -rm -f ./$(DEPDIR)/index-mail-binary.Plo + -rm -f ./$(DEPDIR)/index-mail-headers.Plo + -rm -f ./$(DEPDIR)/index-mail.Plo + -rm -f ./$(DEPDIR)/index-mailbox-size.Plo + -rm -f ./$(DEPDIR)/index-pop3-uidl.Plo + -rm -f ./$(DEPDIR)/index-rebuild.Plo + -rm -f ./$(DEPDIR)/index-search-mime.Plo + -rm -f ./$(DEPDIR)/index-search-result.Plo + -rm -f ./$(DEPDIR)/index-search.Plo + -rm -f ./$(DEPDIR)/index-sort-string.Plo + -rm -f ./$(DEPDIR)/index-sort.Plo + -rm -f ./$(DEPDIR)/index-status.Plo + -rm -f ./$(DEPDIR)/index-storage.Plo + -rm -f ./$(DEPDIR)/index-sync-changes.Plo + -rm -f ./$(DEPDIR)/index-sync-pvt.Plo + -rm -f ./$(DEPDIR)/index-sync-search.Plo + -rm -f ./$(DEPDIR)/index-sync.Plo + -rm -f ./$(DEPDIR)/index-thread-finish.Plo + -rm -f ./$(DEPDIR)/index-thread-links.Plo + -rm -f ./$(DEPDIR)/index-thread.Plo + -rm -f ./$(DEPDIR)/index-transaction.Plo + -rm -f ./$(DEPDIR)/istream-mail.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-recursive + +dvi-am: + +html: html-recursive + +html-am: + +info: info-recursive + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-recursive + +install-dvi-am: + +install-exec-am: + +install-html: install-html-recursive + +install-html-am: + +install-info: install-info-recursive + +install-info-am: + +install-man: + +install-pdf: install-pdf-recursive + +install-pdf-am: + +install-ps: install-ps-recursive + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-recursive + -rm -f ./$(DEPDIR)/index-attachment.Plo + -rm -f ./$(DEPDIR)/index-attribute.Plo + -rm -f ./$(DEPDIR)/index-mail-binary.Plo + -rm -f ./$(DEPDIR)/index-mail-headers.Plo + -rm -f ./$(DEPDIR)/index-mail.Plo + -rm -f ./$(DEPDIR)/index-mailbox-size.Plo + -rm -f ./$(DEPDIR)/index-pop3-uidl.Plo + -rm -f ./$(DEPDIR)/index-rebuild.Plo + -rm -f ./$(DEPDIR)/index-search-mime.Plo + -rm -f ./$(DEPDIR)/index-search-result.Plo + -rm -f ./$(DEPDIR)/index-search.Plo + -rm -f ./$(DEPDIR)/index-sort-string.Plo + -rm -f ./$(DEPDIR)/index-sort.Plo + -rm -f ./$(DEPDIR)/index-status.Plo + -rm -f ./$(DEPDIR)/index-storage.Plo + -rm -f ./$(DEPDIR)/index-sync-changes.Plo + -rm -f ./$(DEPDIR)/index-sync-pvt.Plo + -rm -f ./$(DEPDIR)/index-sync-search.Plo + -rm -f ./$(DEPDIR)/index-sync.Plo + -rm -f ./$(DEPDIR)/index-thread-finish.Plo + -rm -f ./$(DEPDIR)/index-thread-links.Plo + -rm -f ./$(DEPDIR)/index-thread.Plo + -rm -f ./$(DEPDIR)/index-transaction.Plo + -rm -f ./$(DEPDIR)/istream-mail.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-recursive + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-recursive + +pdf-am: + +ps: ps-recursive + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: $(am__recursive_targets) install-am install-strip + +.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am \ + am--depfiles check check-am clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES cscopelist-am ctags ctags-am distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-pkginc_libHEADERS \ + install-ps install-ps-am install-strip installcheck \ + installcheck-am installdirs installdirs-am maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/dbox-common/Makefile.am b/src/lib-storage/index/dbox-common/Makefile.am new file mode 100644 index 0000000..ec3d511 --- /dev/null +++ b/src/lib-storage/index/dbox-common/Makefile.am @@ -0,0 +1,29 @@ +noinst_LTLIBRARIES = libstorage_dbox_common.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_dbox_common_la_SOURCES = \ + dbox-attachment.c \ + dbox-file.c \ + dbox-file-fix.c \ + dbox-mail.c \ + dbox-save.c \ + dbox-storage.c + +headers = \ + dbox-attachment.h \ + dbox-file.h \ + dbox-mail.h \ + dbox-save.h \ + dbox-storage.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/dbox-common/Makefile.in b/src/lib-storage/index/dbox-common/Makefile.in new file mode 100644 index 0000000..9c0eed4 --- /dev/null +++ b/src/lib-storage/index/dbox-common/Makefile.in @@ -0,0 +1,843 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/dbox-common +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_dbox_common_la_LIBADD = +am_libstorage_dbox_common_la_OBJECTS = dbox-attachment.lo dbox-file.lo \ + dbox-file-fix.lo dbox-mail.lo dbox-save.lo dbox-storage.lo +libstorage_dbox_common_la_OBJECTS = \ + $(am_libstorage_dbox_common_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/dbox-attachment.Plo \ + ./$(DEPDIR)/dbox-file-fix.Plo ./$(DEPDIR)/dbox-file.Plo \ + ./$(DEPDIR)/dbox-mail.Plo ./$(DEPDIR)/dbox-save.Plo \ + ./$(DEPDIR)/dbox-storage.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_dbox_common_la_SOURCES) +DIST_SOURCES = $(libstorage_dbox_common_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_dbox_common.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_dbox_common_la_SOURCES = \ + dbox-attachment.c \ + dbox-file.c \ + dbox-file-fix.c \ + dbox-mail.c \ + dbox-save.c \ + dbox-storage.c + +headers = \ + dbox-attachment.h \ + dbox-file.h \ + dbox-mail.h \ + dbox-save.h \ + dbox-storage.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/dbox-common/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/dbox-common/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_dbox_common.la: $(libstorage_dbox_common_la_OBJECTS) $(libstorage_dbox_common_la_DEPENDENCIES) $(EXTRA_libstorage_dbox_common_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_dbox_common_la_OBJECTS) $(libstorage_dbox_common_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-attachment.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-file-fix.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-storage.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/dbox-attachment.Plo + -rm -f ./$(DEPDIR)/dbox-file-fix.Plo + -rm -f ./$(DEPDIR)/dbox-file.Plo + -rm -f ./$(DEPDIR)/dbox-mail.Plo + -rm -f ./$(DEPDIR)/dbox-save.Plo + -rm -f ./$(DEPDIR)/dbox-storage.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/dbox-attachment.Plo + -rm -f ./$(DEPDIR)/dbox-file-fix.Plo + -rm -f ./$(DEPDIR)/dbox-file.Plo + -rm -f ./$(DEPDIR)/dbox-mail.Plo + -rm -f ./$(DEPDIR)/dbox-save.Plo + -rm -f ./$(DEPDIR)/dbox-storage.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/dbox-common/dbox-attachment.c b/src/lib-storage/index/dbox-common/dbox-attachment.c new file mode 100644 index 0000000..cfc3f62 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-attachment.c @@ -0,0 +1,77 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "str.h" +#include "dbox-file.h" +#include "dbox-save.h" +#include "dbox-attachment.h" + +void dbox_attachment_save_write_metadata(struct mail_save_context *ctx, + string_t *str) +{ + const ARRAY_TYPE(mail_attachment_extref) *extrefs; + + extrefs = index_attachment_save_get_extrefs(ctx); + if (extrefs == NULL || array_count(extrefs) == 0) + return; + + str_append_c(str, DBOX_METADATA_EXT_REF); + index_attachment_append_extrefs(str, extrefs); + str_append_c(str, '\n'); +} + +static int +dbox_attachment_file_get_stream_from(struct dbox_file *file, + const char *ext_refs, + struct istream **stream, + const char **error_r) +{ + const char *path_suffix; + uoff_t msg_size; + + if (file->storage->attachment_dir == NULL) { + mail_storage_set_critical(&file->storage->storage, + "%s contains references to external attachments, " + "but mail_attachment_dir is unset", file->cur_path); + return -1; + } + + msg_size = dbox_file_get_plaintext_size(file); + path_suffix = file->storage->v.get_attachment_path_suffix(file); + if (index_attachment_stream_get(file->storage->attachment_fs, + file->storage->attachment_dir, + path_suffix, stream, msg_size, + ext_refs, error_r) < 0) + return 0; + return 1; +} + +int dbox_attachment_file_get_stream(struct dbox_file *file, + struct istream **stream) +{ + const char *ext_refs, *error; + int ret; + + /* need to read metadata in case there are external references */ + if ((ret = dbox_file_metadata_read(file)) <= 0) + return ret; + + i_stream_seek(file->input, file->cur_offset + file->msg_header_size); + + ext_refs = dbox_file_metadata_get(file, DBOX_METADATA_EXT_REF); + if (ext_refs == NULL) + return 1; + + /* we have external references. */ + T_BEGIN { + ret = dbox_attachment_file_get_stream_from(file, ext_refs, + stream, &error); + if (ret == 0) { + dbox_file_set_corrupted(file, + "Corrupted ext-refs metadata %s: %s", + ext_refs, error); + } + } T_END; + return ret; +} diff --git a/src/lib-storage/index/dbox-common/dbox-attachment.h b/src/lib-storage/index/dbox-common/dbox-attachment.h new file mode 100644 index 0000000..a90ba54 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-attachment.h @@ -0,0 +1,16 @@ +#ifndef DBOX_ATTACHMENT_H +#define DBOX_ATTACHMENT_H + +#include "index-attachment.h" + +struct dbox_file; + +void dbox_attachment_save_write_metadata(struct mail_save_context *ctx, + string_t *str); + +/* Build a single message body stream out of the current message and all of its + attachments. */ +int dbox_attachment_file_get_stream(struct dbox_file *file, + struct istream **stream); + +#endif diff --git a/src/lib-storage/index/dbox-common/dbox-file-fix.c b/src/lib-storage/index/dbox-common/dbox-file-fix.c new file mode 100644 index 0000000..1c44ca4 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-file-fix.c @@ -0,0 +1,519 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hex-dec.h" +#include "istream.h" +#include "ostream.h" +#include "message-size.h" +#include "dbox-storage.h" +#include "dbox-file.h" + +#include <stdio.h> + +#define DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX ".broken" + +static int +dbox_file_match_pre_magic(struct istream *input, + uoff_t *pre_offset, size_t *need_bytes) +{ + const struct dbox_message_header *hdr; + const unsigned char *data; + size_t size; + uoff_t offset = input->v_offset; + bool have_lf = FALSE; + + data = i_stream_get_data(input, &size); + if (data[0] == '\n') { + data++; size--; offset++; + have_lf = TRUE; + } + i_assert(data[0] == DBOX_MAGIC_PRE[0]); + if (size < sizeof(*hdr)) { + *need_bytes = sizeof(*hdr) + (have_lf ? 1 : 0); + return -1; + } + hdr = (const void *)data; + if (memcmp(hdr->magic_pre, DBOX_MAGIC_PRE, strlen(DBOX_MAGIC_PRE)) != 0) + return 0; + if (hdr->type != DBOX_MESSAGE_TYPE_NORMAL) + return 0; + if (hdr->space1 != ' ' || hdr->space2 != ' ') + return 0; + if (hex2dec(hdr->message_size_hex, sizeof(hdr->message_size_hex)) == 0 && + memcmp(hdr->message_size_hex, "0000000000000000", sizeof(hdr->message_size_hex)) != 0) + return 0; + + *pre_offset = offset; + return 1; +} + +static bool memchr_nocontrol(const unsigned char *data, char chr, + unsigned int len, const unsigned char **pos_r) +{ + unsigned int i; + + for (i = 0; i < len; i++) { + if (data[i] == chr) { + *pos_r = data+i; + return TRUE; + } + if (data[i] < ' ') + return FALSE; + } + *pos_r = NULL; + return TRUE; +} + +static int +dbox_file_match_post_magic(struct istream *input, bool input_full, + size_t *need_bytes) +{ + const unsigned char *data, *p; + size_t i, size; + bool allow_control; + + data = i_stream_get_data(input, &size); + if (size < strlen(DBOX_MAGIC_POST)) { + *need_bytes = strlen(DBOX_MAGIC_POST); + return -1; + } + if (memcmp(data, DBOX_MAGIC_POST, strlen(DBOX_MAGIC_POST)) != 0) + return 0; + + /* see if the metadata block looks valid */ + for (i = strlen(DBOX_MAGIC_POST); i < size; ) { + switch (data[i]) { + case '\n': + return 1; + case DBOX_METADATA_GUID: + case DBOX_METADATA_POP3_UIDL: + case DBOX_METADATA_ORIG_MAILBOX: + case DBOX_METADATA_OLDV1_KEYWORDS: + /* these could contain anything */ + allow_control = TRUE; + break; + case DBOX_METADATA_POP3_ORDER: + case DBOX_METADATA_RECEIVED_TIME: + case DBOX_METADATA_PHYSICAL_SIZE: + case DBOX_METADATA_VIRTUAL_SIZE: + case DBOX_METADATA_EXT_REF: + case DBOX_METADATA_OLDV1_EXPUNGED: + case DBOX_METADATA_OLDV1_FLAGS: + case DBOX_METADATA_OLDV1_SAVE_TIME: + case DBOX_METADATA_OLDV1_SPACE: + /* no control chars */ + allow_control = FALSE; + break; + default: + if (data[i] < 'A' || data[i] > 'Z') + return 0; + /* unknown */ + allow_control = TRUE; + break; + } + if (allow_control) { + p = memchr(data+i, '\n', size-i); + } else { + if (!memchr_nocontrol(data+i, '\n', size-i, &p)) + return 0; + } + if (p == NULL) { + /* LF not found - try to find the end-of-metadata LF */ + if (input_full) { + /* can't look any further - assume it's ok */ + return 1; + } + *need_bytes = size+1; + return -1; + } + i = p - data+1; + } + *need_bytes = size+1; + return -1; +} + +static int +dbox_file_find_next_magic(struct dbox_file *file, uoff_t *offset_r, bool *pre_r) +{ + /* We're scanning message bodies here, trying to find the beginning of + the next message. Although our magic strings are very unlikely to + be found in regular emails, they are much more likely when emails + are stored compressed.. So try to be sure we find the correct + magic markers. */ + + struct istream *input = file->input; + uoff_t orig_offset, pre_offset, post_offset, prev_offset; + const unsigned char *data, *magic; + size_t size, need_bytes, prev_need_bytes; + int ret, match; + + *pre_r = FALSE; + + orig_offset = prev_offset = input->v_offset; + need_bytes = strlen(DBOX_MAGIC_POST); prev_need_bytes = 0; + while ((ret = i_stream_read_bytes(input, &data, &size, need_bytes)) > 0 || + ret == -2) { + /* search for the beginning of a potential pre/post magic */ + i_assert(size > 1); + i_assert(prev_offset != input->v_offset || + need_bytes > prev_need_bytes); + prev_offset = input->v_offset; + prev_need_bytes = need_bytes; + + magic = memchr(data, DBOX_MAGIC_PRE[0], size); + if (magic == NULL) { + i_stream_skip(input, size-1); + need_bytes = strlen(DBOX_MAGIC_POST); + continue; + } + if (magic == data && input->v_offset == orig_offset) { + /* beginning of the file */ + } else if (magic != data && magic[-1] == '\n') { + /* PRE/POST block? leave \n */ + i_stream_skip(input, magic-data-1); + } else { + i_stream_skip(input, magic-data+1); + need_bytes = strlen(DBOX_MAGIC_POST); + continue; + } + + pre_offset = UOFF_T_MAX; + match = dbox_file_match_pre_magic(input, &pre_offset, &need_bytes); + if (match < 0) { + /* more data needed */ + if (ret == -2) { + i_stream_skip(input, 2); + need_bytes = strlen(DBOX_MAGIC_POST); + } + continue; + } + if (match > 0) + *pre_r = TRUE; + + match = dbox_file_match_post_magic(input, ret == -2, &need_bytes); + if (match < 0) { + /* more data needed */ + if (ret == -2) { + i_stream_skip(input, 2); + need_bytes = strlen(DBOX_MAGIC_POST); + } + continue; + } + if (match > 0) { + post_offset = input->v_offset; + if (pre_offset == UOFF_T_MAX || + post_offset < pre_offset) { + pre_offset = post_offset; + *pre_r = FALSE; + } + } + + if (pre_offset != UOFF_T_MAX) { + *offset_r = pre_offset; + ret = 1; + break; + } + i_stream_skip(input, size-1); + } + if (ret <= 0) { + i_assert(ret == -1); + if (input->stream_errno != 0) + dbox_file_set_syscall_error(file, "read()"); + else { + ret = 0; + *offset_r = input->v_offset; + } + } + i_stream_seek(input, orig_offset); + return ret <= 0 ? ret : 1; +} + +static int +stream_copy(struct dbox_file *file, struct ostream *output, + const char *out_path, uoff_t count) +{ + struct istream *input; + int ret = 0; + + input = i_stream_create_limit(file->input, count); + o_stream_nsend_istream(output, input); + + if (input->stream_errno != 0) { + mail_storage_set_critical(&file->storage->storage, + "read(%s) failed: %s", file->cur_path, + i_stream_get_error(input)); + ret = -1; + } else if (o_stream_flush(output) < 0) { + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %s", out_path, + o_stream_get_error(output)); + ret = -1; + } else if (input->v_offset != count) { + mail_storage_set_critical(&file->storage->storage, + "o_stream_send_istream(%s) copied only %" + PRIuUOFF_T" of %"PRIuUOFF_T" bytes", + out_path, input->v_offset, count); + ret = -1; + } + i_stream_unref(&input); + return ret; +} + +static void dbox_file_skip_broken_header(struct dbox_file *file) +{ + const size_t magic_len = strlen(DBOX_MAGIC_PRE); + const unsigned char *data; + size_t i, size; + + /* if there's LF close to our position, assume that the header ends + there. */ + data = i_stream_get_data(file->input, &size); + if (size > file->msg_header_size + 16) + size = file->msg_header_size + 16; + for (i = 0; i < size; i++) { + if (data[i] == '\n') { + i_stream_skip(file->input, i); + return; + } + } + + /* skip at least the magic bytes if possible */ + if (size > magic_len && memcmp(data, DBOX_MAGIC_PRE, magic_len) == 0) + i_stream_skip(file->input, magic_len); +} + +static void +dbox_file_copy_metadata(struct dbox_file *file, struct ostream *output, + bool *have_guid_r) +{ + const char *line; + uoff_t prev_offset = file->input->v_offset; + + *have_guid_r = FALSE; + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + return; + } + if (*line < 32) { + /* broken - possibly a new pre-magic block */ + i_stream_seek(file->input, prev_offset); + return; + } + if (*line == DBOX_METADATA_VIRTUAL_SIZE) { + /* it may be wrong - recreate it */ + continue; + } + if (*line == DBOX_METADATA_GUID) + *have_guid_r = TRUE; + o_stream_nsend_str(output, line); + o_stream_nsend_str(output, "\n"); + } +} + +static int +dbox_file_fix_write_stream(struct dbox_file *file, uoff_t start_offset, + const char *temp_path, struct ostream *output) +{ + struct dbox_message_header msg_hdr; + uoff_t offset, msg_size, hdr_offset, body_offset; + bool pre, write_header, have_guid; + struct message_size body; + bool has_nuls; + struct istream *body_input; + guid_128_t guid_128; + int ret; + + i_stream_seek(file->input, 0); + if (start_offset > 0) { + /* copy the valid data */ + if (stream_copy(file, output, temp_path, start_offset) < 0) + return -1; + } else { + /* the file header is broken. recreate it */ + if (dbox_file_header_write(file, output) < 0) { + dbox_file_set_syscall_error(file, "write()"); + return -1; + } + } + + while ((ret = dbox_file_find_next_magic(file, &offset, &pre)) > 0) { + msg_size = offset - file->input->v_offset; + if (msg_size < 256 && pre) { + /* probably some garbage or some broken headers. + we most likely don't miss anything by skipping + over this data. */ + i_stream_skip(file->input, msg_size); + hdr_offset = file->input->v_offset; + ret = dbox_file_read_mail_header(file, &msg_size); + if (ret <= 0) { + if (ret < 0) + return -1; + dbox_file_skip_broken_header(file); + body_offset = file->input->v_offset; + msg_size = UOFF_T_MAX; + } else { + i_stream_skip(file->input, + file->msg_header_size); + body_offset = file->input->v_offset; + i_stream_skip(file->input, msg_size); + } + + ret = dbox_file_find_next_magic(file, &offset, &pre); + if (ret <= 0) + break; + + if (!pre && msg_size == offset - body_offset) { + /* msg header ok, copy it */ + i_stream_seek(file->input, hdr_offset); + if (stream_copy(file, output, temp_path, + file->msg_header_size) < 0) + return -1; + write_header = FALSE; + } else { + /* msg header is broken. write our own. */ + i_stream_seek(file->input, body_offset); + if (msg_size != UOFF_T_MAX) { + /* previous magic find might have + skipped too much. seek back and + make sure */ + ret = dbox_file_find_next_magic(file, &offset, &pre); + if (ret <= 0) + break; + } + + write_header = TRUE; + msg_size = offset - body_offset; + } + } else { + /* treat this data as a separate message. */ + write_header = TRUE; + body_offset = file->input->v_offset; + } + /* write msg header */ + if (write_header) { + dbox_msg_header_fill(&msg_hdr, msg_size); + o_stream_nsend(output, &msg_hdr, sizeof(msg_hdr)); + } + /* write msg body */ + i_assert(file->input->v_offset == body_offset); + if (stream_copy(file, output, temp_path, msg_size) < 0) + return -1; + i_assert(file->input->v_offset == offset); + + /* get message body size */ + i_stream_seek(file->input, body_offset); + body_input = i_stream_create_limit(file->input, msg_size); + ret = message_get_body_size(body_input, &body, &has_nuls); + i_stream_unref(&body_input); + if (ret < 0) { + mail_storage_set_critical(&file->storage->storage, + "read(%s) failed: %s", file->cur_path, + i_stream_get_error(body_input)); + return -1; + } + + /* write msg metadata. */ + i_assert(file->input->v_offset == offset); + ret = dbox_file_metadata_skip_header(file); + if (ret < 0) + return -1; + o_stream_nsend_str(output, DBOX_MAGIC_POST); + if (ret == 0) + have_guid = FALSE; + else + dbox_file_copy_metadata(file, output, &have_guid); + if (!have_guid) { + guid_128_generate(guid_128); + o_stream_nsend_str(output, + t_strdup_printf("%c%s\n", DBOX_METADATA_GUID, + guid_128_to_string(guid_128))); + } + o_stream_nsend_str(output, + t_strdup_printf("%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, + (unsigned long long)body.virtual_size)); + o_stream_nsend_str(output, "\n"); + if (output->stream_errno != 0) + break; + } + if (o_stream_flush(output) < 0) { + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %s", temp_path, o_stream_get_error(output)); + ret = -1; + } + return ret; +} + +int dbox_file_fix(struct dbox_file *file, uoff_t start_offset) +{ + struct ostream *output; + const char *dir, *p, *temp_path, *broken_path; + bool deleted, have_messages; + int fd, ret; + + i_assert(dbox_file_is_open(file)); + + p = strrchr(file->cur_path, '/'); + i_assert(p != NULL); + dir = t_strdup_until(file->cur_path, p); + + temp_path = t_strdup_printf("%s/%s", dir, dbox_generate_tmp_filename()); + fd = file->storage->v.file_create_fd(file, temp_path, FALSE); + if (fd == -1) + return -1; + + output = o_stream_create_fd_file(fd, 0, FALSE); + o_stream_cork(output); + ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output); + if (ret < 0) + o_stream_abort(output); + have_messages = output->offset > file->file_header_size; + o_stream_unref(&output); + if (close(fd) < 0) { + mail_storage_set_critical(&file->storage->storage, + "close(%s) failed: %m", temp_path); + ret = -1; + } + if (ret < 0) { + if (unlink(temp_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "unlink(%s) failed: %m", temp_path); + } + return -1; + } + /* keep a copy of the original file in case someone wants to look + at it */ + broken_path = t_strconcat(file->cur_path, + DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX, NULL); + if (link(file->cur_path, broken_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "link(%s, %s) failed: %m", + file->cur_path, broken_path); + } else { + i_warning("dbox: Copy of the broken file saved to %s", + broken_path); + } + if (!have_messages) { + /* the resulting file has no messages. just delete the file. */ + dbox_file_close(file); + i_unlink(temp_path); + i_unlink(file->cur_path); + return 0; + } + if (rename(temp_path, file->cur_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "rename(%s, %s) failed: %m", + temp_path, file->cur_path); + return -1; + } + + /* file was successfully recreated - reopen it */ + dbox_file_close(file); + if (dbox_file_open(file, &deleted) <= 0) { + mail_storage_set_critical(&file->storage->storage, + "dbox_file_fix(%s): reopening file failed", + file->cur_path); + return -1; + } + return 1; +} diff --git a/src/lib-storage/index/dbox-common/dbox-file.c b/src/lib-storage/index/dbox-common/dbox-file.c new file mode 100644 index 0000000..16810b0 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-file.c @@ -0,0 +1,796 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "hex-dec.h" +#include "hex-binary.h" +#include "hostpid.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "file-dotlock.h" +#include "mkdir-parents.h" +#include "eacces-error.h" +#include "str.h" +#include "dbox-storage.h" +#include "dbox-file.h" + +#include <stdio.h> +#include <unistd.h> +#include <ctype.h> +#include <fcntl.h> + +#define DBOX_READ_BLOCK_SIZE IO_BLOCK_SIZE + +#ifndef DBOX_FILE_LOCK_METHOD_FLOCK +static const struct dotlock_settings dotlock_set = { + .stale_timeout = 60*10, + .use_excl_lock = TRUE +}; +#endif + +const char *dbox_generate_tmp_filename(void) +{ + static unsigned int create_count = 0; + + return t_strdup_printf(DBOX_TEMP_FILE_PREFIX"%"PRIdTIME_T".P%sQ%uM%u.%s", + ioloop_timeval.tv_sec, my_pid, + create_count++, + (unsigned int)ioloop_timeval.tv_usec, + my_hostname); +} + +void dbox_file_set_syscall_error(struct dbox_file *file, const char *function) +{ + mail_storage_set_critical(&file->storage->storage, + "%s failed for file %s: %m", + function, file->cur_path); +} + +void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) +{ + va_list args; + + va_start(args, reason); + mail_storage_set_critical(&file->storage->storage, + "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s", + file->cur_path, file->input == NULL ? 0 : file->input->v_offset, + t_strdup_vprintf(reason, args)); + va_end(args); + + file->storage->v.set_file_corrupted(file); +} + +void dbox_file_init(struct dbox_file *file) +{ + file->refcount = 1; + file->fd = -1; + file->cur_offset = UOFF_T_MAX; + file->cur_path = file->primary_path; +} + +void dbox_file_free(struct dbox_file *file) +{ + i_assert(file->refcount == 0); + + pool_unref(&file->metadata_pool); + dbox_file_close(file); + i_free(file->primary_path); + i_free(file->alt_path); + i_free(file); +} + +void dbox_file_unref(struct dbox_file **_file) +{ + struct dbox_file *file = *_file; + + *_file = NULL; + + i_assert(file->refcount > 0); + if (--file->refcount == 0) + file->storage->v.file_unrefed(file); +} + +static int dbox_file_parse_header(struct dbox_file *file, const char *line) +{ + const char *const *tmp, *value; + enum dbox_header_key key; + + file->file_version = *line - '0'; + if (!i_isdigit(line[0]) || line[1] != ' ' || + (file->file_version != 1 && file->file_version != DBOX_VERSION)) { + dbox_file_set_corrupted(file, "Invalid dbox version"); + return -1; + } + line += 2; + + file->msg_header_size = 0; + + for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) { + uintmax_t time; + key = **tmp; + value = *tmp + 1; + + switch (key) { + case DBOX_HEADER_OLDV1_APPEND_OFFSET: + break; + case DBOX_HEADER_MSG_HEADER_SIZE: + if (str_to_uint_hex(value, &file->msg_header_size) < 0) { + dbox_file_set_corrupted(file, "Invalid message header size"); + return -1; + } + break; + case DBOX_HEADER_CREATE_STAMP: + if (str_to_uintmax_hex(value, &time) < 0) { + dbox_file_set_corrupted(file, "Invalid create time stamp"); + return -1; + } + file->create_time = (time_t)time; + break; + } + } + + if (file->msg_header_size == 0) { + dbox_file_set_corrupted(file, "Missing message header size"); + return -1; + } + return 0; +} + +static int dbox_file_read_header(struct dbox_file *file) +{ + const char *line; + unsigned int hdr_size; + int ret; + + i_stream_seek(file->input, 0); + line = i_stream_read_next_line(file->input); + if (line == NULL) { + if (file->input->stream_errno == 0) { + dbox_file_set_corrupted(file, + "EOF while reading file header"); + return 0; + } + + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + hdr_size = file->input->v_offset; + T_BEGIN { + ret = dbox_file_parse_header(file, line) < 0 ? 0 : 1; + } T_END; + if (ret > 0) + file->file_header_size = hdr_size; + return ret; +} + +static int dbox_file_open_fd(struct dbox_file *file, bool try_altpath) +{ + const char *path; + int flags = O_RDWR; + bool alt = FALSE; + + /* try the primary path first */ + path = file->primary_path; + while ((file->fd = open(path, flags)) == -1) { + if (errno == EACCES && flags == O_RDWR) { + flags = O_RDONLY; + continue; + } + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "open(%s) failed: %m", path); + return -1; + } + + if (file->alt_path == NULL || alt || !try_altpath) { + /* not found */ + return 0; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + file->cur_path = path; + return 1; +} + +static int dbox_file_open_full(struct dbox_file *file, bool try_altpath, + bool *notfound_r) +{ + int ret, fd; + + *notfound_r = FALSE; + if (file->input != NULL) + return 1; + + if (file->fd == -1) { + T_BEGIN { + ret = dbox_file_open_fd(file, try_altpath); + } T_END; + if (ret <= 0) { + if (ret < 0) + return -1; + *notfound_r = TRUE; + return 1; + } + } + + /* we're manually checking at dbox_file_close() if we need to close the + fd or not. */ + fd = file->fd; + file->input = i_stream_create_fd_autoclose(&fd, DBOX_READ_BLOCK_SIZE); + i_stream_set_name(file->input, file->cur_path); + i_stream_set_init_buffer_size(file->input, DBOX_READ_BLOCK_SIZE); + return dbox_file_read_header(file); +} + +int dbox_file_open(struct dbox_file *file, bool *deleted_r) +{ + return dbox_file_open_full(file, TRUE, deleted_r); +} + +int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r) +{ + return dbox_file_open_full(file, FALSE, notfound_r); +} + +int dbox_file_stat(struct dbox_file *file, struct stat *st_r) +{ + const char *path; + bool alt = FALSE; + + if (dbox_file_is_open(file)) { + if (fstat(file->fd, st_r) < 0) { + mail_storage_set_critical(&file->storage->storage, + "fstat(%s) failed: %m", file->cur_path); + return -1; + } + return 0; + } + + /* try the primary path first */ + path = file->primary_path; + while (stat(path, st_r) < 0) { + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "stat(%s) failed: %m", path); + return -1; + } + + if (file->alt_path == NULL || alt) { + /* not found */ + return -1; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + file->cur_path = path; + return 0; +} + +int dbox_file_header_write(struct dbox_file *file, struct ostream *output) +{ + string_t *hdr; + + hdr = t_str_new(128); + str_printfa(hdr, "%u %c%x %c%x\n", DBOX_VERSION, + DBOX_HEADER_MSG_HEADER_SIZE, + (unsigned int)sizeof(struct dbox_message_header), + DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time); + + file->file_version = DBOX_VERSION; + file->file_header_size = str_len(hdr); + file->msg_header_size = sizeof(struct dbox_message_header); + return o_stream_send(output, str_data(hdr), str_len(hdr)); +} + +void dbox_file_close(struct dbox_file *file) +{ + dbox_file_unlock(file); + if (file->input != NULL) { + /* stream autocloses the fd when it gets destroyed. note that + the stream may outlive the struct dbox_file. */ + i_stream_unref(&file->input); + file->fd = -1; + } else if (file->fd != -1) { + if (close(file->fd) < 0) + dbox_file_set_syscall_error(file, "close()"); + file->fd = -1; + } + file->cur_offset = UOFF_T_MAX; +} + +int dbox_file_try_lock(struct dbox_file *file) +{ + const char *error; + int ret; + + i_assert(file->fd != -1); + +#ifdef DBOX_FILE_LOCK_METHOD_FLOCK + struct file_lock_settings lock_set = { + .lock_method = FILE_LOCK_METHOD_FLOCK, + }; + ret = file_try_lock(file->fd, file->cur_path, F_WRLCK, + &lock_set, &file->lock, &error); + if (ret < 0) { + mail_storage_set_critical(&file->storage->storage, + "file_try_lock(%s) failed: %s", file->cur_path, error); + } +#else + ret = file_dotlock_create(&dotlock_set, file->cur_path, + DOTLOCK_CREATE_FLAG_NONBLOCK, &file->lock); + if (ret < 0) { + mail_storage_set_critical(&file->storage->storage, + "file_dotlock_create(%s) failed: %m", file->cur_path); + } +#endif + return ret; +} + +void dbox_file_unlock(struct dbox_file *file) +{ + i_assert(!file->appending || file->lock == NULL); + + if (file->lock != NULL) { +#ifdef DBOX_FILE_LOCK_METHOD_FLOCK + file_unlock(&file->lock); +#else + file_dotlock_delete(&file->lock); +#endif + } + if (file->input != NULL) + i_stream_sync(file->input); +} + +int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) +{ + struct dbox_message_header hdr; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(file->input, &data, &size, + file->msg_header_size); + if (ret <= 0) { + if (file->input->stream_errno == 0) { + /* EOF, broken offset or file truncated */ + dbox_file_set_corrupted(file, "EOF reading msg header " + "(got %zu/%u bytes)", + size, file->msg_header_size); + return 0; + } + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size)); + if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) { + /* probably broken offset */ + dbox_file_set_corrupted(file, "msg header has bad magic value"); + return 0; + } + + if (data[file->msg_header_size-1] != '\n') { + dbox_file_set_corrupted(file, "msg header doesn't end with LF"); + return 0; + } + + *physical_size_r = hex2dec(hdr.message_size_hex, + sizeof(hdr.message_size_hex)); + return 1; +} + +int dbox_file_seek(struct dbox_file *file, uoff_t offset) +{ + uoff_t size; + int ret; + + i_assert(file->input != NULL); + + if (offset == 0) + offset = file->file_header_size; + + if (offset != file->cur_offset) { + i_stream_seek(file->input, offset); + ret = dbox_file_read_mail_header(file, &size); + if (ret <= 0) + return ret; + file->cur_offset = offset; + file->cur_physical_size = size; + } + i_stream_seek(file->input, offset + file->msg_header_size); + return 1; +} + +static int +dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset) +{ + const char *line; + size_t buf_size; + int ret; + + i_stream_seek(file->input, *offset); + if ((ret = dbox_file_metadata_skip_header(file)) <= 0) + return ret; + + /* skip over the actual metadata */ + buf_size = i_stream_get_max_buffer_size(file->input); + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + break; + } + } + i_stream_set_max_buffer_size(file->input, buf_size); + *offset = file->input->v_offset; + return 1; +} + +void dbox_file_seek_rewind(struct dbox_file *file) +{ + file->cur_offset = UOFF_T_MAX; +} + +int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r) +{ + uoff_t offset; + int ret; + + i_assert(file->input != NULL); + + if (file->cur_offset == UOFF_T_MAX) { + /* first mail. we may not have read the file at all yet, + so set the offset afterwards. */ + offset = 0; + } else { + offset = file->cur_offset + file->msg_header_size + + file->cur_physical_size; + if ((ret = dbox_file_seek_next_at_metadata(file, &offset)) <= 0) { + *offset_r = file->cur_offset; + return ret; + } + if (i_stream_read_eof(file->input)) { + *last_r = TRUE; + return 0; + } + } + *offset_r = offset; + + *last_r = FALSE; + + ret = dbox_file_seek(file, offset); + if (*offset_r == 0) + *offset_r = file->file_header_size; + return ret; +} + +struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file) +{ + struct dbox_file_append_context *ctx; + + i_assert(!file->appending); + + file->appending = TRUE; + + ctx = i_new(struct dbox_file_append_context, 1); + ctx->file = file; + if (file->fd != -1) { + ctx->output = o_stream_create_fd_file(file->fd, 0, FALSE); + o_stream_set_name(ctx->output, file->cur_path); + o_stream_set_finish_via_child(ctx->output, FALSE); + o_stream_cork(ctx->output); + } + return ctx; +} + +int dbox_file_append_commit(struct dbox_file_append_context **_ctx) +{ + struct dbox_file_append_context *ctx = *_ctx; + int ret; + + i_assert(ctx->file->appending); + + *_ctx = NULL; + + ret = dbox_file_append_flush(ctx); + if (ctx->last_checkpoint_offset != ctx->output->offset) { + o_stream_close(ctx->output); + if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "ftruncate()"); + return -1; + } + } + o_stream_unref(&ctx->output); + ctx->file->appending = FALSE; + i_free(ctx); + return ret; +} + +void dbox_file_append_rollback(struct dbox_file_append_context **_ctx) +{ + struct dbox_file_append_context *ctx = *_ctx; + struct dbox_file *file = ctx->file; + bool close_file = FALSE; + + i_assert(ctx->file->appending); + + *_ctx = NULL; + if (ctx->first_append_offset == 0) { + /* nothing changed */ + } else if (ctx->first_append_offset == file->file_header_size) { + /* rolling back everything */ + if (unlink(file->cur_path) < 0) + dbox_file_set_syscall_error(file, "unlink()"); + close_file = TRUE; + } else { + /* truncating only some mails */ + o_stream_close(ctx->output); + if (ftruncate(file->fd, ctx->first_append_offset) < 0) + dbox_file_set_syscall_error(file, "ftruncate()"); + } + if (ctx->output != NULL) { + o_stream_abort(ctx->output); + o_stream_unref(&ctx->output); + } + i_free(ctx); + + if (close_file) + dbox_file_close(file); + file->appending = FALSE; +} + +int dbox_file_append_flush(struct dbox_file_append_context *ctx) +{ + struct mail_storage *storage = &ctx->file->storage->storage; + + if (ctx->last_flush_offset == ctx->output->offset && + ctx->last_checkpoint_offset == ctx->output->offset) + return 0; + + if (o_stream_flush(ctx->output) < 0) { + dbox_file_set_syscall_error(ctx->file, "write()"); + return -1; + } + + if (ctx->last_checkpoint_offset != ctx->output->offset) { + if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "ftruncate()"); + return -1; + } + if (o_stream_seek(ctx->output, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "lseek()"); + return -1; + } + } + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync(ctx->file->fd) < 0) { + dbox_file_set_syscall_error(ctx->file, "fdatasync()"); + return -1; + } + } + ctx->last_flush_offset = ctx->output->offset; + return 0; +} + +void dbox_file_append_checkpoint(struct dbox_file_append_context *ctx) +{ + ctx->last_checkpoint_offset = ctx->output->offset; +} + +int dbox_file_get_append_stream(struct dbox_file_append_context *ctx, + struct ostream **output_r) +{ + struct dbox_file *file = ctx->file; + struct stat st; + + if (ctx->output == NULL) { + /* file creation had failed */ + return -1; + } + if (ctx->last_checkpoint_offset != ctx->output->offset) { + /* a message was aborted. don't try appending to this + file anymore. */ + return -1; + } + + if (file->file_version == 0) { + /* newly created file, write the file header */ + if (dbox_file_header_write(file, ctx->output) < 0) { + dbox_file_set_syscall_error(file, "write()"); + return -1; + } + *output_r = ctx->output; + return 1; + } + + /* file has existing mails */ + if (file->file_version != DBOX_VERSION || + file->msg_header_size != sizeof(struct dbox_message_header)) { + /* created by an incompatible version, can't append */ + return 0; + } + + if (ctx->output->offset == 0) { + /* first append to existing file. seek to eof first. */ + if (fstat(file->fd, &st) < 0) { + dbox_file_set_syscall_error(file, "fstat()"); + return -1; + } + if (st.st_size < file->msg_header_size) { + dbox_file_set_corrupted(file, + "dbox file size too small"); + return 0; + } + if (o_stream_seek(ctx->output, st.st_size) < 0) { + dbox_file_set_syscall_error(file, "lseek()"); + return -1; + } + } + *output_r = ctx->output; + return 1; +} + +int dbox_file_metadata_skip_header(struct dbox_file *file) +{ + struct dbox_metadata_header metadata_hdr; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(file->input, &data, &size, + sizeof(metadata_hdr)); + if (ret <= 0) { + if (file->input->stream_errno == 0) { + /* EOF, broken offset */ + dbox_file_set_corrupted(file, + "Unexpected EOF while reading metadata header"); + return 0; + } + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + memcpy(&metadata_hdr, data, sizeof(metadata_hdr)); + if (memcmp(metadata_hdr.magic_post, DBOX_MAGIC_POST, + sizeof(metadata_hdr.magic_post)) != 0) { + /* probably broken offset */ + dbox_file_set_corrupted(file, + "metadata header has bad magic value"); + return 0; + } + i_stream_skip(file->input, sizeof(metadata_hdr)); + return 1; +} + +static int +dbox_file_metadata_read_at(struct dbox_file *file, uoff_t metadata_offset) +{ + const char *line; + size_t buf_size; + int ret; + + if (file->metadata_pool != NULL) + p_clear(file->metadata_pool); + else { + file->metadata_pool = + pool_alloconly_create("dbox metadata", 1024); + } + p_array_init(&file->metadata, file->metadata_pool, 16); + + i_stream_seek(file->input, metadata_offset); + if ((ret = dbox_file_metadata_skip_header(file)) <= 0) + return ret; + + ret = 0; + buf_size = i_stream_get_max_buffer_size(file->input); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + ret = 1; + break; + } + line = p_strdup(file->metadata_pool, line); + array_push_back(&file->metadata, &line); + } + i_stream_set_max_buffer_size(file->input, buf_size); + if (ret == 0) + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return ret; +} + +int dbox_file_metadata_read(struct dbox_file *file) +{ + uoff_t metadata_offset; + int ret; + + i_assert(file->cur_offset != UOFF_T_MAX); + + if (file->metadata_read_offset == file->cur_offset) + return 1; + + metadata_offset = file->cur_offset + file->msg_header_size + + file->cur_physical_size; + ret = dbox_file_metadata_read_at(file, metadata_offset); + if (ret <= 0) + return ret; + + file->metadata_read_offset = file->cur_offset; + return 1; +} + +const char *dbox_file_metadata_get(struct dbox_file *file, + enum dbox_metadata_key key) +{ + const char *const *metadata; + unsigned int i, count; + + metadata = array_get(&file->metadata, &count); + for (i = 0; i < count; i++) { + if (*metadata[i] == (char)key) + return metadata[i] + 1; + } + return NULL; +} + +uoff_t dbox_file_get_plaintext_size(struct dbox_file *file) +{ + const char *value; + uintmax_t size; + + i_assert(file->metadata_read_offset == file->cur_offset); + + /* see if we have it in metadata */ + value = dbox_file_metadata_get(file, DBOX_METADATA_PHYSICAL_SIZE); + if (value == NULL || + str_to_uintmax_hex(value, &size) < 0 || + size > UOFF_T_MAX) { + /* no. that means we can use the size in the header */ + return file->cur_physical_size; + } + + return (uoff_t)size; +} + +void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, + uoff_t message_size) +{ + memset(dbox_msg_hdr, ' ', sizeof(*dbox_msg_hdr)); + memcpy(dbox_msg_hdr->magic_pre, DBOX_MAGIC_PRE, + sizeof(dbox_msg_hdr->magic_pre)); + dbox_msg_hdr->type = DBOX_MESSAGE_TYPE_NORMAL; + dec2hex(dbox_msg_hdr->message_size_hex, message_size, + sizeof(dbox_msg_hdr->message_size_hex)); + dbox_msg_hdr->save_lf = '\n'; +} + +int dbox_file_unlink(struct dbox_file *file) +{ + const char *path; + bool alt = FALSE; + + path = file->primary_path; + while (unlink(path) < 0) { + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "unlink(%s) failed: %m", path); + return -1; + } + if (file->alt_path == NULL || alt) { + /* not found */ + return 0; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + return 1; +} diff --git a/src/lib-storage/index/dbox-common/dbox-file.h b/src/lib-storage/index/dbox-common/dbox-file.h new file mode 100644 index 0000000..309c705 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-file.h @@ -0,0 +1,218 @@ +#ifndef DBOX_FILE_H +#define DBOX_FILE_H + +/* The file begins with a header followed by zero or more messages: + + <dbox message header> + <LF> + <message body> + <metadata> + + Metadata block begins with DBOX_MAGIC_POST, followed by zero or more lines + in format <key character><value><LF>. The block ends with an empty line. + Unknown metadata should be ignored, but preserved when copying. + + There should be no duplicates for the current metadata, but future + extensions may need them so they should be preserved. +*/ +#define DBOX_VERSION 2 +#define DBOX_MAGIC_PRE "\001\002" +#define DBOX_MAGIC_POST "\n\001\003\n" + +/* prefer flock(). fcntl() locking currently breaks if trying to access the + same file from multiple mail_storages within same process. that's why we + fallback to dotlocks. */ +#ifdef HAVE_FLOCK +# define DBOX_FILE_LOCK_METHOD_FLOCK +#endif + +struct dbox_file; +struct stat; + +enum dbox_header_key { + /* Must be sizeof(struct dbox_message_header) when appending (hex) */ + DBOX_HEADER_MSG_HEADER_SIZE = 'M', + /* Creation UNIX timestamp (hex) */ + DBOX_HEADER_CREATE_STAMP = 'C', + + /* metadata used by old Dovecot versions */ + DBOX_HEADER_OLDV1_APPEND_OFFSET = 'A' +}; + +/* NOTE: all valid keys are uppercase characters. if this changes, change + dbox-file-fix.c:dbox_file_match_post_magic() to recognize them */ +enum dbox_metadata_key { + /* Globally unique identifier for the message. Preserved when + copying. */ + DBOX_METADATA_GUID = 'G', + /* POP3 UIDL overriding the default format */ + DBOX_METADATA_POP3_UIDL = 'P', + /* POP3 message ordering (for migrated mails) */ + DBOX_METADATA_POP3_ORDER = 'O', + /* Received UNIX timestamp in hex */ + DBOX_METADATA_RECEIVED_TIME = 'R', + /* Physical message size in hex. Necessary only if it differs from + the dbox_message_header.message_size_hex, for example because the + message is compressed. */ + DBOX_METADATA_PHYSICAL_SIZE = 'Z', + /* Virtual message size in hex (line feeds counted as CRLF) */ + DBOX_METADATA_VIRTUAL_SIZE = 'V', + /* Pointer to external message data. Format is: + 1*(<start offset> <byte count> <options> <ref>) */ + DBOX_METADATA_EXT_REF = 'X', + /* Mailbox name where this message was originally saved to. + When rebuild finds a message whose mailbox is unknown, it's + placed to this mailbox. */ + DBOX_METADATA_ORIG_MAILBOX = 'B', + + /* metadata used by old Dovecot versions */ + DBOX_METADATA_OLDV1_EXPUNGED = 'E', + DBOX_METADATA_OLDV1_FLAGS = 'F', + DBOX_METADATA_OLDV1_KEYWORDS = 'K', + DBOX_METADATA_OLDV1_SAVE_TIME = 'S', + DBOX_METADATA_OLDV1_SPACE = ' ' +}; + +enum dbox_message_type { + /* Normal message */ + DBOX_MESSAGE_TYPE_NORMAL = 'N' +}; + +struct dbox_message_header { + unsigned char magic_pre[2]; + unsigned char type; + unsigned char space1; + unsigned char oldv1_uid_hex[8]; + unsigned char space2; + unsigned char message_size_hex[16]; + /* <space reserved for future extensions, LF is always last> */ + unsigned char save_lf; +}; + +struct dbox_metadata_header { + unsigned char magic_post[sizeof(DBOX_MAGIC_POST)-1]; +}; + +struct dbox_file { + struct dbox_storage *storage; + int refcount; + + time_t create_time; + unsigned int file_version; + unsigned int file_header_size; + unsigned int msg_header_size; + + const char *cur_path; + char *primary_path, *alt_path; + int fd; + struct istream *input; +#ifdef DBOX_FILE_LOCK_METHOD_FLOCK + struct file_lock *lock; +#else + struct dotlock *lock; +#endif + + uoff_t cur_offset; + uoff_t cur_physical_size; + + /* Metadata for the currently seeked metadata block. */ + pool_t metadata_pool; + ARRAY(const char *) metadata; + uoff_t metadata_read_offset; + + bool appending:1; + bool corrupted:1; +}; + +struct dbox_file_append_context { + struct dbox_file *file; + + uoff_t first_append_offset, last_checkpoint_offset, last_flush_offset; + struct ostream *output; +}; + +#define dbox_file_is_open(file) ((file)->fd != -1) +#define dbox_file_is_in_alt(file) ((file)->cur_path == (file)->alt_path) + +void dbox_file_init(struct dbox_file *file); +void dbox_file_unref(struct dbox_file **file); + +/* Open the file. Returns 1 if ok, 0 if file header is corrupted, -1 if error. + If file is deleted, deleted_r=TRUE and 1 is returned. */ +int dbox_file_open(struct dbox_file *file, bool *deleted_r); +/* Try to open file only from primary path. */ +int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r); +/* Close the file handle from the file, but don't free it. */ +void dbox_file_close(struct dbox_file *file); + +/* fstat() or stat() the file. If file is already deleted, fails with + errno=ENOENT. */ +int dbox_file_stat(struct dbox_file *file, struct stat *st_r); + +/* Try to lock the dbox file. Returns 1 if ok, 0 if already locked by someone + else, -1 if error. */ +int dbox_file_try_lock(struct dbox_file *file); +void dbox_file_unlock(struct dbox_file *file); + +/* Seek to given offset in file. Returns 1 if ok/expunged, 0 if file/offset is + corrupted, -1 if I/O error. */ +int dbox_file_seek(struct dbox_file *file, uoff_t offset); +/* Start seeking at the beginning of the file. */ +void dbox_file_seek_rewind(struct dbox_file *file); +/* Seek to next message after current one. If there are no more messages, + returns 0 and last_r is set to TRUE. Returns 1 if ok, 0 if file is + corrupted, -1 if I/O error. */ +int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r); + +/* Start appending to dbox file */ +struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file); +/* Finish writing appended mails. */ +int dbox_file_append_commit(struct dbox_file_append_context **ctx); +/* Truncate appended mails. */ +void dbox_file_append_rollback(struct dbox_file_append_context **ctx); +/* Get output stream for appending a new message. Returns 1 if ok, 0 if file + can't be appended to (old file version or corruption) or -1 if error. */ +int dbox_file_get_append_stream(struct dbox_file_append_context *ctx, + struct ostream **output_r); +/* Call after message has been fully saved. If this isn't done, the writes + since the last checkpoint are truncated. */ +void dbox_file_append_checkpoint(struct dbox_file_append_context *ctx); +/* Flush output buffer. */ +int dbox_file_append_flush(struct dbox_file_append_context *ctx); + +/* Read current message's metadata. Returns 1 if ok, 0 if metadata is + corrupted, -1 if I/O error. */ +int dbox_file_metadata_read(struct dbox_file *file); +/* Return wanted metadata value, or NULL if not found. */ +const char *dbox_file_metadata_get(struct dbox_file *file, + enum dbox_metadata_key key); + +/* Returns DBOX_METADATA_PHYSICAL_SIZE if set, otherwise physical size from + header. They differ only for e.g. compressed mails. */ +uoff_t dbox_file_get_plaintext_size(struct dbox_file *file); + +/* Fix a broken dbox file by rename()ing over it with a fixed file. Everything + before start_offset is assumed to be valid and is simply copied. The file + is reopened afterwards. Returns 1 if ok, 0 if the resulting file has no + mails and was deleted, -1 if I/O error. */ +int dbox_file_fix(struct dbox_file *file, uoff_t start_offset); +/* Delete the given dbox file. Returns 1 if deleted, 0 if file wasn't found + or -1 if error. */ +int dbox_file_unlink(struct dbox_file *file); + +/* Fill dbox_message_header with given size. */ +void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, + uoff_t message_size); + +void dbox_file_set_syscall_error(struct dbox_file *file, const char *function); +void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) + ATTR_FORMAT(2, 3); + +/* private: */ +const char *dbox_generate_tmp_filename(void); +void dbox_file_free(struct dbox_file *file); +int dbox_file_header_write(struct dbox_file *file, struct ostream *output); +int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r); +int dbox_file_metadata_skip_header(struct dbox_file *file); + +#endif diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c new file mode 100644 index 0000000..49279f9 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-mail.c @@ -0,0 +1,318 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "str.h" +#include "index-storage.h" +#include "index-mail.h" +#include "index-pop3-uidl.h" +#include "dbox-attachment.h" +#include "dbox-storage.h" +#include "dbox-file.h" +#include "dbox-mail.h" + + +struct mail * +dbox_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct dbox_mail *mail; + pool_t pool; + + pool = pool_alloconly_create("mail", 2048); + mail = p_new(pool, struct dbox_mail, 1); + + index_mail_init(&mail->imail, t, wanted_fields, wanted_headers, pool, NULL); + return &mail->imail.mail.mail; +} + +void dbox_mail_close(struct mail *_mail) +{ + struct dbox_mail *mail = DBOX_MAIL(_mail); + + index_mail_close(_mail); + /* close the dbox file only after index is closed, since it may still + try to read from it. */ + if (mail->open_file != NULL) + dbox_file_unref(&mail->open_file); +} + +int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r) +{ + struct dbox_storage *storage = + DBOX_STORAGE(mail->imail.mail.mail.box->storage); + uoff_t offset; + + if (storage->v.mail_open(mail, &offset, file_r) < 0) + return -1; + + if (dbox_file_seek(*file_r, offset) <= 0) + return -1; + if (dbox_file_metadata_read(*file_r) <= 0) + return -1; + + if (mail->imail.data.stream != NULL) { + /* we just messed up mail's input stream by reading metadata */ + i_stream_seek((*file_r)->input, offset); + i_stream_sync(mail->imail.data.stream); + } + return 0; +} + +static int +dbox_mail_metadata_get(struct dbox_mail *mail, enum dbox_metadata_key key, + const char **value_r) +{ + struct dbox_file *file; + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + *value_r = dbox_file_metadata_get(file, key); + return 0; +} + +int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct index_mail_data *data = &mail->imail.data; + struct dbox_file *file; + + if (index_mail_get_physical_size(_mail, size_r) == 0) + return 0; + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + data->physical_size = dbox_file_get_plaintext_size(file); + *size_r = data->physical_size; + return 0; +} + +int dbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) +{ + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct index_mail_data *data = &mail->imail.data; + const char *value; + uintmax_t size; + + if (index_mail_get_cached_virtual_size(&mail->imail, size_r)) + return 0; + + if (dbox_mail_metadata_get(mail, DBOX_METADATA_VIRTUAL_SIZE, + &value) < 0) + return -1; + if (value == NULL) + return index_mail_get_virtual_size(_mail, size_r); + + if (str_to_uintmax_hex(value, &size) < 0 || size > UOFF_T_MAX) + return -1; + data->virtual_size = (uoff_t)size; + *size_r = data->virtual_size; + return 0; +} + +int dbox_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct index_mail_data *data = &mail->imail.data; + const char *value; + uintmax_t time; + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (dbox_mail_metadata_get(mail, DBOX_METADATA_RECEIVED_TIME, + &value) < 0) + return -1; + + time = 0; + if (value != NULL && str_to_uintmax_hex(value, &time) < 0) + return -1; + + data->received_date = (time_t)time; + *date_r = data->received_date; + return 0; +} + +int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct dbox_storage *storage = DBOX_STORAGE(_mail->box->storage); + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct index_mail_data *data = &mail->imail.data; + struct dbox_file *file; + struct stat st; + uoff_t offset; + + if (index_mail_get_save_date(_mail, date_r) > 0) + return 1; + + if (storage->v.mail_open(mail, &offset, &file) < 0) + return -1; + + _mail->transaction->stats.fstat_lookup_count++; + if (dbox_file_stat(file, &st) < 0) { + if (errno == ENOENT) + mail_set_expunged(_mail); + return -1; + } + *date_r = data->save_date = st.st_ctime; + return 1; +} + +static int +dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key, + enum index_cache_field cache_field, + const char **value_r) +{ + struct index_mail *imail = &mail->imail; + struct index_mailbox_context *ibox = + INDEX_STORAGE_CONTEXT(imail->mail.mail.box); + const char *value; + string_t *str; + uint32_t order; + + str = str_new(imail->mail.data_pool, 64); + if (mail_cache_lookup_field(imail->mail.mail.transaction->cache_view, + str, imail->mail.mail.seq, + ibox->cache_fields[cache_field].idx) > 0) { + if (cache_field == MAIL_CACHE_POP3_ORDER) { + i_assert(str_len(str) == sizeof(order)); + memcpy(&order, str_data(str), sizeof(order)); + str_truncate(str, 0); + if (order != 0) + str_printfa(str, "%u", order); + else { + /* order=0 means it doesn't exist. we don't + want to return "0" though, because then the + mails get ordered to beginning, while + nonexistent are supposed to be ordered at + the end. */ + } + } + *value_r = str_c(str); + return 0; + } + + if (dbox_mail_metadata_get(mail, key, &value) < 0) + return -1; + + if (value == NULL) + value = ""; + if (cache_field != MAIL_CACHE_POP3_ORDER) { + index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx, + value, strlen(value)); + } else { + if (str_to_uint(value, &order) < 0) + order = 0; + index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx, + &order, sizeof(order)); + } + + /* don't return pointer to dbox metadata directly, since it may + change unexpectedly */ + str_truncate(str, 0); + str_append(str, value); + *value_r = str_c(str); + return 0; +} + +int dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct dbox_mail *mail = DBOX_MAIL(_mail); + int ret; + + /* keep the UIDL in cache file, otherwise POP3 would open all + mail files and read the metadata. same for GUIDs if they're + used. */ + switch (field) { + case MAIL_FETCH_UIDL_BACKEND: + if (!index_pop3_uidl_can_exist(_mail)) { + *value_r = ""; + return 0; + } + ret = dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, + MAIL_CACHE_POP3_UIDL, value_r); + if (ret == 0) { + index_pop3_uidl_update_exists(&mail->imail.mail.mail, + (*value_r)[0] != '\0'); + } + return ret; + case MAIL_FETCH_POP3_ORDER: + if (!index_pop3_uidl_can_exist(_mail)) { + /* we're assuming that if there's a POP3 order, there's + also a UIDL */ + *value_r = ""; + return 0; + } + return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_ORDER, + MAIL_CACHE_POP3_ORDER, value_r); + case MAIL_FETCH_GUID: + return dbox_get_cached_metadata(mail, DBOX_METADATA_GUID, + MAIL_CACHE_GUID, value_r); + default: + break; + } + + return index_mail_get_special(_mail, field, value_r); +} + +static int +get_mail_stream(struct dbox_mail *mail, uoff_t offset, + struct istream **stream_r) +{ + struct mail_private *pmail = &mail->imail.mail; + struct dbox_file *file = mail->open_file; + int ret; + + if ((ret = dbox_file_seek(file, offset)) <= 0) { + *stream_r = NULL; + return ret; + } + + *stream_r = i_stream_create_limit(file->input, file->cur_physical_size); + if (pmail->v.istream_opened != NULL) { + if (pmail->v.istream_opened(&pmail->mail, stream_r) < 0) + return -1; + } + if (file->storage->attachment_dir == NULL) + return 1; + else + return dbox_attachment_file_get_stream(file, stream_r); +} + +int dbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) +{ + struct dbox_storage *storage = DBOX_STORAGE(_mail->box->storage); + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct index_mail_data *data = &mail->imail.data; + struct istream *input; + uoff_t offset; + int ret; + + if (data->stream == NULL) { + if (storage->v.mail_open(mail, &offset, &mail->open_file) < 0) + return -1; + + ret = get_mail_stream(mail, offset, &input); + if (ret <= 0) { + if (ret < 0) + return -1; + dbox_file_set_corrupted(mail->open_file, + "uid=%u points to broken data at offset=" + "%"PRIuUOFF_T, _mail->uid, offset); + i_stream_unref(&input); + return -1; + } + data->stream = input; + index_mail_set_read_buffer_size(_mail, input); + } + + return index_mail_init_stream(&mail->imail, hdr_size, body_size, + stream_r); +} diff --git a/src/lib-storage/index/dbox-common/dbox-mail.h b/src/lib-storage/index/dbox-common/dbox-mail.h new file mode 100644 index 0000000..c03652c --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-mail.h @@ -0,0 +1,34 @@ +#ifndef DBOX_MAIL_H +#define DBOX_MAIL_H + +#include "index-mail.h" + +struct dbox_mail { + struct index_mail imail; + + struct dbox_file *open_file; + uoff_t offset; +}; + +#define DBOX_MAIL(s) container_of(s, struct dbox_mail, imail.mail.mail) + +struct mail * +dbox_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); +void dbox_mail_close(struct mail *mail); + +int dbox_mail_get_physical_size(struct mail *mail, uoff_t *size_r); +int dbox_mail_get_virtual_size(struct mail *mail, uoff_t *size_r); +int dbox_mail_get_received_date(struct mail *mail, time_t *date_r); +int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r); +int dbox_mail_get_special(struct mail *mail, enum mail_fetch_field field, + const char **value_r); +int dbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r); + +int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r); + +#endif diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c new file mode 100644 index 0000000..c5af8cc --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-save.c @@ -0,0 +1,226 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "str.h" +#include "hex-binary.h" +#include "index-mail.h" +#include "index-storage.h" +#include "dbox-attachment.h" +#include "dbox-file.h" +#include "dbox-save.h" + +void dbox_save_add_to_index(struct dbox_save_context *ctx) +{ + struct mail_save_data *mdata = &ctx->ctx.data; + enum mail_flags save_flags; + + save_flags = mdata->flags & ENUM_NEGATE(MAIL_RECENT); + mail_index_append(ctx->trans, mdata->uid, &ctx->seq); + mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE, + save_flags); + if (mdata->keywords != NULL) { + mail_index_update_keywords(ctx->trans, ctx->seq, + MODIFY_REPLACE, mdata->keywords); + } + if (mdata->min_modseq != 0) { + mail_index_update_modseq(ctx->trans, ctx->seq, + mdata->min_modseq); + } +} + +void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input) +{ + struct mail_save_context *_ctx = &ctx->ctx; + struct mail_storage *_storage = _ctx->transaction->box->storage; + struct dbox_storage *storage = DBOX_STORAGE(_storage); + struct dbox_message_header dbox_msg_hdr; + struct istream *crlf_input; + + dbox_save_add_to_index(ctx); + + mail_set_seq_saving(_ctx->dest_mail, ctx->seq); + + crlf_input = i_stream_create_lf(input); + ctx->input = index_mail_cache_parse_init(_ctx->dest_mail, crlf_input); + i_stream_unref(&crlf_input); + + /* write a dummy header. it'll get rewritten when we're finished */ + i_zero(&dbox_msg_hdr); + o_stream_cork(ctx->dbox_output); + if (o_stream_send(ctx->dbox_output, &dbox_msg_hdr, + sizeof(dbox_msg_hdr)) < 0) { + mail_set_critical(_ctx->dest_mail, "write(%s) failed: %s", + o_stream_get_name(ctx->dbox_output), + o_stream_get_error(ctx->dbox_output)); + ctx->failed = TRUE; + } + _ctx->data.output = ctx->dbox_output; + + if (_ctx->data.received_date == (time_t)-1) + _ctx->data.received_date = ioloop_time; + index_attachment_save_begin(_ctx, storage->attachment_fs, ctx->input); +} + +int dbox_save_continue(struct mail_save_context *_ctx) +{ + struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx); + + if (ctx->failed) + return -1; + + if (_ctx->data.attach != NULL) + return index_attachment_save_continue(_ctx); + + if (index_storage_save_continue(_ctx, ctx->input, + _ctx->dest_mail) < 0) { + ctx->failed = TRUE; + return -1; + } + return 0; +} + +void dbox_save_end(struct dbox_save_context *ctx) +{ + struct mail_save_data *mdata = &ctx->ctx.data; + struct ostream *dbox_output = ctx->dbox_output; + int ret; + + i_assert(mdata->output != NULL); + + if (mdata->attach != NULL && !ctx->failed) { + if (index_attachment_save_finish(&ctx->ctx) < 0) + ctx->failed = TRUE; + } + if (mdata->output != dbox_output) { + /* e.g. zlib plugin had changed this. make sure we + successfully write the trailer. */ + ret = o_stream_finish(mdata->output); + } else { + /* no plugins - flush the output so far */ + ret = o_stream_flush(mdata->output); + } + if (ret < 0) { + mail_set_critical(ctx->ctx.dest_mail, + "write(%s) failed: %s", + o_stream_get_name(mdata->output), + o_stream_get_error(mdata->output)); + ctx->failed = TRUE; + } + if (mdata->output != dbox_output) { + o_stream_ref(dbox_output); + o_stream_destroy(&mdata->output); + mdata->output = dbox_output; + } + index_mail_cache_parse_deinit(ctx->ctx.dest_mail, + ctx->ctx.data.received_date, + !ctx->failed); + if (!ctx->failed) + index_mail_cache_pop3_data(ctx->ctx.dest_mail, + mdata->pop3_uidl, + mdata->pop3_order); +} + +void dbox_save_write_metadata(struct mail_save_context *_ctx, + struct ostream *output, uoff_t output_msg_size, + const char *orig_mailbox_name, + guid_128_t guid_128) +{ + struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx); + struct mail_save_data *mdata = &ctx->ctx.data; + struct dbox_metadata_header metadata_hdr; + const char *guid; + string_t *str; + uoff_t vsize; + + i_zero(&metadata_hdr); + memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST, + sizeof(metadata_hdr.magic_post)); + o_stream_nsend(output, &metadata_hdr, sizeof(metadata_hdr)); + + str = t_str_new(256); + if (output_msg_size != ctx->input->v_offset) { + /* a plugin changed the data written to disk, so the + "message size" dbox header doesn't contain the actual + "physical" message size. we need to save it as a + separate metadata header. */ + str_printfa(str, "%c%llx\n", DBOX_METADATA_PHYSICAL_SIZE, + (unsigned long long)ctx->input->v_offset); + } + str_printfa(str, "%c%"PRIxTIME_T"\n", DBOX_METADATA_RECEIVED_TIME, + mdata->received_date); + if (mail_get_virtual_size(_ctx->dest_mail, &vsize) < 0) + i_unreached(); + str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, + (unsigned long long)vsize); + if (mdata->pop3_uidl != NULL) { + i_assert(strchr(mdata->pop3_uidl, '\n') == NULL); + str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL, + mdata->pop3_uidl); + ctx->have_pop3_uidls = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); + } + if (mdata->pop3_order != 0) { + str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER, + mdata->pop3_order); + ctx->have_pop3_orders = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); + } + + guid = mdata->guid; + if (guid != NULL) + mail_generate_guid_128_hash(guid, guid_128); + else { + guid_128_generate(guid_128); + guid = guid_128_to_string(guid_128); + } + str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid); + + if (orig_mailbox_name != NULL && + strchr(orig_mailbox_name, '\r') == NULL && + strchr(orig_mailbox_name, '\n') == NULL) { + /* save the original mailbox name so if mailbox indexes get + corrupted we can place at least some (hopefully most) of + the messages to correct mailboxes. */ + str_printfa(str, "%c%s\n", DBOX_METADATA_ORIG_MAILBOX, + orig_mailbox_name); + } + + dbox_attachment_save_write_metadata(_ctx, str); + + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); +} + +void dbox_save_update_header_flags(struct dbox_save_context *ctx, + struct mail_index_view *sync_view, + uint32_t ext_id, + unsigned int flags_offset) +{ + const void *data; + size_t data_size; + uint8_t old_flags = 0, flags; + + mail_index_get_header_ext(sync_view, ext_id, &data, &data_size); + if (flags_offset < data_size) + old_flags = *((const uint8_t *)data + flags_offset); + else { + /* grow old dbox header */ + mail_index_ext_resize_hdr(ctx->trans, ext_id, flags_offset+1); + } + + flags = old_flags; + if (ctx->have_pop3_uidls) + flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS; + if (ctx->have_pop3_orders) + flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS; + if (flags != old_flags) { + /* flags changed, update them */ + mail_index_update_header_ext(ctx->trans, ext_id, + flags_offset, &flags, 1); + } +} diff --git a/src/lib-storage/index/dbox-common/dbox-save.h b/src/lib-storage/index/dbox-common/dbox-save.h new file mode 100644 index 0000000..a17c923 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-save.h @@ -0,0 +1,41 @@ +#ifndef DBOX_SAVE_H +#define DBOX_SAVE_H + +#include "dbox-storage.h" + +struct dbox_save_context { + struct mail_save_context ctx; + struct mail_index_transaction *trans; + + /* updated for each appended mail: */ + uint32_t seq; + struct istream *input; + + struct ostream *dbox_output; + + uint32_t highest_pop3_uidl_seq; + bool failed:1; + bool finished:1; + bool have_pop3_uidls:1; + bool have_pop3_orders:1; +}; + +#define DBOX_SAVECTX(s) container_of(s, struct dbox_save_context, ctx) + +void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input); +int dbox_save_continue(struct mail_save_context *_ctx); +void dbox_save_end(struct dbox_save_context *ctx); + +void dbox_save_write_metadata(struct mail_save_context *ctx, + struct ostream *output, uoff_t output_msg_size, + const char *orig_mailbox_name, + guid_128_t guid_128_r) ATTR_NULL(4); + +void dbox_save_add_to_index(struct dbox_save_context *ctx); + +void dbox_save_update_header_flags(struct dbox_save_context *ctx, + struct mail_index_view *sync_view, + uint32_t ext_id, + unsigned int flags_offset); + +#endif diff --git a/src/lib-storage/index/dbox-common/dbox-storage.c b/src/lib-storage/index/dbox-common/dbox-storage.c new file mode 100644 index 0000000..3087174 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-storage.c @@ -0,0 +1,465 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "crc32.h" +#include "path-util.h" +#include "ioloop.h" +#include "fs-api.h" +#include "mkdir-parents.h" +#include "unlink-old-files.h" +#include "mailbox-uidvalidity.h" +#include "mailbox-list-private.h" +#include "index-storage.h" +#include "dbox-storage.h" + +#include <stdio.h> +#include <dirent.h> +#include <unistd.h> +#include <utime.h> + +void dbox_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_FS; + if (set->subscription_fname == NULL) + set->subscription_fname = DBOX_SUBSCRIPTION_FILE_NAME; + if (*set->maildir_name == '\0') + set->maildir_name = DBOX_MAILDIR_NAME; + if (*set->mailbox_dir_name == '\0') + set->mailbox_dir_name = DBOX_MAILBOX_DIR_NAME; +} + +static bool +dbox_alt_path_has_changed(const char *root_dir, const char *alt_path, + const char *alt_path2, const char *alt_symlink_path) +{ + const char *linkpath, *error; + + if (t_readlink(alt_symlink_path, &linkpath, &error) < 0) { + if (errno == ENOENT) + return alt_path != NULL; + i_error("t_readlink(%s) failed: %s", alt_symlink_path, error); + return FALSE; + } + + if (alt_path == NULL) { + i_warning("dbox %s: Original ALT=%s, " + "but currently no ALT path set", root_dir, linkpath); + return TRUE; + } else if (strcmp(linkpath, alt_path) != 0) { + if (strcmp(linkpath, alt_path2) == 0) { + /* FIXME: for backwards compatibility. old versions + created the symlink to mailboxes/ directory, which + was fine with sdbox, but didn't even exist with + mdbox. we'll silently replace the symlink. */ + return TRUE; + } + i_warning("dbox %s: Original ALT=%s, " + "but currently ALT=%s", root_dir, linkpath, alt_path); + return TRUE; + } + return FALSE; +} + +static void dbox_verify_alt_path(struct mailbox_list *list) +{ + const char *root_dir, *alt_symlink_path, *alt_path, *alt_path2; + + root_dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_DIR); + alt_symlink_path = + t_strconcat(root_dir, "/"DBOX_ALT_SYMLINK_NAME, NULL); + (void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_DIR, + &alt_path); + (void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX, + &alt_path2); + if (!dbox_alt_path_has_changed(root_dir, alt_path, alt_path2, + alt_symlink_path)) + return; + + /* unlink/create the current alt path symlink */ + i_unlink_if_exists(alt_symlink_path); + if (alt_path != NULL) { + int ret = symlink(alt_path, alt_symlink_path); + if (ret < 0 && errno == ENOENT) { + /* root_dir doesn't exist yet - create it */ + if (mailbox_list_mkdir_root(list, root_dir, + MAILBOX_LIST_PATH_TYPE_DIR) < 0) + return; + ret = symlink(alt_path, alt_symlink_path); + } + if (ret < 0 && errno != EEXIST) { + i_error("symlink(%s, %s) failed: %m", + alt_path, alt_symlink_path); + } + } +} + +int dbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, + const char **error_r) +{ + struct dbox_storage *storage = DBOX_STORAGE(_storage); + const struct mail_storage_settings *set = _storage->set; + const char *error; + + if (*set->mail_attachment_fs != '\0' && + *set->mail_attachment_dir != '\0') { + const char *name, *args, *dir; + + args = strpbrk(set->mail_attachment_fs, ": "); + if (args == NULL) { + name = set->mail_attachment_fs; + args = ""; + } else { + name = t_strdup_until(set->mail_attachment_fs, args++); + } + if (strcmp(name, "sis-queue") == 0 && + (_storage->class_flags & MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG) != 0) { + /* FIXME: the deduplication part doesn't work, because + sdbox renames the files.. */ + *error_r = "mail_attachment_fs: " + "sis-queue not currently supported by sdbox"; + return -1; + } + dir = mail_user_home_expand(_storage->user, + set->mail_attachment_dir); + storage->attachment_dir = p_strdup(_storage->pool, dir); + + if (mailbox_list_init_fs(ns->list, _storage->event, name, args, + storage->attachment_dir, + &storage->attachment_fs, &error) < 0) { + *error_r = t_strdup_printf("mail_attachment_fs: %s", + error); + return -1; + } + } + + if (!ns->list->set.alt_dir_nocheck) + dbox_verify_alt_path(ns->list); + return 0; +} + +void dbox_storage_destroy(struct mail_storage *_storage) +{ + struct dbox_storage *storage = DBOX_STORAGE(_storage); + + fs_deinit(&storage->attachment_fs); + index_storage_destroy(_storage); +} + +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"DBOX_UIDVALIDITY_FILE_NAME, NULL); + return mailbox_uidvalidity_next(list, path); +} + +void dbox_notify_changes(struct mailbox *box) +{ + const char *dir, *path; + + if (box->notify_callback == NULL) + mailbox_watch_remove_all(box); + else { + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, + &dir) <= 0) + return; + path = t_strdup_printf("%s/"MAIL_INDEX_PREFIX".log", dir); + mailbox_watch_add(box, path); + } +} + +static time_t cleanup_interval(struct mail_user *user) +{ + const struct mail_storage_settings *set = + mail_user_set_get_storage_set(user); + time_t interval = set->mail_temp_scan_interval; + + /* No need for a cryptographic-quality hash here. */ + unsigned int hash = crc32_str(user->username); + + /* spread from 0.00 to to 30.00% more than the base interval */ + unsigned int spread_factor = 100000 + hash % 30001; + return (interval * spread_factor) / 100000; +} + +static bool +dbox_cleanup_temp_files(struct mail_user *user, const char *path, + time_t last_scan_time, time_t last_change_time) +{ + /* check once in a while if there are temp files to clean up */ + time_t interval = cleanup_interval(user); + if (interval == 0) { + /* disabled */ + return FALSE; + } + + time_t deadline = ioloop_time - interval; + if (last_scan_time >= deadline) { + /* not the time to scan it yet */ + return FALSE; + } + + bool stated = FALSE; + if (last_change_time == (time_t)-1) { + /* Don't know the ctime yet - look it up. */ + struct stat st; + if (stat(path, &st) < 0) { + if (errno != ENOENT) + i_error("stat(%s) failed: %m", path); + return FALSE; + } + last_change_time = st.st_ctime; + stated = TRUE; + } + + if (last_scan_time > last_change_time + DBOX_TMP_DELETE_SECS) { + /* there haven't been any changes to this directory + since we last checked it. If we did an extra stat(), + we need to update the last_scan_time to avoid + stat()ing the next time. */ + return stated; + } + + (void)unlink_old_files(path, DBOX_TEMP_FILE_PREFIX, + ioloop_time - DBOX_TMP_DELETE_SECS); + return TRUE; +} + +int dbox_mailbox_check_existence(struct mailbox *box) +{ + const char *index_path, *box_path = mailbox_get_path(box); + struct stat st; + int ret = -1; + + if (box->list->set.iter_from_index_dir) { + /* Just because the index directory exists, it doesn't mean + that the mailbox is selectable. Check that by seeing if + dovecot.index.log exists. If it doesn't, fallback to + checking for the dbox-Mails in the mail root directory. + So this also means that if a mailbox is \NoSelect, listing + it will always do a stat() for dbox-Mails in the mail root + directory. That's not ideal, but this makes the behavior + safer and \NoSelect mailboxes are somewhat rare. */ + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, + &index_path) < 0) + return -1; + i_assert(index_path != NULL); + index_path = t_strconcat(index_path, "/", box->index_prefix, + ".log", NULL); + ret = stat(index_path, &st); + } + if (ret < 0) { + ret = stat(box_path, &st); + } + + if (ret == 0) { + return 0; + } else if (errno == ENOENT || errno == ENAMETOOLONG) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } else if (errno == EACCES) { + mailbox_set_critical(box, "%s", + mail_error_eacces_msg("stat", box_path)); + return -1; + } else { + mailbox_set_critical(box, "stat(%s) failed: %m", box_path); + return -1; + } +} + +int dbox_mailbox_open(struct mailbox *box) +{ + if (index_storage_mailbox_open(box, FALSE) < 0) + return -1; + mail_index_set_fsync_mode(box->index, + box->storage->set->parsed_fsync_mode, + MAIL_INDEX_FSYNC_MASK_APPENDS | + MAIL_INDEX_FSYNC_MASK_EXPUNGES); + return 0; +} + +int dbox_mailbox_list_cleanup(struct mail_user *user, const char *path, + time_t last_temp_file_scan) +{ + time_t change_time = -1; + + if (last_temp_file_scan == 0) { + /* Try to fetch the scan time from the directory's atime + if the directory exists. In case, get also the ctime */ + struct stat stats; + if (stat(path, &stats) == 0) { + last_temp_file_scan = stats.st_atim.tv_sec; + change_time = stats.st_ctim.tv_sec; + } else { + if (errno != ENOENT) + e_error(user->event, "stat(%s) failed: %m", path); + return -1; + } + } + + if (dbox_cleanup_temp_files(user, path, last_temp_file_scan, change_time) || + last_temp_file_scan == 0) { + /* temp files were scanned. update the last scan timestamp. */ + return 1; + } + return 0; +} + +void dbox_mailbox_close_cleanup(struct mailbox *box) +{ + if (box->view == NULL) + return; + + const struct mail_index_header *hdr = + mail_index_get_header(box->view); + if (dbox_mailbox_list_cleanup(box->storage->user, + mailbox_get_path(box), + hdr->last_temp_file_scan) > 0) + index_mailbox_update_last_temp_file_scan(box); +} + +void dbox_mailbox_close(struct mailbox *box) +{ + index_storage_mailbox_close(box); +} + +static int dir_is_empty(struct mail_storage *storage, const char *path) +{ + DIR *dir; + struct dirent *d; + int ret = 1; + + dir = opendir(path); + if (dir == NULL) { + if (errno == ENOENT) { + /* race condition with DELETE/RENAME? */ + return 1; + } + mail_storage_set_critical(storage, "opendir(%s) failed: %m", + path); + return -1; + } + while ((d = readdir(dir)) != NULL) { + if (*d->d_name == '.') + continue; + + ret = 0; + break; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(storage, "closedir(%s) failed: %m", + path); + ret = -1; + } + return ret; +} + +int dbox_mailbox_create(struct mailbox *box, + const struct mailbox_update *update, bool directory) +{ + struct dbox_storage *storage = DBOX_STORAGE(box->storage); + const char *alt_path; + struct stat st; + int ret; + + if ((ret = index_storage_mailbox_create(box, directory)) <= 0) + return ret; + if (mailbox_open(box) < 0) + return -1; + if (mail_index_get_header(box->view)->uid_validity != 0 && + !box->storage->rebuilding_list_index) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } + + /* if alt path already exists and contains files, rebuild storage so + that we don't start overwriting files. */ + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX, &alt_path); + if (ret > 0 && stat(alt_path, &st) == 0) { + ret = dir_is_empty(box->storage, alt_path); + if (ret < 0) + return -1; + if (ret == 0) { + mailbox_set_critical(box, + "Existing files in alt path, " + "rebuilding storage to avoid losing messages"); + storage->v.set_mailbox_corrupted(box); + return -1; + } + /* dir is empty, ignore it */ + } + if (dbox_mailbox_create_indexes(box, update) < 0) + return -1; + return index_mailbox_update_last_temp_file_scan(box); +} + +int dbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update) +{ + struct dbox_storage *storage = DBOX_STORAGE(box->storage); + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + int ret; + + /* use syncing as a lock */ + ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mailbox_set_index_error(box); + return -1; + } + + if (mail_index_get_header(view)->uid_validity == 0) { + if (storage->v.mailbox_create_indexes(box, update, trans) < 0) { + mail_index_sync_rollback(&sync_ctx); + return -1; + } + } + + return mail_index_sync_commit(&sync_ctx); +} + +int dbox_verify_alt_storage(struct mailbox_list *list) +{ + const char *alt_path; + struct stat st; + + if (!mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_DIR, + &alt_path)) + return 0; + + /* make sure alt storage is mounted. if it's not, abort the rebuild. */ + if (stat(alt_path, &st) == 0) + return 0; + if (errno != ENOENT) { + i_error("stat(%s) failed: %m", alt_path); + return -1; + } + + /* try to create the alt directory. if it fails, it means alt + storage isn't mounted. */ + if (mailbox_list_mkdir_root(list, alt_path, + MAILBOX_LIST_PATH_TYPE_ALT_DIR) < 0) + return -1; + return 0; +} + +bool dbox_header_have_flag(struct mailbox *box, uint32_t ext_id, + unsigned int flags_offset, uint8_t flag) +{ + const void *data; + size_t data_size; + uint8_t flags = 0; + + mail_index_get_header_ext(box->view, ext_id, &data, &data_size); + if (flags_offset < data_size) + flags = *((const uint8_t *)data + flags_offset); + return (flags & flag) != 0; +} diff --git a/src/lib-storage/index/dbox-common/dbox-storage.h b/src/lib-storage/index/dbox-common/dbox-storage.h new file mode 100644 index 0000000..8e8aaa1 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-storage.h @@ -0,0 +1,89 @@ +#ifndef DBOX_STORAGE_H +#define DBOX_STORAGE_H + +#include "mail-storage-private.h" + +struct dbox_file; +struct dbox_mail; +struct dbox_storage; +struct dbox_save_context; + +#define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions" +#define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity" +#define DBOX_TEMP_FILE_PREFIX ".temp." +#define DBOX_ALT_SYMLINK_NAME "dbox-alt-root" + +#define DBOX_MAILBOX_DIR_NAME "mailboxes" +#define DBOX_TRASH_DIR_NAME "trash" +#define DBOX_MAILDIR_NAME "dbox-Mails" + +/* Delete temp files having ctime older than this. */ +#define DBOX_TMP_DELETE_SECS (36*60*60) + +/* Flag specifies if the message should be in primary or alternative storage */ +#define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND + +enum dbox_index_header_flags { + /* messages' metadata contain POP3 UIDLs */ + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS = 0x01, + /* messages' metadata contain POP3 orders */ + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS = 0x02 +}; + +struct dbox_storage_vfuncs { + /* dbox file has zero references now. it should be either freed or + left open in case it's accessed again soon */ + void (*file_unrefed)(struct dbox_file *file); + /* create a new file using the same permissions as file. + if parents=TRUE, create the directory if necessary */ + int (*file_create_fd)(struct dbox_file *file, const char *path, + bool parents); + /* open the mail and return its file/offset */ + int (*mail_open)(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r); + /* create/update mailbox indexes */ + int (*mailbox_create_indexes)(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans); + /* returns attachment path suffix. mdbox returns "", sdbox returns + "-<mailbox_guid>-<uid>" */ + const char *(*get_attachment_path_suffix)(struct dbox_file *file); + /* mark the mailbox corrupted */ + void (*set_mailbox_corrupted)(struct mailbox *box); + /* mark the file corrupted */ + void (*set_file_corrupted)(struct dbox_file *file); +}; + +struct dbox_storage { + struct mail_storage storage; + struct dbox_storage_vfuncs v; + + struct fs *attachment_fs; + const char *attachment_dir; +}; + +#define DBOX_STORAGE(s) container_of(s, struct dbox_storage, storage) + +void dbox_storage_get_list_settings(const struct mail_namespace *ns, + struct mailbox_list_settings *set); +int dbox_storage_create(struct mail_storage *storage, + struct mail_namespace *ns, + const char **error_r); +void dbox_storage_destroy(struct mail_storage *storage); +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); +void dbox_notify_changes(struct mailbox *box); +int dbox_mailbox_check_existence(struct mailbox *box); +int dbox_mailbox_open(struct mailbox *box); +void dbox_mailbox_close(struct mailbox *box); +void dbox_mailbox_close_cleanup(struct mailbox *box); +int dbox_mailbox_list_cleanup(struct mail_user *user, const char *path, + time_t last_temp_file_scan); +int dbox_mailbox_create(struct mailbox *box, + const struct mailbox_update *update, bool directory); +int dbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update); +int dbox_verify_alt_storage(struct mailbox_list *list); +bool dbox_header_have_flag(struct mailbox *box, uint32_t ext_id, + unsigned int flags_offset, uint8_t flag); + +#endif diff --git a/src/lib-storage/index/dbox-multi/Makefile.am b/src/lib-storage/index/dbox-multi/Makefile.am new file mode 100644 index 0000000..810edd2 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/Makefile.am @@ -0,0 +1,36 @@ +noinst_LTLIBRARIES = libstorage_dbox_multi.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_multi_la_SOURCES = \ + mdbox-deleted-storage.c \ + mdbox-file.c \ + mdbox-mail.c \ + mdbox-map.c \ + mdbox-purge.c \ + mdbox-save.c \ + mdbox-settings.c \ + mdbox-sync.c \ + mdbox-storage.c \ + mdbox-storage-rebuild.c + +headers = \ + mdbox-file.h \ + mdbox-map.h \ + mdbox-map-private.h \ + mdbox-settings.h \ + mdbox-storage.h \ + mdbox-storage-rebuild.h \ + mdbox-sync.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/dbox-multi/Makefile.in b/src/lib-storage/index/dbox-multi/Makefile.in new file mode 100644 index 0000000..bf482ae --- /dev/null +++ b/src/lib-storage/index/dbox-multi/Makefile.in @@ -0,0 +1,866 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/dbox-multi +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_dbox_multi_la_LIBADD = +am_libstorage_dbox_multi_la_OBJECTS = mdbox-deleted-storage.lo \ + mdbox-file.lo mdbox-mail.lo mdbox-map.lo mdbox-purge.lo \ + mdbox-save.lo mdbox-settings.lo mdbox-sync.lo mdbox-storage.lo \ + mdbox-storage-rebuild.lo +libstorage_dbox_multi_la_OBJECTS = \ + $(am_libstorage_dbox_multi_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/mdbox-deleted-storage.Plo \ + ./$(DEPDIR)/mdbox-file.Plo ./$(DEPDIR)/mdbox-mail.Plo \ + ./$(DEPDIR)/mdbox-map.Plo ./$(DEPDIR)/mdbox-purge.Plo \ + ./$(DEPDIR)/mdbox-save.Plo ./$(DEPDIR)/mdbox-settings.Plo \ + ./$(DEPDIR)/mdbox-storage-rebuild.Plo \ + ./$(DEPDIR)/mdbox-storage.Plo ./$(DEPDIR)/mdbox-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_dbox_multi_la_SOURCES) +DIST_SOURCES = $(libstorage_dbox_multi_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_dbox_multi.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_multi_la_SOURCES = \ + mdbox-deleted-storage.c \ + mdbox-file.c \ + mdbox-mail.c \ + mdbox-map.c \ + mdbox-purge.c \ + mdbox-save.c \ + mdbox-settings.c \ + mdbox-sync.c \ + mdbox-storage.c \ + mdbox-storage-rebuild.c + +headers = \ + mdbox-file.h \ + mdbox-map.h \ + mdbox-map-private.h \ + mdbox-settings.h \ + mdbox-storage.h \ + mdbox-storage-rebuild.h \ + mdbox-sync.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/dbox-multi/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/dbox-multi/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_dbox_multi.la: $(libstorage_dbox_multi_la_OBJECTS) $(libstorage_dbox_multi_la_DEPENDENCIES) $(EXTRA_libstorage_dbox_multi_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_dbox_multi_la_OBJECTS) $(libstorage_dbox_multi_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-deleted-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-map.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-purge.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-storage-rebuild.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/mdbox-deleted-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-file.Plo + -rm -f ./$(DEPDIR)/mdbox-mail.Plo + -rm -f ./$(DEPDIR)/mdbox-map.Plo + -rm -f ./$(DEPDIR)/mdbox-purge.Plo + -rm -f ./$(DEPDIR)/mdbox-save.Plo + -rm -f ./$(DEPDIR)/mdbox-settings.Plo + -rm -f ./$(DEPDIR)/mdbox-storage-rebuild.Plo + -rm -f ./$(DEPDIR)/mdbox-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/mdbox-deleted-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-file.Plo + -rm -f ./$(DEPDIR)/mdbox-mail.Plo + -rm -f ./$(DEPDIR)/mdbox-map.Plo + -rm -f ./$(DEPDIR)/mdbox-purge.Plo + -rm -f ./$(DEPDIR)/mdbox-save.Plo + -rm -f ./$(DEPDIR)/mdbox-settings.Plo + -rm -f ./$(DEPDIR)/mdbox-storage-rebuild.Plo + -rm -f ./$(DEPDIR)/mdbox-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c new file mode 100644 index 0000000..4fdf1ab --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c @@ -0,0 +1,319 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "master-service.h" +#include "mail-index-modseq.h" +#include "mail-index-alloc-cache.h" +#include "mailbox-log.h" +#include "mailbox-list-private.h" +#include "mail-copy.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-storage.h" + +extern struct mail_storage mdbox_deleted_storage; +extern struct mailbox mdbox_deleted_mailbox; +extern struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs; + +static struct mail_storage *mdbox_deleted_storage_alloc(void) +{ + struct mdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mdbox deleted storage", 2048); + storage = p_new(pool, struct mdbox_storage, 1); + storage->storage.v = mdbox_dbox_storage_vfuncs; + storage->storage.storage = mdbox_deleted_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +static struct mailbox * +mdbox_deleted_mailbox_alloc(struct mail_storage *storage, + struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mdbox_mailbox *mbox; + pool_t pool; + + flags |= MAILBOX_FLAG_READONLY | MAILBOX_FLAG_NO_INDEX_FILES; + + pool = pool_alloconly_create("mdbox deleted mailbox", 1024*3); + mbox = p_new(pool, struct mdbox_mailbox, 1); + mbox->box = mdbox_deleted_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mdbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = MDBOX_STORAGE(storage); + return &mbox->box; +} + +static int +mdbox_deleted_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + struct mail_index_transaction *new_trans = NULL; + uint32_t uid_validity = ioloop_time; + uint32_t uid_next = 1; + + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + + if (trans == NULL) { + new_trans = mail_index_transaction_begin(box->view, 0); + trans = new_trans; + } + + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + mbox->creating = TRUE; + mdbox_update_header(mbox, trans, update); + mbox->creating = FALSE; + + if (new_trans != NULL) { + if (mail_index_transaction_commit(&new_trans) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + return 0; +} + +static const char * +mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) +{ + return ""; +} + +static int +mdbox_deleted_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + + if ((items & MAILBOX_METADATA_GUID) != 0) + guid_128_generate(metadata_r->guid); + return 0; +} + +static struct mail_save_context * +mdbox_deleted_save_alloc(struct mailbox_transaction_context *t) +{ + struct mail_save_context *ctx; + + ctx = i_new(struct mail_save_context, 1); + ctx->transaction = t; + return ctx; +} + +static int +mdbox_deleted_save_begin(struct mail_save_context *ctx, + struct istream *input ATTR_UNUSED) +{ + mail_storage_set_error(ctx->transaction->box->storage, + MAIL_ERROR_NOTPOSSIBLE, "mdbox_deleted doesn't support saving mails"); + return -1; +} + +static int +mdbox_deleted_save_continue(struct mail_save_context *ctx ATTR_UNUSED) +{ + return -1; +} + +static int mdbox_deleted_save_finish(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); + return -1; +} + +static void +mdbox_deleted_save_cancel(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); +} + +static int mdbox_deleted_sync(struct mdbox_mailbox *mbox, + enum mdbox_sync_flags flags ATTR_UNUSED) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mdbox_mail_index_record rec; + struct mdbox_map_mail_index_record map_rec; + enum mail_index_sync_flags sync_flags; + uint16_t refcount; + uint32_t map_seq, map_count, seq, uid = 0; + int ret = 0; + + if (mbox->mdbox_deleted_synced) { + /* don't bother supporting incremental syncs */ + return 0; + } + if (!mbox->box.inbox_user && mbox->box.name[0] != '\0') { + /* since mailbox list currently shows all the existing + mailboxes, we don't want all of them to list the deleted + messages. only show messages in user's INBOX or the + namespace prefix. */ + return 0; + } + + if (mdbox_map_open(mbox->storage->map) < 0) + return -1; + + if (mdbox_deleted_mailbox_create_indexes(&mbox->box, NULL, NULL) < 0) + return -1; + + i_zero(&rec); + rec.save_date = ioloop_time; + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if (mail_index_sync_begin(mbox->box.index, &index_sync_ctx, + &sync_view, &trans, sync_flags) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + map_count = mdbox_map_get_messages_count(mbox->storage->map); + for (map_seq = 1; map_seq <= map_count; map_seq++) { + if (mdbox_map_lookup_seq_full(mbox->storage->map, map_seq, + &map_rec, &refcount) < 0) { + ret = -1; + break; + } + if (refcount == 0) { + rec.map_uid = mdbox_map_lookup_uid(mbox->storage->map, + map_seq); + mail_index_append(trans, ++uid, &seq); + mail_index_update_ext(trans, seq, + mbox->ext_id, &rec, NULL); + } + } + + if (ret < 0) + mail_index_sync_rollback(&index_sync_ctx); + else { + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } else { + mbox->mdbox_deleted_synced = TRUE; + } + } + return ret; +} + +static struct mailbox_sync_context * +mdbox_deleted_storage_sync_init(struct mailbox *box, + enum mailbox_sync_flags flags) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + enum mdbox_sync_flags mdbox_sync_flags = 0; + int ret = 0; + + if (index_mailbox_want_full_sync(&mbox->box, flags) || + mbox->storage->corrupted) + ret = mdbox_deleted_sync(mbox, mdbox_sync_flags); + + return index_mailbox_sync_init(box, flags, ret < 0); +} + +struct mail_storage mdbox_deleted_storage = { + .name = MDBOX_DELETED_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + + .v = { + mdbox_get_setting_parser_info, + mdbox_deleted_storage_alloc, + mdbox_storage_create, + mdbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + NULL, + mdbox_deleted_mailbox_alloc, + NULL, + NULL, + } +}; + +struct mailbox mdbox_deleted_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mdbox_mailbox_open, + index_storage_mailbox_close, + index_storage_mailbox_free, + dbox_mailbox_create, + index_storage_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mdbox_deleted_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + mdbox_deleted_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + dbox_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + mdbox_deleted_save_alloc, + mdbox_deleted_save_begin, + mdbox_deleted_save_continue, + mdbox_deleted_save_finish, + mdbox_deleted_save_cancel, + mail_storage_copy, + NULL, + NULL, + NULL, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs = { + mdbox_file_unrefed, + mdbox_file_create_fd, + mdbox_mail_open, + mdbox_deleted_mailbox_create_indexes, + mdbox_get_attachment_path_suffix, + mdbox_set_mailbox_corrupted, + mdbox_set_file_corrupted +}; diff --git a/src/lib-storage/index/dbox-multi/mdbox-file.c b/src/lib-storage/index/dbox-multi/mdbox-file.c new file mode 100644 index 0000000..65138bd --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-file.c @@ -0,0 +1,349 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "hex-dec.h" +#include "hex-binary.h" +#include "hostpid.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "file-set-size.h" +#include "mkdir-parents.h" +#include "fdatasync-path.h" +#include "eacces-error.h" +#include "str.h" +#include "mailbox-list-private.h" +#include "mdbox-storage.h" +#include "mdbox-map-private.h" +#include "mdbox-file.h" + +#include <stdio.h> +#include <unistd.h> +#include <ctype.h> +#include <fcntl.h> + +static struct mdbox_file * +mdbox_find_and_move_open_file(struct mdbox_storage *storage, uint32_t file_id) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) { + if (files[i]->file_id == file_id) + return files[i]; + } + return NULL; +} + +void mdbox_files_free(struct mdbox_storage *storage) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) + dbox_file_free(&files[i]->file); + array_clear(&storage->open_files); +} + +void mdbox_files_sync_input(struct mdbox_storage *storage) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) { + if (files[i]->file.input != NULL) + i_stream_sync(files[i]->file.input); + } +} + +static void +mdbox_close_open_files(struct mdbox_storage *storage, unsigned int close_count) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count;) { + if (files[i]->file.refcount == 0) { + dbox_file_free(&files[i]->file); + array_delete(&storage->open_files, i, 1); + + if (--close_count == 0) + break; + + files = array_get(&storage->open_files, &count); + } else { + i++; + } + } +} + +static void +mdbox_file_init_paths(struct mdbox_file *file, const char *fname, bool alt) +{ + i_free(file->file.primary_path); + i_free(file->file.alt_path); + file->file.primary_path = + i_strdup_printf("%s/%s", file->storage->storage_dir, fname); + if (file->storage->alt_storage_dir != NULL) { + file->file.alt_path = + i_strdup_printf("%s/%s", file->storage->alt_storage_dir, + fname); + } + file->file.cur_path = !alt ? file->file.primary_path : + file->file.alt_path; +} + +static int mdbox_file_create(struct mdbox_file *file) +{ + struct dbox_file *_file = &file->file; + bool create_parents; + int ret; + + create_parents = dbox_file_is_in_alt(_file); + _file->fd = _file->storage->v. + file_create_fd(_file, _file->cur_path, create_parents); + if (_file->fd == -1) + return -1; + + if (file->storage->preallocate_space) { + ret = file_preallocate(_file->fd, + file->storage->set->mdbox_rotate_size); + if (ret < 0) { + switch (errno) { + case ENOSPC: + case EDQUOT: + /* ignore */ + break; + default: + i_error("file_preallocate(%s) failed: %m", + _file->cur_path); + break; + } + } else if (ret == 0) { + /* not supported by filesystem, disable. */ + file->storage->preallocate_space = FALSE; + } + } + return 0; +} + +static struct dbox_file * +mdbox_file_init_full(struct mdbox_storage *storage, + uint32_t file_id, bool alt_dir) +{ + struct mdbox_file *file; + const char *fname; + unsigned int count; + + file = file_id == 0 ? NULL : + mdbox_find_and_move_open_file(storage, file_id); + if (file != NULL) { + file->file.refcount++; + return &file->file; + } + + count = array_count(&storage->open_files); + if (count > MDBOX_MAX_OPEN_UNUSED_FILES) { + mdbox_close_open_files(storage, + count - MDBOX_MAX_OPEN_UNUSED_FILES); + } + + file = i_new(struct mdbox_file, 1); + file->storage = storage; + file->file.storage = &storage->storage; + file->file_id = file_id; + fname = file_id == 0 ? dbox_generate_tmp_filename() : + t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id); + mdbox_file_init_paths(file, fname, FALSE); + dbox_file_init(&file->file); + if (alt_dir) + file->file.cur_path = file->file.alt_path; + + if (file_id != 0) + array_push_back(&storage->open_files, &file); + else + (void)mdbox_file_create(file); + return &file->file; +} + +struct dbox_file * +mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id) +{ + return mdbox_file_init_full(storage, file_id, FALSE); +} + +struct dbox_file * +mdbox_file_init_new_alt(struct mdbox_storage *storage) +{ + return mdbox_file_init_full(storage, 0, TRUE); +} + +int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id) +{ + struct stat st; + const char *old_path; + const char *new_dir, *new_fname, *new_path; + + i_assert(file->file_id == 0); + i_assert(file_id != 0); + + old_path = file->file.cur_path; + new_fname = t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id); + new_dir = !dbox_file_is_in_alt(&file->file) ? + file->storage->storage_dir : file->storage->alt_storage_dir; + new_path = t_strdup_printf("%s/%s", new_dir, new_fname); + + if (stat(new_path, &st) == 0) { + mail_storage_set_critical(&file->file.storage->storage, + "mdbox: %s already exists, rebuilding index", new_path); + mdbox_storage_set_corrupted(file->storage); + return -1; + } + if (rename(old_path, new_path) < 0) { + mail_storage_set_critical(&file->storage->storage.storage, + "rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + mdbox_file_init_paths(file, new_fname, + dbox_file_is_in_alt(&file->file)); + file->file_id = file_id; + array_push_back(&file->storage->open_files, &file); + return 0; +} + +static struct mdbox_file * +mdbox_find_oldest_unused_file(struct mdbox_storage *storage, + unsigned int *idx_r) +{ + struct mdbox_file *const *files, *oldest_file = NULL; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + *idx_r = count; + for (i = 0; i < count; i++) { + if (files[i]->file.refcount == 0) { + if (oldest_file == NULL || + files[i]->close_time < oldest_file->close_time) { + oldest_file = files[i]; + *idx_r = i; + } + } + } + return oldest_file; +} + +static void mdbox_file_close_timeout(struct mdbox_storage *storage) +{ + struct mdbox_file *oldest; + unsigned int i; + time_t close_time = ioloop_time - MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS; + + while ((oldest = mdbox_find_oldest_unused_file(storage, &i)) != NULL) { + if (oldest->close_time > close_time) + break; + array_delete(&storage->open_files, i, 1); + dbox_file_free(&oldest->file); + } + + if (oldest == NULL) + timeout_remove(&storage->to_close_unused_files); +} + +static void mdbox_file_close_later(struct mdbox_file *mfile) +{ + if (mfile->storage->to_close_unused_files == NULL) { + mfile->storage->to_close_unused_files = + timeout_add(MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS*1000, + mdbox_file_close_timeout, mfile->storage); + } +} + +void mdbox_file_unrefed(struct dbox_file *file) +{ + struct mdbox_file *mfile = (struct mdbox_file *)file; + struct mdbox_file *oldest_file; + unsigned int i, count; + + /* don't cache metadata seeks while file isn't being referenced */ + file->metadata_read_offset = UOFF_T_MAX; + mfile->close_time = ioloop_time; + + if (mfile->file_id != 0) { + count = array_count(&mfile->storage->open_files); + if (count <= MDBOX_MAX_OPEN_UNUSED_FILES) { + /* we can leave this file open for now */ + mdbox_file_close_later(mfile); + return; + } + + /* close the oldest file with refcount=0 */ + oldest_file = mdbox_find_oldest_unused_file(mfile->storage, &i); + i_assert(oldest_file != NULL); + array_delete(&mfile->storage->open_files, i, 1); + if (oldest_file != mfile) { + dbox_file_free(&oldest_file->file); + mdbox_file_close_later(mfile); + return; + } + /* have to close ourself */ + } + dbox_file_free(file); +} + +int mdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents) +{ + struct mdbox_file *mfile = (struct mdbox_file *)file; + struct mdbox_map *map = mfile->storage->map; + struct mailbox_permissions perm; + mode_t old_mask; + const char *p, *dir; + int fd; + + mailbox_list_get_root_permissions(map->root_list, &perm); + + old_mask = umask(0666 & ~perm.file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + if (fd == -1 && errno == ENOENT && parents && + (p = strrchr(path, '/')) != NULL) { + dir = t_strdup_until(path, p); + if (mailbox_list_mkdir_root(map->root_list, dir, + path != file->alt_path ? + MAILBOX_LIST_PATH_TYPE_DIR : + MAILBOX_LIST_PATH_TYPE_ALT_DIR) < 0) { + mail_storage_copy_list_error(&file->storage->storage, + map->root_list); + return -1; + } + /* try again */ + old_mask = umask(0666 & ~perm.file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + } + if (fd == -1) { + mail_storage_set_critical(&file->storage->storage, + "open(%s, O_CREAT) failed: %m", path); + } else if (perm.file_create_gid == (gid_t)-1) { + /* no group change */ + } else if (fchown(fd, (uid_t)-1, perm.file_create_gid) < 0) { + if (errno == EPERM) { + mail_storage_set_critical(&file->storage->storage, "%s", + eperm_error_get_chgrp("fchown", path, + perm.file_create_gid, + perm.file_create_gid_origin)); + } else { + mail_storage_set_critical(&file->storage->storage, + "fchown(%s, -1, %ld) failed: %m", + path, (long)perm.file_create_gid); + } + /* continue anyway */ + } + return fd; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-file.h b/src/lib-storage/index/dbox-multi/mdbox-file.h new file mode 100644 index 0000000..9095cce --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-file.h @@ -0,0 +1,29 @@ +#ifndef MDBOX_FILE_H +#define MDBOX_FILE_H + +#include "dbox-file.h" + +struct mdbox_file { + struct dbox_file file; + struct mdbox_storage *storage; + + uint32_t file_id; + time_t close_time; +}; + +struct dbox_file * +mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id); +struct dbox_file * +mdbox_file_init_new_alt(struct mdbox_storage *storage); + +/* Assign file ID for a newly created file. */ +int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id); + +void mdbox_file_unrefed(struct dbox_file *file); +int mdbox_file_create_fd(struct dbox_file *file, const char *path, + bool parents); + +void mdbox_files_free(struct mdbox_storage *storage); +void mdbox_files_sync_input(struct mdbox_storage *storage); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-mail.c b/src/lib-storage/index/dbox-multi/mdbox-mail.c new file mode 100644 index 0000000..ed3fe06 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c @@ -0,0 +1,265 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "str.h" +#include "index-mail.h" +#include "dbox-mail.h" +#include "mdbox-storage.h" +#include "mdbox-sync.h" +#include "mdbox-map.h" +#include "mdbox-file.h" + +#include <sys/stat.h> + +int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, uint32_t *map_uid_r) +{ + const struct mdbox_mail_index_record *dbox_rec; + struct mdbox_index_header hdr; + const void *data; + uint32_t uid, cur_map_uid_validity; + bool need_resize; + + mail_index_lookup_ext(view, seq, mbox->ext_id, &data, NULL); + dbox_rec = data; + if (dbox_rec == NULL || dbox_rec->map_uid == 0) { + mail_index_lookup_uid(view, seq, &uid); + mailbox_set_critical(&mbox->box, + "mdbox: map uid lost for uid %u", uid); + mdbox_storage_set_corrupted(mbox->storage); + return -1; + } + + if (mbox->map_uid_validity == 0) { + if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) + return -1; + mbox->map_uid_validity = hdr.map_uid_validity; + } + if (mdbox_map_open_or_create(mbox->storage->map) < 0) + return -1; + + cur_map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map); + if (cur_map_uid_validity != mbox->map_uid_validity) { + mailbox_set_critical(&mbox->box, + "mdbox: map uidvalidity mismatch (%u vs %u)", + mbox->map_uid_validity, cur_map_uid_validity); + mdbox_storage_set_corrupted(mbox->storage); + return -1; + } + *map_uid_r = dbox_rec->map_uid; + return 0; +} + +static void dbox_mail_set_expunged(struct dbox_mail *mail, uint32_t map_uid) +{ + struct mail *_mail = &mail->imail.mail.mail; + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->box); + + mail_index_refresh(_mail->box->index); + if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) { + mail_set_expunged(_mail); + return; + } + + mdbox_map_set_corrupted(mbox->storage->map, + "Unexpectedly lost %s uid=%u map_uid=%u", + mailbox_get_vname(_mail->box), + _mail->uid, map_uid); +} + +static int dbox_mail_open_init(struct dbox_mail *mail, uint32_t map_uid) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->imail.mail.mail.box); + uint32_t file_id; + int ret; + + if ((ret = mdbox_map_lookup(mbox->storage->map, map_uid, + &file_id, &mail->offset)) <= 0) { + if (ret < 0) + return -1; + + /* map_uid doesn't exist anymore. either it + got just expunged or the map index is + corrupted. */ + dbox_mail_set_expunged(mail, map_uid); + return -1; + } else { + mail->open_file = mdbox_file_init(mbox->storage, file_id); + } + return 0; +} + +int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r) +{ + struct mail *_mail = &mail->imail.mail.mail; + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->box); + uint32_t prev_file_id = 0, map_uid = 0; + bool deleted; + + if (!mail_stream_access_start(_mail)) + return -1; + + do { + if (mail->open_file != NULL) { + /* already open */ + } else if (!_mail->saving) { + if (mdbox_mail_lookup(mbox, _mail->transaction->view, + _mail->seq, &map_uid) < 0) + return -1; + if (dbox_mail_open_init(mail, map_uid) < 0) + return -1; + } else { + /* mail is being saved in this transaction */ + mail->open_file = + mdbox_save_file_get_file(_mail->transaction, + _mail->seq, + &mail->offset); + } + + if (!dbox_file_is_open(mail->open_file)) + _mail->transaction->stats.open_lookup_count++; + if (dbox_file_open(mail->open_file, &deleted) <= 0) + return -1; + if (deleted) { + /* either it's expunged now or moved to another file. */ + struct mdbox_file *mfile = + (struct mdbox_file *)mail->open_file; + + if (mfile->file_id == prev_file_id) { + dbox_mail_set_expunged(mail, map_uid); + return -1; + } + prev_file_id = mfile->file_id; + if (mdbox_map_refresh(mbox->storage->map) < 0) + return -1; + dbox_file_unref(&mail->open_file); + } + } while (mail->open_file == NULL); + + *file_r = mail->open_file; + *offset_r = mail->offset; + return 0; +} + +static int mdbox_mail_get_save_date(struct mail *mail, time_t *date_r) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->transaction->box); + const struct mdbox_mail_index_record *dbox_rec; + const void *data; + + mail_index_lookup_ext(mail->transaction->view, mail->seq, + mbox->ext_id, &data, NULL); + dbox_rec = data; + if (dbox_rec == NULL || dbox_rec->map_uid == 0) { + /* lost for some reason, use fallback */ + return dbox_mail_get_save_date(mail, date_r); + } + + *date_r = dbox_rec->save_date; + return 1; +} + +static int +mdbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->transaction->box); + struct mdbox_map_mail_index_record rec; + uint32_t map_uid; + uint16_t refcount; + + switch (field) { + case MAIL_FETCH_REFCOUNT: + if (mdbox_mail_lookup(mbox, _mail->transaction->view, + _mail->seq, &map_uid) < 0) + return -1; + if (mdbox_map_lookup_full(mbox->storage->map, map_uid, + &rec, &refcount) < 0) + return -1; + *value_r = p_strdup_printf(mail->imail.mail.data_pool, "%u", + refcount); + return 0; + case MAIL_FETCH_REFCOUNT_ID: + if (mdbox_mail_lookup(mbox, _mail->transaction->view, + _mail->seq, &map_uid) < 0) + return -1; + *value_r = p_strdup_printf(mail->imail.mail.data_pool, "%u", + map_uid); + return 0; + case MAIL_FETCH_UIDL_BACKEND: + if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id, + offsetof(struct mdbox_index_header, flags), + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) { + *value_r = ""; + return 0; + } + break; + case MAIL_FETCH_POP3_ORDER: + if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id, + offsetof(struct mdbox_index_header, flags), + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) { + *value_r = ""; + return 0; + } + break; + default: + break; + } + return dbox_mail_get_special(_mail, field, value_r); +} + +static void +mdbox_mail_update_flags(struct mail *mail, enum modify_type modify_type, + enum mail_flags flags) +{ + if ((flags & DBOX_INDEX_FLAG_ALT) != 0) { + mdbox_purge_alt_flag_change(mail, modify_type != MODIFY_REMOVE); + flags &= ENUM_NEGATE(DBOX_INDEX_FLAG_ALT); + if (flags == 0 && modify_type != MODIFY_REPLACE) + return; + } + + index_mail_update_flags(mail, modify_type, flags); +} + +struct mail_vfuncs mdbox_mail_vfuncs = { + dbox_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + dbox_mail_get_received_date, + mdbox_mail_get_save_date, + dbox_mail_get_virtual_size, + dbox_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + dbox_mail_get_stream, + index_mail_get_binary_stream, + mdbox_mail_get_special, + index_mail_get_backend_mail, + mdbox_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/dbox-multi/mdbox-map-private.h b/src/lib-storage/index/dbox-multi/mdbox-map-private.h new file mode 100644 index 0000000..259d854 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-map-private.h @@ -0,0 +1,64 @@ +#ifndef MDBOX_MAP_PRIVATE_H +#define MDBOX_MAP_PRIVATE_H + +#include "mdbox-map.h" + +struct dbox_mail_lookup_rec { + uint32_t map_uid; + uint16_t refcount; + struct mdbox_map_mail_index_record rec; +}; + +struct mdbox_map { + struct mdbox_storage *storage; + const struct mdbox_settings *set; + char *path, *index_path; + + struct mail_index *index; + struct mail_index_view *view; + + uint32_t map_ext_id, ref_ext_id; + + struct mailbox_list *root_list; + + bool verify_existing_file_ids:1; +}; + +struct mdbox_map_append { + struct dbox_file_append_context *file_append; + uoff_t offset, size; +}; + +struct mdbox_map_append_context { + struct mdbox_map *map; + struct mdbox_map_atomic_context *atomic; + struct mail_index_transaction *trans; + + ARRAY(struct dbox_file_append_context *) file_appends; + ARRAY(struct dbox_file *) files; + ARRAY(struct mdbox_map_append) appends; + + uint32_t first_new_file_id; + + unsigned int files_nonappendable_count; + + bool failed:1; +}; + +struct mdbox_map_atomic_context { + struct mdbox_map *map; + struct mail_index_transaction *sync_trans; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *sync_view; + + bool map_refreshed:1; + bool locked:1; + bool success:1; + bool failed:1; +}; + +int mdbox_map_view_lookup_rec(struct mdbox_map *map, + struct mail_index_view *view, uint32_t seq, + struct dbox_mail_lookup_rec *rec_r); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c new file mode 100644 index 0000000..f9bac0b --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c @@ -0,0 +1,1492 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "ostream.h" +#include "mkdir-parents.h" +#include "unlink-old-files.h" +#include "mailbox-list-private.h" +#include "mdbox-storage.h" +#include "mdbox-file.h" +#include "mdbox-map-private.h" + +#include <dirent.h> + +#define MAX_BACKWARDS_LOOKUPS 10 + +#define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10) +#define DBOX_FORCE_PURGE_MIN_RATIO 0.5 + +#define MAP_STORAGE(map) (&(map)->storage->storage.storage) + +struct mdbox_map_transaction_context { + struct mdbox_map_atomic_context *atomic; + struct mail_index_transaction *trans; + + bool changed:1; + bool committed:1; +}; + +static int mdbox_map_generate_uid_validity(struct mdbox_map *map); + +void mdbox_map_set_corrupted(struct mdbox_map *map, const char *format, ...) +{ + va_list args; + + va_start(args, format); + mail_storage_set_critical(MAP_STORAGE(map), + "mdbox map %s corrupted: %s", + map->index->filepath, + t_strdup_vprintf(format, args)); + va_end(args); + + mdbox_storage_set_corrupted(map->storage); +} + +struct mdbox_map * +mdbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list) +{ + struct mdbox_map *map; + const char *root, *index_root; + + root = mailbox_list_get_root_forced(root_list, MAILBOX_LIST_PATH_TYPE_DIR); + index_root = mailbox_list_get_root_forced(root_list, MAILBOX_LIST_PATH_TYPE_INDEX); + + map = i_new(struct mdbox_map, 1); + map->storage = storage; + map->set = storage->set; + map->path = i_strconcat(root, "/"MDBOX_GLOBAL_DIR_NAME, NULL); + map->index_path = + i_strconcat(index_root, "/"MDBOX_GLOBAL_DIR_NAME, NULL); + map->index = mail_index_alloc(storage->storage.storage.event, + map->index_path, + MDBOX_GLOBAL_INDEX_PREFIX); + mail_index_set_fsync_mode(map->index, + MAP_STORAGE(map)->set->parsed_fsync_mode, 0); + mail_index_set_lock_method(map->index, + MAP_STORAGE(map)->set->parsed_lock_method, + mail_storage_get_lock_timeout(MAP_STORAGE(map), UINT_MAX)); + map->root_list = root_list; + map->map_ext_id = mail_index_ext_register(map->index, "map", + sizeof(struct mdbox_map_mail_index_header), + sizeof(struct mdbox_map_mail_index_record), + sizeof(uint32_t)); + map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0, + sizeof(uint16_t), sizeof(uint16_t)); + return map; +} + +static void mdbox_map_deinit_cleanup(struct mdbox_map *map) +{ + if (map->view == NULL) + return; + + const struct mail_index_header *hdr = + mail_index_get_header(map->view); + if (dbox_mailbox_list_cleanup(map->storage->storage.storage.user, + map->path, hdr->last_temp_file_scan) > 0) + index_mailbox_view_update_last_temp_file_scan(map->view); +} + +void mdbox_map_deinit(struct mdbox_map **_map) +{ + struct mdbox_map *map = *_map; + + *_map = NULL; + + mdbox_map_deinit_cleanup(map); + if (map->view != NULL) { + mail_index_view_close(&map->view); + mail_index_close(map->index); + } + mail_index_free(&map->index); + i_free(map->index_path); + i_free(map->path); + i_free(map); +} + +static int mdbox_map_mkdir_storage(struct mdbox_map *map) +{ + if (mailbox_list_mkdir_root(map->root_list, map->path, + MAILBOX_LIST_PATH_TYPE_DIR) < 0) { + mail_storage_copy_list_error(MAP_STORAGE(map), map->root_list); + return -1; + } + + if (strcmp(map->path, map->index_path) != 0 && + mailbox_list_mkdir_root(map->root_list, map->index_path, + MAILBOX_LIST_PATH_TYPE_INDEX) < 0) { + mail_storage_copy_list_error(MAP_STORAGE(map), map->root_list); + return -1; + } + return 0; +} + +static int mdbox_map_open_internal(struct mdbox_map *map, bool create_missing) +{ + enum mail_index_open_flags open_flags; + struct mailbox_permissions perm; + int ret = 0; + + if (map->view != NULL) { + /* already opened */ + return 1; + } + + mailbox_list_get_root_permissions(map->root_list, &perm); + mail_index_set_permissions(map->index, perm.file_create_mode, + perm.file_create_gid, + perm.file_create_gid_origin); + + open_flags = MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY | + mail_storage_settings_to_index_flags(MAP_STORAGE(map)->set); + if (create_missing) { + if ((ret = mdbox_map_mkdir_storage(map)) < 0) + return -1; + if (ret > 0) { + /* storage/ directory already existed. + the index should exist also. */ + } else { + open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; + } + } + ret = mail_index_open(map->index, open_flags); + if (ret == 0 && create_missing) { + /* storage/ already existed, but indexes didn't. we'll need to + take extra steps to make sure we won't overwrite any m.* + files that may already exist. */ + map->verify_existing_file_ids = TRUE; + open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; + ret = mail_index_open(map->index, open_flags); + } + if (ret < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + return -1; + } + if (ret == 0) { + /* index not found - for now just return failure */ + i_assert(!create_missing); + return 0; + } + + map->view = mail_index_view_open(map->index); + + if (mail_index_get_header(map->view)->uid_validity == 0) { + if (mdbox_map_generate_uid_validity(map) < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + mail_index_close(map->index); + return -1; + } + if (mdbox_map_refresh(map) < 0) { + mail_index_close(map->index); + return -1; + } + } + return 1; +} + +int mdbox_map_open(struct mdbox_map *map) +{ + return mdbox_map_open_internal(map, FALSE); +} + +int mdbox_map_open_or_create(struct mdbox_map *map) +{ + return mdbox_map_open_internal(map, TRUE) <= 0 ? -1 : 0; +} + +int mdbox_map_refresh(struct mdbox_map *map) +{ + struct mail_index_view_sync_ctx *ctx; + bool delayed_expunges, fscked; + int ret = 0; + + /* some open files may have read partially written mails. now that + map syncing makes the new mails visible, we need to make sure the + partial data is flushed out of memory */ + mdbox_files_sync_input(map->storage); + + if (mail_index_refresh(map->view->index) < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + return -1; + } + if (mail_index_view_have_transactions(map->view)) { + /* can't sync when there are transactions */ + return 0; + } + + ctx = mail_index_view_sync_begin(map->view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + fscked = mail_index_reset_fscked(map->view->index); + if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + ret = -1; + } + if (fscked) + mdbox_storage_set_corrupted(map->storage); + return ret; +} + +bool mdbox_map_is_fscked(struct mdbox_map *map) +{ + const struct mail_index_header *hdr; + + if (map->view == NULL) { + /* map isn't opened yet. don't bother. */ + return FALSE; + } + + hdr = mail_index_get_header(map->view); + return (hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0; +} + +static void +mdbox_map_get_ext_hdr(struct mdbox_map *map, struct mail_index_view *view, + struct mdbox_map_mail_index_header *hdr_r) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size); + i_zero(hdr_r); + memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r))); +} + +uint32_t mdbox_map_get_rebuild_count(struct mdbox_map *map) +{ + struct mdbox_map_mail_index_header hdr; + + mdbox_map_get_ext_hdr(map, map->view, &hdr); + return hdr.rebuild_count; +} + +static int +mdbox_map_lookup_seq(struct mdbox_map *map, uint32_t seq, + const struct mdbox_map_mail_index_record **rec_r) +{ + const struct mdbox_map_mail_index_record *rec; + const void *data; + uint32_t uid; + + mail_index_lookup_ext(map->view, seq, map->map_ext_id, &data, NULL); + rec = data; + + if (rec == NULL || rec->file_id == 0) { + mail_index_lookup_uid(map->view, seq, &uid); + mdbox_map_set_corrupted(map, "file_id=0 for map_uid=%u", uid); + return -1; + } + *rec_r = rec; + return 0; +} + +static int +mdbox_map_get_seq(struct mdbox_map *map, uint32_t map_uid, uint32_t *seq_r) +{ + if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) { + /* not found - try again after a refresh */ + if (mdbox_map_refresh(map) < 0) + return -1; + if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) + return 0; + } + return 1; +} + +int mdbox_map_lookup(struct mdbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r) +{ + const struct mdbox_map_mail_index_record *rec; + uint32_t seq; + int ret; + + if (mdbox_map_open_or_create(map) < 0) + return -1; + + if ((ret = mdbox_map_get_seq(map, map_uid, &seq)) <= 0) + return ret; + + if (mdbox_map_lookup_seq(map, seq, &rec) < 0) + return -1; + *file_id_r = rec->file_id; + *offset_r = rec->offset; + return 1; +} + +int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r) +{ + uint32_t seq; + int ret; + + if (mdbox_map_open_or_create(map) < 0) + return -1; + + if ((ret = mdbox_map_get_seq(map, map_uid, &seq)) <= 0) + return ret; + + return mdbox_map_lookup_seq_full(map, seq, rec_r, refcount_r); +} + +int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r) +{ + const struct mdbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + + if (mdbox_map_lookup_seq(map, seq, &rec) < 0) + return -1; + *rec_r = *rec; + + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing ref extension"); + return -1; + } + ref16_p = data; + *refcount_r = *ref16_p; + return 1; +} + +uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq) +{ + uint32_t uid; + + mail_index_lookup_uid(map->view, seq, &uid); + return uid; +} + +unsigned int mdbox_map_get_messages_count(struct mdbox_map *map) +{ + return mail_index_view_get_messages_count(map->view); +} + +int mdbox_map_view_lookup_rec(struct mdbox_map *map, + struct mail_index_view *view, uint32_t seq, + struct dbox_mail_lookup_rec *rec_r) +{ + const uint16_t *ref16_p; + const void *data; + + i_zero(rec_r); + mail_index_lookup_uid(view, seq, &rec_r->map_uid); + + mail_index_lookup_ext(view, seq, map->map_ext_id, &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing map extension"); + return -1; + } + memcpy(&rec_r->rec, data, sizeof(rec_r->rec)); + + mail_index_lookup_ext(view, seq, map->ref_ext_id, &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing ref extension"); + return -1; + } + ref16_p = data; + rec_r->refcount = *ref16_p; + return 0; +} + +int mdbox_map_get_file_msgs(struct mdbox_map *map, uint32_t file_id, + ARRAY_TYPE(mdbox_map_file_msg) *recs) +{ + const struct mail_index_header *hdr; + struct dbox_mail_lookup_rec rec; + struct mdbox_map_file_msg msg; + uint32_t seq; + + if (mdbox_map_refresh(map) < 0) + return -1; + hdr = mail_index_get_header(map->view); + + i_zero(&msg); + for (seq = 1; seq <= hdr->messages_count; seq++) { + if (mdbox_map_view_lookup_rec(map, map->view, seq, &rec) < 0) + return -1; + + if (rec.rec.file_id == file_id) { + msg.map_uid = rec.map_uid; + msg.offset = rec.rec.offset; + msg.refcount = rec.refcount; + array_push_back(recs, &msg); + } + } + return 0; +} + +int mdbox_map_get_zero_ref_files(struct mdbox_map *map, + ARRAY_TYPE(seq_range) *file_ids_r) +{ + const struct mail_index_header *hdr; + const struct mdbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + uint32_t seq; + bool expunged; + int ret; + + if ((ret = mdbox_map_open(map)) <= 0) { + /* no map / internal error */ + return ret; + } + if (mdbox_map_refresh(map) < 0) + return -1; + + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + if (data != NULL && !expunged) { + ref16_p = data; + if (*ref16_p != 0) + continue; + } + + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data != NULL && !expunged) { + rec = data; + seq_range_array_add(file_ids_r, rec->file_id); + } + } + return 0; +} + +struct mdbox_map_atomic_context *mdbox_map_atomic_begin(struct mdbox_map *map) +{ + struct mdbox_map_atomic_context *atomic; + + atomic = i_new(struct mdbox_map_atomic_context, 1); + atomic->map = map; + return atomic; +} + +static void +mdbox_map_sync_handle(struct mdbox_map *map, + struct mail_index_sync_ctx *sync_ctx) +{ + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + uoff_t offset1, offset2; + + mail_index_sync_get_offsets(sync_ctx, &seq1, &offset1, &seq2, &offset2); + if (offset1 != offset2 || seq1 != seq2) { + /* something had crashed. need a full resync. */ + i_warning("mdbox %s: Inconsistency in map index " + "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")", + map->path, seq1, offset1, seq2, offset2); + mdbox_storage_set_corrupted(map->storage); + } + while (mail_index_sync_next(sync_ctx, &sync_rec)) ; +} + +int mdbox_map_atomic_lock(struct mdbox_map_atomic_context *atomic, + const char *reason) +{ + int ret; + + if (atomic->locked) + return 0; + + if (mdbox_map_open_or_create(atomic->map) < 0) + return -1; + + /* use syncing to lock the transaction log, so that we always see + log's head_offset = tail_offset */ + ret = mail_index_sync_begin(atomic->map->index, &atomic->sync_ctx, + &atomic->sync_view, &atomic->sync_trans, + MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET); + if (mail_index_reset_fscked(atomic->map->index)) + mdbox_storage_set_corrupted(atomic->map->storage); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_index_error(MAP_STORAGE(atomic->map), + atomic->map->index); + return -1; + } + mail_index_sync_set_reason(atomic->sync_ctx, reason); + atomic->locked = TRUE; + /* reset refresh state so that if it's wanted to be done locked, + it gets the latest changes */ + atomic->map_refreshed = FALSE; + mdbox_map_sync_handle(atomic->map, atomic->sync_ctx); + return 0; +} + +bool mdbox_map_atomic_is_locked(struct mdbox_map_atomic_context *atomic) +{ + return atomic->locked; +} + +void mdbox_map_atomic_set_failed(struct mdbox_map_atomic_context *atomic) +{ + atomic->success = FALSE; + atomic->failed = TRUE; +} + +void mdbox_map_atomic_set_success(struct mdbox_map_atomic_context *atomic) +{ + if (!atomic->failed) + atomic->success = TRUE; +} + +void mdbox_map_atomic_unset_fscked(struct mdbox_map_atomic_context *atomic) +{ + mail_index_unset_fscked(atomic->sync_trans); +} + +int mdbox_map_atomic_finish(struct mdbox_map_atomic_context **_atomic) +{ + struct mdbox_map_atomic_context *atomic = *_atomic; + int ret = 0; + + *_atomic = NULL; + + if (atomic->sync_ctx == NULL) { + /* not locked */ + i_assert(!atomic->locked); + } else if (atomic->success) { + if (mail_index_sync_commit(&atomic->sync_ctx) < 0) { + mail_storage_set_index_error(MAP_STORAGE(atomic->map), + atomic->map->index); + ret = -1; + } + } else { + mail_index_sync_rollback(&atomic->sync_ctx); + } + i_free(atomic); + return ret; +} + +struct mdbox_map_transaction_context * +mdbox_map_transaction_begin(struct mdbox_map_atomic_context *atomic, + bool external) +{ + struct mdbox_map_transaction_context *ctx; + enum mail_index_transaction_flags flags = + MAIL_INDEX_TRANSACTION_FLAG_FSYNC; + bool success; + + if (external) + flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + + ctx = i_new(struct mdbox_map_transaction_context, 1); + ctx->atomic = atomic; + if (atomic->locked && atomic->map_refreshed) { + /* already refreshed within a lock, don't do it again */ + success = TRUE; + } else { + success = mdbox_map_open(atomic->map) > 0 && + mdbox_map_refresh(atomic->map) == 0; + } + + if (success) { + atomic->map_refreshed = TRUE; + ctx->trans = mail_index_transaction_begin(atomic->map->view, + flags); + } + return ctx; +} + +int mdbox_map_transaction_commit(struct mdbox_map_transaction_context *ctx, + const char *reason) +{ + i_assert(!ctx->committed); + + ctx->committed = TRUE; + if (!ctx->changed) + return 0; + + if (mdbox_map_atomic_lock(ctx->atomic, reason) < 0) + return -1; + + if (mail_index_transaction_commit(&ctx->trans) < 0) { + mail_storage_set_index_error(MAP_STORAGE(ctx->atomic->map), + ctx->atomic->map->index); + return -1; + } + mdbox_map_atomic_set_success(ctx->atomic); + return 0; +} + +void mdbox_map_transaction_free(struct mdbox_map_transaction_context **_ctx) +{ + struct mdbox_map_transaction_context *ctx = *_ctx; + + *_ctx = NULL; + + if (ctx->trans != NULL) + mail_index_transaction_rollback(&ctx->trans); + i_free(ctx); +} + +int mdbox_map_update_refcount(struct mdbox_map_transaction_context *ctx, + uint32_t map_uid, int diff) +{ + struct mdbox_map *map = ctx->atomic->map; + const void *data; + uint32_t seq; + int old_diff, new_diff; + + if (unlikely(ctx->trans == NULL)) + return -1; + + if (!mail_index_lookup_seq(map->view, map_uid, &seq)) { + /* we can't refresh map here since view has a + transaction open. */ + if (diff > 0) { + /* the message was probably just purged */ + mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_EXPUNGED, + "Some of the requested messages no longer exist."); + } else { + mdbox_map_set_corrupted(map, + "refcount update lost map_uid=%u", map_uid); + } + return -1; + } + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, &data, NULL); + old_diff = data == NULL ? 0 : *((const uint16_t *)data); + ctx->changed = TRUE; + new_diff = mail_index_atomic_inc_ext(ctx->trans, seq, + map->ref_ext_id, diff); + if (old_diff + new_diff < 0) { + mdbox_map_set_corrupted(map, "map_uid=%u refcount too low", + map_uid); + return -1; + } + if (old_diff + new_diff >= 32768 && new_diff > 0) { + /* we're getting close to the 64k limit. fail early + to make it less likely that two processes increase + the refcount enough times to cross the limit */ + mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_LIMIT, + t_strdup_printf("Message has been copied too many times (%d + %d)", + old_diff, new_diff)); + return -1; + } + return 0; +} + +int mdbox_map_update_refcounts(struct mdbox_map_transaction_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, int diff) +{ + const uint32_t *uidp; + unsigned int i, count; + + if (unlikely(ctx->trans == NULL)) + return -1; + + count = array_count(map_uids); + for (i = 0; i < count; i++) { + uidp = array_idx(map_uids, i); + if (mdbox_map_update_refcount(ctx, *uidp, diff) < 0) + return -1; + } + return 0; +} + +int mdbox_map_remove_file_id(struct mdbox_map *map, uint32_t file_id) +{ + struct mdbox_map_atomic_context *atomic; + struct mdbox_map_transaction_context *map_trans; + const struct mail_index_header *hdr; + const struct mdbox_map_mail_index_record *rec; + const void *data; + uint32_t seq; + int ret = 0; + + /* make sure the map is refreshed, otherwise we might be expunging + messages that have already been moved to other files. */ + + /* we need a per-file transaction, otherwise we can't refresh the map */ + atomic = mdbox_map_atomic_begin(map); + map_trans = mdbox_map_transaction_begin(atomic, TRUE); + + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing map extension"); + ret = -1; + break; + } + + rec = data; + if (rec->file_id == file_id) { + map_trans->changed = TRUE; + mail_index_expunge(map_trans->trans, seq); + } + } + if (ret == 0) + ret = mdbox_map_transaction_commit(map_trans, "removing file"); + mdbox_map_transaction_free(&map_trans); + if (mdbox_map_atomic_finish(&atomic) < 0) + ret = -1; + return ret; +} + +struct mdbox_map_append_context * +mdbox_map_append_begin(struct mdbox_map_atomic_context *atomic) +{ + struct mdbox_map_append_context *ctx; + + ctx = i_new(struct mdbox_map_append_context, 1); + ctx->atomic = atomic; + ctx->map = atomic->map; + ctx->first_new_file_id = (uint32_t)-1; + i_array_init(&ctx->file_appends, 64); + i_array_init(&ctx->files, 64); + i_array_init(&ctx->appends, 128); + + if (mdbox_map_open_or_create(atomic->map) < 0) + ctx->failed = TRUE; + else { + /* refresh the map so we can try appending to the + latest files */ + if (mdbox_map_refresh(atomic->map) == 0) + atomic->map_refreshed = TRUE; + else + ctx->failed = TRUE; + } + return ctx; +} + +static time_t day_begin_stamp(unsigned int interval) +{ + struct tm tm; + time_t stamp; + unsigned int unit = 1; + + if (interval == 0) + return 0; + + /* get the beginning of day/hour/minute depending on how large + the interval is */ + tm = *localtime(&ioloop_time); + if (interval >= 60) { + tm.tm_sec = 0; + unit = 60; + if (interval >= 3600) { + tm.tm_min = 0; + unit = 3600; + if (interval >= 3600*24) { + tm.tm_hour = 0; + unit = 3600*24; + } + } + } + stamp = mktime(&tm); + if (stamp == (time_t)-1) + i_panic("mktime(today) failed"); + + return stamp - (interval - unit); +} + +static bool dbox_try_open(struct dbox_file *file, bool want_altpath) +{ + bool notfound; + + if (want_altpath) { + if (dbox_file_open(file, ¬found) <= 0) + return FALSE; + } else { + if (dbox_file_open_primary(file, ¬found) <= 0) + return FALSE; + } + if (notfound) + return FALSE; + + if (file->lock != NULL) { + /* already locked, we're possibly in the middle of purging it + in which case we really don't want to write there. */ + return FALSE; + } + if (dbox_file_is_in_alt(file) != want_altpath) { + /* different alt location than what we want, can't use it */ + return FALSE; + } + return TRUE; +} + +static bool dbox_file_is_ok_at(struct dbox_file *file, uoff_t offset) +{ + bool last; + int ret; + + if (dbox_file_seek(file, offset) == 0) + return FALSE; + + while ((ret = dbox_file_seek_next(file, &offset, &last)) > 0); + if (ret == 0 && !last) + return FALSE; + return TRUE; +} + +static bool +mdbox_map_file_try_append(struct mdbox_map_append_context *ctx, + bool want_altpath, + const struct mdbox_map_mail_index_record *rec, + time_t stamp, uoff_t mail_size, + struct dbox_file_append_context **file_append_r, + struct ostream **output_r, bool *retry_later_r) +{ + struct mdbox_map *map = ctx->map; + struct mdbox_storage *storage = map->storage; + struct dbox_file *file; + struct dbox_file_append_context *file_append; + struct stat st; + bool file_too_old = FALSE; + int ret; + + *file_append_r = NULL; + *output_r = NULL; + *retry_later_r = FALSE; + + file = mdbox_file_init(storage, rec->file_id); + if (!dbox_try_open(file, want_altpath)) { + dbox_file_unref(&file); + return TRUE; + } + + if (file->create_time < stamp) + file_too_old = TRUE; + else if ((ret = dbox_file_try_lock(file)) <= 0) { + /* locking failed */ + *retry_later_r = ret == 0; + } else if (stat(file->cur_path, &st) < 0) { + if (errno != ENOENT) + i_error("stat(%s) failed: %m", file->cur_path); + /* the file was unlinked between opening and locking it. */ + } else if (st.st_size != rec->offset + rec->size && + /* check if there's any garbage at the end of file. + note that there may be valid messages added by another + session before we locked it (but after we refreshed + map index). */ + !dbox_file_is_ok_at(file, rec->offset + rec->size)) { + /* error message was already logged */ + } else { + file_append = dbox_file_append_init(file); + if (dbox_file_get_append_stream(file_append, output_r) <= 0) { + /* couldn't append to this file */ + } else if ((*output_r)->offset + mail_size > map->set->mdbox_rotate_size) { + /* file was too large after all */ + } else { + /* success */ + *file_append_r = file_append; + return TRUE; + } + dbox_file_append_rollback(&file_append); + } + + /* failure */ + dbox_file_unlock(file); + dbox_file_unref(&file); + return !file_too_old; +} + +static bool +mdbox_map_is_appending(struct mdbox_map_append_context *ctx, uint32_t file_id) +{ + struct dbox_file_append_context *const *file_appends; + unsigned int i, count; + + /* there shouldn't be many files open, don't bother with anything + faster. */ + file_appends = array_get(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)file_appends[i]->file; + + if (mfile->file_id == file_id) + return TRUE; + } + return FALSE; +} + +static struct dbox_file_append_context * +mdbox_map_find_existing_append(struct mdbox_map_append_context *ctx, + uoff_t mail_size, bool want_altpath, + struct ostream **output_r) +{ + struct mdbox_map *map = ctx->map; + struct dbox_file_append_context *const *file_appends, *append; + struct mdbox_file *mfile; + unsigned int i, count; + uoff_t append_offset; + + /* first try to use files already used in this append */ + file_appends = array_get(&ctx->file_appends, &count); + for (i = count; i > ctx->files_nonappendable_count; i--) { + append = file_appends[i-1]; + + if (dbox_file_is_in_alt(append->file) != want_altpath) + continue; + if (append->file->fd == -1) { + /* already closed it (below). we might be able to still + fit some small mail there, but that's too much + trouble */ + continue; + } + + append_offset = append->output->offset; + if (append_offset + mail_size <= map->set->mdbox_rotate_size && + dbox_file_get_append_stream(append, output_r) > 0) + return append; + + /* can't append to this file anymore. if we created this file, + close it so we don't waste fds. if we didn't, we can't close + it without also losing our lock too early. */ + mfile = (struct mdbox_file *)append->file; + if (mfile->file_id == 0 && dbox_file_append_flush(append) == 0) + dbox_file_close(append->file); + } + ctx->files_nonappendable_count = count; + return NULL; +} + +static int +mdbox_map_find_primary_files(struct mdbox_map_append_context *ctx, + ARRAY_TYPE(seq_range) *file_ids_r) +{ + struct mdbox_storage *dstorage = ctx->map->storage; + struct mail_storage *storage = &dstorage->storage.storage; + DIR *dir; + struct dirent *d; + uint32_t file_id; + int ret = 0; + + /* we want to quickly find the latest alt file, but we also want to + avoid accessing the alt storage as much as possible. typically most + of the older mails would be in alt storage, so we'll just put the + few m.* files in primary storage to checked_file_ids array. other + files are then known to exist in alt storage. */ + dir = opendir(dstorage->storage_dir); + if (dir == NULL) { + mail_storage_set_critical(storage, + "opendir(%s) failed: %m", dstorage->storage_dir); + return -1; + } + + for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) { + if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, + strlen(MDBOX_MAIL_FILE_PREFIX)) != 0) + continue; + if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX), + &file_id) < 0) + continue; + + seq_range_array_add(file_ids_r, file_id); + } + if (errno != 0) { + mail_storage_set_critical(storage, + "readdir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(storage, + "closedir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + return ret; +} + +static int +mdbox_map_find_appendable_file(struct mdbox_map_append_context *ctx, + uoff_t mail_size, bool want_altpath, + struct dbox_file_append_context **file_append_r, + struct ostream **output_r) +{ + struct mdbox_map *map = ctx->map; + ARRAY_TYPE(seq_range) checked_file_ids; + const struct mail_index_header *hdr; + const struct mdbox_map_mail_index_record *rec; + unsigned int backwards_lookup_count; + uint32_t seq, seq1, uid; + time_t stamp; + bool retry_later; + + if (mail_size >= map->set->mdbox_rotate_size) + return 0; + + /* try to find an existing appendable file */ + stamp = day_begin_stamp(map->set->mdbox_rotate_interval); + hdr = mail_index_get_header(map->view); + + backwards_lookup_count = 0; + t_array_init(&checked_file_ids, 16); + + if (want_altpath) { + /* we want to save to alt storage. */ + if (mdbox_map_find_primary_files(ctx, &checked_file_ids) < 0) + return -1; + } + + for (seq = hdr->messages_count; seq > 0; seq--) { + if (mdbox_map_lookup_seq(map, seq, &rec) < 0) + return -1; + + if (seq_range_exists(&checked_file_ids, rec->file_id)) + continue; + seq_range_array_add(&checked_file_ids, rec->file_id); + + if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) { + /* we've wasted enough time here */ + break; + } + + /* first lookup: this should be enough usually, but we can't + be sure until after locking. also if messages were recently + moved, this message might not be the last one in the file. */ + if (rec->offset + rec->size + mail_size >= + map->set->mdbox_rotate_size) + continue; + + if (mdbox_map_is_appending(ctx, rec->file_id)) { + /* already checked this */ + continue; + } + + mail_index_lookup_uid(map->view, seq, &uid); + if (!mdbox_map_file_try_append(ctx, want_altpath, rec, + stamp, mail_size, file_append_r, + output_r, &retry_later)) { + /* file is too old. the rest of the files are too. */ + break; + } + /* NOTE: we've now refreshed map view. there are no guarantees + about sequences anymore. */ + if (*file_append_r != NULL) + return 1; + /* FIXME: use retry_later somehow */ + if (uid == 1 || + !mail_index_lookup_seq_range(map->view, 1, uid-1, + &seq1, &seq)) + break; + seq++; + } + return 0; +} + +int mdbox_map_append_next(struct mdbox_map_append_context *ctx, + uoff_t mail_size, enum mdbox_map_append_flags flags, + struct dbox_file_append_context **file_append_ctx_r, + struct ostream **output_r) +{ + struct dbox_file *file; + struct mdbox_map_append *append; + struct dbox_file_append_context *file_append; + bool existing, want_altpath; + int ret; + + if (ctx->failed) + return -1; + + want_altpath = (flags & DBOX_MAP_APPEND_FLAG_ALT) != 0; + file_append = mdbox_map_find_existing_append(ctx, mail_size, + want_altpath, output_r); + if (file_append != NULL) { + ret = 1; + existing = TRUE; + } else { + ret = mdbox_map_find_appendable_file(ctx, mail_size, want_altpath, + &file_append, output_r); + existing = FALSE; + } + if (ret > 0) + file = file_append->file; + else if (ret < 0) + return -1; + else { + /* create a new file */ + file = (flags & DBOX_MAP_APPEND_FLAG_ALT) == 0 ? + mdbox_file_init(ctx->map->storage, 0) : + mdbox_file_init_new_alt(ctx->map->storage); + file_append = dbox_file_append_init(file); + + ret = dbox_file_get_append_stream(file_append, output_r); + if (ret <= 0) { + i_assert(ret < 0); + dbox_file_append_rollback(&file_append); + dbox_file_unref(&file); + return -1; + } + } + + append = array_append_space(&ctx->appends); + append->file_append = file_append; + append->offset = (*output_r)->offset; + append->size = (uint32_t)-1; + if (!existing) { + i_assert(file_append->first_append_offset == 0); + file_append->first_append_offset = file_append->output->offset; + array_push_back(&ctx->file_appends, &file_append); + array_push_back(&ctx->files, &file); + } + *file_append_ctx_r = file_append; + return 0; +} + +static void +mdbox_map_append_close_if_unneeded(struct mdbox_map *map, + struct dbox_file_append_context *append_ctx) +{ + struct mdbox_file *mfile = + (struct mdbox_file *)append_ctx->file; + uoff_t end_offset = append_ctx->output->offset; + + /* if this file is now large enough not to fit any other + mails and we created it, close its fd since it's not + needed anymore. */ + if (end_offset > map->set->mdbox_rotate_size && + mfile->file_id == 0 && + dbox_file_append_flush(append_ctx) == 0) + dbox_file_close(append_ctx->file); +} + +void mdbox_map_append_finish(struct mdbox_map_append_context *ctx) +{ + struct mdbox_map_append *appends, *last; + unsigned int count; + uoff_t cur_offset; + + appends = array_get_modifiable(&ctx->appends, &count); + i_assert(count > 0); + last = &appends[count-1]; + i_assert(last->size == (uint32_t)-1); + + cur_offset = last->file_append->output->offset; + i_assert(cur_offset >= last->offset); + last->size = cur_offset - last->offset; + dbox_file_append_checkpoint(last->file_append); + + mdbox_map_append_close_if_unneeded(ctx->map, last->file_append); +} + +void mdbox_map_append_abort(struct mdbox_map_append_context *ctx) +{ + struct mdbox_map_append *appends; + unsigned int count; + + appends = array_get_modifiable(&ctx->appends, &count); + i_assert(count > 0 && appends[count-1].size == (uint32_t)-1); + array_delete(&ctx->appends, count-1, 1); +} + +static int +mdbox_find_highest_file_id(struct mdbox_map *map, uint32_t *file_id_r) +{ + const size_t prefix_len = strlen(MDBOX_MAIL_FILE_PREFIX); + DIR *dir; + struct dirent *d; + unsigned int id, highest_id = 0; + + dir = opendir(map->path); + if (dir == NULL) { + i_error("opendir(%s) failed: %m", map->path); + return -1; + } + while ((d = readdir(dir)) != NULL) { + if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, prefix_len) == 0 && + str_to_uint(d->d_name + prefix_len, &id) == 0) { + if (highest_id < id) + highest_id = id; + } + } + (void)closedir(dir); + + *file_id_r = highest_id; + return 0; +} + +static int +mdbox_map_assign_file_ids(struct mdbox_map_append_context *ctx, + bool separate_transaction, const char *reason) +{ + struct dbox_file_append_context *const *file_appends; + unsigned int i, count; + struct mdbox_map_mail_index_header hdr; + uint32_t first_file_id, file_id, existing_id; + + /* start the syncing. we'll need it even if there are no file ids to + be assigned. */ + if (mdbox_map_atomic_lock(ctx->atomic, reason) < 0) + return -1; + + mdbox_map_get_ext_hdr(ctx->map, ctx->atomic->sync_view, &hdr); + file_id = hdr.highest_file_id + 1; + + if (ctx->map->verify_existing_file_ids) { + /* storage/ directory had been already created but + without indexes. scan to see if there exists a higher + m.* file id than what is in header, so we won't + accidentally overwrite any existing files. */ + if (mdbox_find_highest_file_id(ctx->map, &existing_id) < 0) + return -1; + if (file_id < existing_id+1) + file_id = existing_id+1; + } + + /* assign file_ids for newly created files */ + first_file_id = file_id; + file_appends = array_get(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)file_appends[i]->file; + + if (dbox_file_append_flush(file_appends[i]) < 0) + return -1; + + if (mfile->file_id == 0) { + if (mdbox_file_assign_file_id(mfile, file_id++) < 0) + return -1; + } + } + + ctx->trans = !separate_transaction ? NULL : + mail_index_transaction_begin(ctx->map->view, + MAIL_INDEX_TRANSACTION_FLAG_FSYNC); + + /* update the highest used file_id */ + if (first_file_id != file_id) { + file_id--; + mail_index_update_header_ext(ctx->trans != NULL ? ctx->trans : + ctx->atomic->sync_trans, + ctx->map->map_ext_id, + 0, &file_id, sizeof(file_id)); + } + return 0; +} + +int mdbox_map_append_assign_map_uids(struct mdbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r) +{ + const struct mdbox_map_append *appends; + const struct mail_index_header *hdr; + struct mdbox_map_mail_index_record rec; + unsigned int i, count; + ARRAY_TYPE(seq_range) uids; + const struct seq_range *range; + uint32_t seq; + uint16_t ref16; + int ret = 0; + + if (array_count(&ctx->appends) == 0) { + *first_map_uid_r = 0; + *last_map_uid_r = 0; + return 0; + } + + if (mdbox_map_assign_file_ids(ctx, TRUE, "saving - assign uids") < 0) + return -1; + + /* append map records to index */ + i_zero(&rec); + ref16 = 1; + appends = array_get(&ctx->appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)appends[i].file_append->file; + + i_assert(appends[i].offset <= (uint32_t)-1); + i_assert(appends[i].size <= (uint32_t)-1); + + rec.file_id = mfile->file_id; + rec.offset = appends[i].offset; + rec.size = appends[i].size; + + mail_index_append(ctx->trans, 0, &seq); + mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id, + &rec, NULL); + mail_index_update_ext(ctx->trans, seq, ctx->map->ref_ext_id, + &ref16, NULL); + } + + /* assign map UIDs for appended records */ + hdr = mail_index_get_header(ctx->atomic->sync_view); + t_array_init(&uids, 1); + mail_index_append_finish_uids(ctx->trans, hdr->next_uid, &uids); + range = array_front(&uids); + i_assert(range[0].seq2 - range[0].seq1 + 1 == count); + + if (hdr->uid_validity == 0) { + /* we don't really care about uidvalidity, but it can't be 0 */ + uint32_t uid_validity = ioloop_time; + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + if (mail_index_transaction_commit(&ctx->trans) < 0) { + mail_storage_set_index_error(MAP_STORAGE(ctx->map), + ctx->map->index); + return -1; + } + + *first_map_uid_r = range[0].seq1; + *last_map_uid_r = range[0].seq2; + return ret; +} + +int mdbox_map_append_move(struct mdbox_map_append_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, + const ARRAY_TYPE(seq_range) *expunge_map_uids) +{ + const struct mdbox_map_append *appends; + struct mdbox_map_mail_index_record rec; + struct seq_range_iter iter; + const uint32_t *uids; + unsigned int i, j, map_uids_count, appends_count; + uint32_t uid, seq, next_uid; + + /* map is locked by this call */ + if (mdbox_map_assign_file_ids(ctx, FALSE, "purging - update uids") < 0) + return -1; + + i_zero(&rec); + appends = array_get(&ctx->appends, &appends_count); + + next_uid = mail_index_get_header(ctx->atomic->sync_view)->next_uid; + uids = array_get(map_uids, &map_uids_count); + for (i = j = 0; i < map_uids_count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)appends[j].file_append->file; + + i_assert(j < appends_count); + rec.file_id = mfile->file_id; + rec.offset = appends[j].offset; + rec.size = appends[j].size; + j++; + + if (!mail_index_lookup_seq(ctx->atomic->sync_view, + uids[i], &seq)) { + /* We wrote the email to the new m.* file, but another + process already expunged it and purged it. Deleting + the email from the new m.* file would be problematic + at this point, so just add the mail back to the map + with refcount=0 and the next purge will remove it. */ + mail_index_append(ctx->atomic->sync_trans, + next_uid++, &seq); + } + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->map->map_ext_id, &rec, NULL); + } + + seq_range_array_iter_init(&iter, expunge_map_uids); i = 0; + while (seq_range_array_iter_nth(&iter, i++, &uid)) { + if (!mail_index_lookup_seq(ctx->atomic->sync_view, uid, &seq)) + i_unreached(); + mail_index_expunge(ctx->atomic->sync_trans, seq); + } + return 0; +} + +int mdbox_map_append_flush(struct mdbox_map_append_context *ctx) +{ + struct dbox_file_append_context **file_appends; + unsigned int i, count; + + i_assert(ctx->trans == NULL); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (dbox_file_append_flush(file_appends[i]) < 0) + return -1; + } + return 0; +} + +int mdbox_map_append_commit(struct mdbox_map_append_context *ctx) +{ + struct dbox_file_append_context **file_appends; + unsigned int i, count; + + i_assert(ctx->trans == NULL); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (dbox_file_append_commit(&file_appends[i]) < 0) + return -1; + } + mdbox_map_atomic_set_success(ctx->atomic); + return 0; +} + +void mdbox_map_append_free(struct mdbox_map_append_context **_ctx) +{ + struct mdbox_map_append_context *ctx = *_ctx; + struct dbox_file_append_context **file_appends; + struct dbox_file **files; + unsigned int i, count; + + *_ctx = NULL; + + if (ctx->trans != NULL) + mail_index_transaction_rollback(&ctx->trans); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (file_appends[i] != NULL) + dbox_file_append_rollback(&file_appends[i]); + } + + files = array_get_modifiable(&ctx->files, &count); + for (i = 0; i < count; i++) { + dbox_file_unlock(files[i]); + dbox_file_unref(&files[i]); + } + + array_free(&ctx->appends); + array_free(&ctx->file_appends); + array_free(&ctx->files); + i_free(ctx); +} + +static int mdbox_map_generate_uid_validity(struct mdbox_map *map) +{ + const struct mail_index_header *hdr; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + uint32_t uid_validity; + int ret; + + /* do this inside syncing, so that we're locked and there are no + race conditions */ + ret = mail_index_sync_begin(map->index, &sync_ctx, &view, &trans, + MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET); + if (ret <= 0) { + i_assert(ret != 0); + return -1; + } + mdbox_map_sync_handle(map, sync_ctx); + + hdr = mail_index_get_header(map->view); + if (hdr->uid_validity != 0) { + /* someone else beat us to it */ + } else { + uid_validity = ioloop_time; + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + mail_index_sync_set_reason(sync_ctx, "uidvalidity initialization"); + return mail_index_sync_commit(&sync_ctx); +} + +uint32_t mdbox_map_get_uid_validity(struct mdbox_map *map) +{ + uint32_t uid_validity; + + i_assert(map->view != NULL); + + uid_validity = mail_index_get_header(map->view)->uid_validity; + if (uid_validity == 0) + mdbox_map_set_corrupted(map, "lost uidvalidity"); + return uid_validity; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.h b/src/lib-storage/index/dbox-multi/mdbox-map.h new file mode 100644 index 0000000..9571f0e --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-map.h @@ -0,0 +1,144 @@ +#ifndef MDBOX_MAP_H +#define MDBOX_MAP_H + +#include "seq-range-array.h" + +struct dbox_file_append_context; +struct mdbox_map_append_context; +struct mdbox_storage; + +enum mdbox_map_append_flags { + DBOX_MAP_APPEND_FLAG_ALT = 0x01 +}; + +struct mdbox_map_mail_index_header { + uint32_t highest_file_id; + /* increased every time storage is rebuilt */ + uint32_t rebuild_count; +}; + +struct mdbox_map_mail_index_record { + uint32_t file_id; + uint32_t offset; + uint32_t size; /* including pre/post metadata */ +}; + +struct mdbox_map_file_msg { + uint32_t map_uid; + uint32_t offset; + uint32_t refcount; +}; +ARRAY_DEFINE_TYPE(mdbox_map_file_msg, struct mdbox_map_file_msg); + +struct mdbox_map * +mdbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list); +void mdbox_map_deinit(struct mdbox_map **map); + +/* Open the map. Returns 1 if ok, 0 if map doesn't exist, -1 if error. */ +int mdbox_map_open(struct mdbox_map *map); +/* Open or create the map. This is done automatically for most operations. + Returns 0 if ok, -1 if error. */ +int mdbox_map_open_or_create(struct mdbox_map *map); +/* Refresh the map. Returns 0 if ok, -1 if error. */ +int mdbox_map_refresh(struct mdbox_map *map); +/* Returns TRUE if map has been fsck'd. */ +bool mdbox_map_is_fscked(struct mdbox_map *map); + +/* Return the current rebuild counter */ +uint32_t mdbox_map_get_rebuild_count(struct mdbox_map *map); + +/* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID + is already expunged, -1 if error. */ +int mdbox_map_lookup(struct mdbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r); +/* Like mdbox_map_lookup(), but look up everything. */ +int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r); +/* Like mdbox_map_lookup_full(), but look up with sequence. */ +int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r); +/* Return map UID for the map sequence. */ +uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq); +/* Returns the total number of messages in the map. */ +unsigned int mdbox_map_get_messages_count(struct mdbox_map *map); + +/* Get all messages from file */ +int mdbox_map_get_file_msgs(struct mdbox_map *map, uint32_t file_id, + ARRAY_TYPE(mdbox_map_file_msg) *recs); + +/* Begin atomic context. There can be multiple transactions/appends within the + same atomic context. */ +struct mdbox_map_atomic_context *mdbox_map_atomic_begin(struct mdbox_map *map); +/* Lock the map immediately. */ +int mdbox_map_atomic_lock(struct mdbox_map_atomic_context *atomic, + const char *reason); +/* Returns TRUE if map is locked */ +bool mdbox_map_atomic_is_locked(struct mdbox_map_atomic_context *atomic); +/* When finish() is called, rollback the changes. If data was already written + to map's transaction log, this desyncs the map and causes a rebuild */ +void mdbox_map_atomic_set_failed(struct mdbox_map_atomic_context *atomic); +/* Mark this atomic as having succeeded. This is internally done if + transaction or append is committed within this atomic, but not when the + atomic is used standalone. */ +void mdbox_map_atomic_set_success(struct mdbox_map_atomic_context *atomic); +/* Remove fsck'd flag. */ +void mdbox_map_atomic_unset_fscked(struct mdbox_map_atomic_context *atomic); +/* Commit/rollback changes within this atomic context. */ +int mdbox_map_atomic_finish(struct mdbox_map_atomic_context **atomic); + +struct mdbox_map_transaction_context * +mdbox_map_transaction_begin(struct mdbox_map_atomic_context *atomic, + bool external); +/* Write transaction to map and leave it locked. Call _free() to update tail + offset and unlock. */ +int mdbox_map_transaction_commit(struct mdbox_map_transaction_context *ctx, + const char *reason); +void mdbox_map_transaction_free(struct mdbox_map_transaction_context **ctx); + +int mdbox_map_update_refcount(struct mdbox_map_transaction_context *ctx, + uint32_t map_uid, int diff); +int mdbox_map_update_refcounts(struct mdbox_map_transaction_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, int diff); +int mdbox_map_remove_file_id(struct mdbox_map *map, uint32_t file_id); + +/* Return all files containing messages with zero refcount. */ +int mdbox_map_get_zero_ref_files(struct mdbox_map *map, + ARRAY_TYPE(seq_range) *file_ids_r); + +struct mdbox_map_append_context * +mdbox_map_append_begin(struct mdbox_map_atomic_context *atomic); +/* Request file for saving a new message with given size (if available). If an + existing file can be used, the record is locked and updated in index. + Returns 0 if ok, -1 if error. */ +int mdbox_map_append_next(struct mdbox_map_append_context *ctx, uoff_t mail_size, + enum mdbox_map_append_flags flags, + struct dbox_file_append_context **file_append_ctx_r, + struct ostream **output_r); +/* Finished saving the last mail. Saves the message size. */ +void mdbox_map_append_finish(struct mdbox_map_append_context *ctx); +/* Abort saving the last mail. */ +void mdbox_map_append_abort(struct mdbox_map_append_context *ctx); +/* Assign map UIDs to all appended msgs to multi-files. */ +int mdbox_map_append_assign_map_uids(struct mdbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r); +/* The appends are existing messages that were simply moved to a new file. + map_uids contains the moved messages' map UIDs. */ +int mdbox_map_append_move(struct mdbox_map_append_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, + const ARRAY_TYPE(seq_range) *expunge_map_uids); +/* Flush/fsync appends. */ +int mdbox_map_append_flush(struct mdbox_map_append_context *ctx); +/* Returns 0 if ok, -1 if error. */ +int mdbox_map_append_commit(struct mdbox_map_append_context *ctx); +void mdbox_map_append_free(struct mdbox_map_append_context **ctx); + +/* Returns map's uidvalidity */ +uint32_t mdbox_map_get_uid_validity(struct mdbox_map *map); + +void mdbox_map_set_corrupted(struct mdbox_map *map, const char *format, ...) + ATTR_FORMAT(2, 3); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c new file mode 100644 index 0000000..8461074 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c @@ -0,0 +1,690 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "hash.h" +#include "dbox-attachment.h" +#include "mdbox-storage.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-file.h" +#include "mdbox-map.h" +#include "mdbox-sync.h" + +#include <dirent.h> + +/* + Altmoving works like: + + 1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox + code and map UID's alt-refcount is updated. It won't be written to disk. + 2. mdbox_purge() is called, which checks if map UID's refcount equals + to its alt-refcount. If it does, it's moved to alt storage. Moving to + primary storage is done if _ALT flag was removed from any message. +*/ + +enum mdbox_msg_action { + MDBOX_MSG_ACTION_MOVE_TO_ALT = 1, + MDBOX_MSG_ACTION_MOVE_FROM_ALT +}; + +struct mdbox_purge_context { + pool_t pool; + struct mdbox_storage *storage; + + uint32_t lowest_primary_file_id; + /* list of file_ids that exist in primary storage. this list is looked + up while there is no locking, so it may not be accurate anymore by + the time it's used. */ + ARRAY_TYPE(seq_range) primary_file_ids; + /* list of file_ids that we need to purge */ + ARRAY_TYPE(seq_range) purge_file_ids; + + /* uint32_t map_uid => enum mdbox_msg_action action */ + HASH_TABLE(void *, void *) altmoves; + bool have_altmoves; + + struct mdbox_map_atomic_context *atomic; + struct mdbox_map_append_context *append_ctx; +}; + +static int mdbox_map_file_msg_offset_cmp(const struct mdbox_map_file_msg *m1, + const struct mdbox_map_file_msg *m2) +{ + if (m1->offset < m2->offset) + return -1; + else if (m1->offset > m2->offset) + return 1; + else + return 0; +} + +static int +mdbox_file_read_metadata_hdr(struct dbox_file *file, + struct dbox_metadata_header *meta_hdr_r) +{ + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(file->input, &data, &size, + sizeof(*meta_hdr_r)); + if (ret <= 0) { + i_assert(ret == -1); + if (file->input->stream_errno == 0) { + dbox_file_set_corrupted(file, "missing metadata"); + return 0; + } + mail_storage_set_critical(&file->storage->storage, + "read(%s) failed: %s", file->cur_path, + i_stream_get_error(file->input)); + return -1; + } + + memcpy(meta_hdr_r, data, sizeof(*meta_hdr_r)); + if (memcmp(meta_hdr_r->magic_post, DBOX_MAGIC_POST, + sizeof(meta_hdr_r->magic_post)) != 0) { + dbox_file_set_corrupted(file, "invalid metadata magic"); + return 0; + } + i_stream_skip(file->input, sizeof(*meta_hdr_r)); + return 1; +} + +static int +mdbox_file_metadata_copy(struct dbox_file *file, struct ostream *output) +{ + struct dbox_metadata_header meta_hdr; + const char *line; + size_t buf_size; + int ret; + + if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0) + return ret; + + o_stream_nsend(output, &meta_hdr, sizeof(meta_hdr)); + buf_size = i_stream_get_max_buffer_size(file->input); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == '\0') { + /* end of metadata */ + break; + } + o_stream_nsend_str(output, line); + o_stream_nsend(output, "\n", 1); + } + i_stream_set_max_buffer_size(file->input, buf_size); + + if (line == NULL) { + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return 0; + } + o_stream_nsend(output, "\n", 1); + return 1; +} + +static int +mdbox_metadata_get_extrefs(struct dbox_file *file, pool_t ext_refs_pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + struct dbox_metadata_header meta_hdr; + const char *line; + size_t buf_size; + int ret; + + /* skip and ignore the header */ + if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0) + return ret; + + buf_size = i_stream_get_max_buffer_size(file->input); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == '\0') { + /* end of metadata */ + break; + } + if (*line == DBOX_METADATA_EXT_REF) T_BEGIN { + if (!index_attachment_parse_extrefs(line+1, ext_refs_pool, + extrefs)) { + i_warning("%s: Ignoring corrupted extref: %s", + file->cur_path, line); + } + } T_END; + } + i_stream_set_max_buffer_size(file->input, buf_size); + + if (line == NULL) { + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return 0; + } + return 1; +} + +static bool +mdbox_purge_want_altpath(struct mdbox_purge_context *ctx, + struct dbox_file *file, uint32_t map_uid) +{ + enum mdbox_msg_action action; + void *value; + + if (dbox_file_is_in_alt(file)) + return TRUE; + + if (!ctx->have_altmoves) + return FALSE; + + value = hash_table_lookup(ctx->altmoves, POINTER_CAST(map_uid)); + action = POINTER_CAST_TO(value, enum mdbox_msg_action); + return action == MDBOX_MSG_ACTION_MOVE_TO_ALT; +} + +static int +mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file, + const struct mdbox_map_file_msg *msg) +{ + struct dbox_file_append_context *out_file_append; + struct istream *input; + struct ostream *output; + enum mdbox_map_append_flags append_flags; + uoff_t msg_size; + int ret; + + if (ctx->append_ctx == NULL) + ctx->append_ctx = mdbox_map_append_begin(ctx->atomic); + + append_flags = !mdbox_purge_want_altpath(ctx, file, msg->map_uid) ? 0 : + DBOX_MAP_APPEND_FLAG_ALT; + msg_size = file->msg_header_size + file->cur_physical_size; + if (mdbox_map_append_next(ctx->append_ctx, file->cur_physical_size, + append_flags, &out_file_append, &output) < 0) + return -1; + + i_assert(file != out_file_append->file); + + input = i_stream_create_limit(file->input, msg_size); + o_stream_nsend_istream(output, input); + if (o_stream_flush(output) < 0) { + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %s", + out_file_append->file->cur_path, + o_stream_get_error(output)); + ret = -1; + } else if (input->v_offset != msg_size) { + i_assert(input->v_offset < msg_size); + i_assert(i_stream_read_eof(file->input)); + + dbox_file_set_corrupted(file, "truncated message at EOF"); + ret = 0; + } else { + ret = 1; + } + i_stream_unref(&input); + + if (ret > 0) { + /* copy metadata */ + if ((ret = mdbox_file_metadata_copy(file, output)) <= 0) + return ret; + + mdbox_map_append_finish(ctx->append_ctx); + } + return ret; +} + +static int +mdbox_file_purge_check_refcounts(struct mdbox_purge_context *ctx, + const ARRAY_TYPE(mdbox_map_file_msg) *msgs_arr) +{ + struct mdbox_map *map = ctx->storage->map; + struct mdbox_map_mail_index_record rec; + uint16_t refcount; + const struct mdbox_map_file_msg *msgs; + unsigned int i, count; + int ret; + + if (mdbox_map_atomic_lock(ctx->atomic, "purging check") < 0) + return -1; + + msgs = array_get(msgs_arr, &count); + for (i = 0; i < count; i++) { + if (msgs[i].refcount != 0) + continue; + + ret = mdbox_map_lookup_full(map, msgs[i].map_uid, &rec, + &refcount); + if (ret <= 0) { + if (ret < 0) + return -1; + mdbox_map_set_corrupted(map, + "Purging unexpectedly lost map_uid=%u", + msgs[i].map_uid); + return -1; + } + if (refcount > 0) + return 0; + } + return 1; +} + +static int +mdbox_purge_attachments(struct mdbox_purge_context *ctx, + const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr) +{ + struct dbox_storage *storage = &ctx->storage->storage; + const struct mail_attachment_extref *extref; + int ret = 0; + + array_foreach(extrefs_arr, extref) { + if (index_attachment_delete(&storage->storage, + storage->attachment_fs, + extref->path) < 0) + ret = -1; + } + return ret; +} + +static int +mdbox_file_purge(struct mdbox_purge_context *ctx, struct dbox_file *file, + uint32_t file_id) +{ + struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage; + struct stat st; + ARRAY_TYPE(mdbox_map_file_msg) msgs_arr; + const struct mdbox_map_file_msg *msgs; + ARRAY_TYPE(seq_range) expunged_map_uids; + ARRAY_TYPE(uint32_t) copied_map_uids; + ARRAY_TYPE(mail_attachment_extref) ext_refs; + pool_t ext_refs_pool; + unsigned int i, count; + uoff_t offset; + int ret; + + i_assert(ctx->atomic == NULL); + i_assert(ctx->append_ctx == NULL); + + if ((ret = dbox_file_try_lock(file)) <= 0) + return ret; + + /* make sure the file still exists. another process may have already + deleted it. */ + if (stat(file->cur_path, &st) < 0) { + dbox_file_unlock(file); + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(&file->storage->storage, + "stat(%s) failed: %m", file->cur_path); + return -1; + } + + /* get list of map UIDs that exist in this file (again has to be done + after locking) */ + i_array_init(&msgs_arr, 128); + if (mdbox_map_get_file_msgs(dstorage->map, file_id, + &msgs_arr) < 0) { + array_free(&msgs_arr); + dbox_file_unlock(file); + return -1; + } + /* sort messages by their offset */ + array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp); + + ext_refs_pool = pool_alloconly_create("mdbox purge ext refs", 1024); + ctx->atomic = mdbox_map_atomic_begin(ctx->storage->map); + msgs = array_get(&msgs_arr, &count); + i_array_init(&ext_refs, 32); + i_array_init(&copied_map_uids, I_MIN(count, 1)); + i_array_init(&expunged_map_uids, I_MIN(count, 1)); + offset = file->file_header_size; + for (i = 0; i < count; i++) { + if ((ret = dbox_file_seek(file, offset)) <= 0) + break; + + if (msgs[i].offset != offset) { + /* map doesn't match file's actual contents */ + dbox_file_set_corrupted(file, + "purging found mismatched offsets " + "(%"PRIuUOFF_T" vs %u, %u/%u)", + offset, msgs[i].offset, i, count); + ret = 0; + break; + } + + if (msgs[i].refcount == 0) { + /* skip over expunged message */ + i_stream_seek(file->input, offset + + file->msg_header_size + + file->cur_physical_size); + /* skip metadata */ + ret = mdbox_metadata_get_extrefs(file, ext_refs_pool, + &ext_refs); + if (ret <= 0) + break; + seq_range_array_add(&expunged_map_uids, + msgs[i].map_uid); + } else { + /* non-expunged message. write it to output file. */ + i_stream_seek(file->input, offset); + ret = mdbox_purge_save_msg(ctx, file, &msgs[i]); + if (ret <= 0) + break; + array_push_back(&copied_map_uids, &msgs[i].map_uid); + } + offset = file->input->v_offset; + } + if (offset != (uoff_t)st.st_size && ret > 0) { + /* file has more messages than what map tells us */ + dbox_file_set_corrupted(file, + "more messages available than in map " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", offset, st.st_size); + ret = 0; + } + if (ret > 0 && ctx->append_ctx != NULL) { + /* flush writes before locking the map */ + if (mdbox_map_append_flush(ctx->append_ctx) < 0) + ret = -1; + } + + if (ret <= 0) + ret = -1; + else { + /* it's possible that one of the messages we purged was + just copied to another mailbox. the only way to prevent that + would be to keep map locked during the purge, but that could + keep it locked for too long. instead we'll check here if + there are such copies, and if there are cancel this file's + purge. */ + ret = mdbox_file_purge_check_refcounts(ctx, &msgs_arr); + } + array_free(&msgs_arr); msgs = NULL; + + if (ret <= 0) { + /* failed */ + } else if (ctx->append_ctx == NULL) { + /* everything purged from this file */ + ret = 1; + } else { + /* assign new file_id + offset to moved messages */ + if (mdbox_map_append_move(ctx->append_ctx, &copied_map_uids, + &expunged_map_uids) < 0 || + mdbox_map_append_commit(ctx->append_ctx) < 0) + ret = -1; + else + ret = 1; + } + if (ctx->append_ctx != NULL) + mdbox_map_append_free(&ctx->append_ctx); + (void)mdbox_map_atomic_finish(&ctx->atomic); + + /* unlink only after unlocking map, so readers don't see it + temporarily vanished */ + if (ret > 0) { + (void)dbox_file_unlink(file); + if (mdbox_map_remove_file_id(ctx->storage->map, file_id) < 0) + ret = -1; + } else { + dbox_file_unlock(file); + } + array_free(&copied_map_uids); + array_free(&expunged_map_uids); + + (void)mdbox_purge_attachments(ctx, &ext_refs); + array_free(&ext_refs); + pool_unref(&ext_refs_pool); + return ret; +} + +void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->box); + ARRAY_TYPE(uint32_t) *dest; + uint32_t map_uid; + + /* we'll assume here that alt flag won't be changed multiple times + for the same mail. it shouldn't happen with current code, and + checking for it would just slow down the code. + + so the way it works currently is just that map_uids are added to + an array, which is later sorted and processed further. note that + it's still possible that the same map_uid exists in the array + multiple times. */ + if (mdbox_mail_lookup(mbox, mbox->box.view, mail->seq, &map_uid) < 0) + return; + + dest = move_to_alt ? &mbox->storage->move_to_alt_map_uids : + &mbox->storage->move_from_alt_map_uids; + + if (!array_is_created(dest)) + i_array_init(dest, 256); + array_push_back(dest, &map_uid); +} + +static struct mdbox_purge_context * +mdbox_purge_alloc(struct mdbox_storage *storage) +{ + struct mdbox_purge_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("mdbox purge context", 1024*32); + ctx = p_new(pool, struct mdbox_purge_context, 1); + ctx->pool = pool; + ctx->storage = storage; + ctx->lowest_primary_file_id = (uint32_t)-1; + i_array_init(&ctx->primary_file_ids, 64); + i_array_init(&ctx->purge_file_ids, 64); + hash_table_create_direct(&ctx->altmoves, pool, 0); + return ctx; +} + +static void mdbox_purge_free(struct mdbox_purge_context **_ctx) +{ + struct mdbox_purge_context *ctx = *_ctx; + + *_ctx = NULL; + + hash_table_destroy(&ctx->altmoves); + array_free(&ctx->primary_file_ids); + array_free(&ctx->purge_file_ids); + pool_unref(&ctx->pool); +} + +static int mdbox_purge_get_primary_files(struct mdbox_purge_context *ctx) +{ + struct mdbox_storage *dstorage = ctx->storage; + struct mail_storage *storage = &dstorage->storage.storage; + DIR *dir; + struct dirent *d; + string_t *path; + unsigned int file_id; + size_t dir_len; + int ret = 0; + + if (!array_is_created(&dstorage->move_to_alt_map_uids) && + !array_is_created(&dstorage->move_from_alt_map_uids)) { + /* we don't need to do alt moving, don't bother getting list + of primary files */ + return 0; + } + + dir = opendir(dstorage->storage_dir); + if (dir == NULL) { + if (errno == ENOENT) { + /* no storage directory at all yet */ + return 0; + } + mail_storage_set_critical(storage, + "opendir(%s) failed: %m", dstorage->storage_dir); + return -1; + } + + path = t_str_new(256); + str_append(path, dstorage->storage_dir); + str_append_c(path, '/'); + dir_len = str_len(path); + + for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) { + if (!str_begins(d->d_name, MDBOX_MAIL_FILE_PREFIX)) + continue; + if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX), + &file_id) < 0) + continue; + + str_truncate(path, dir_len); + str_append(path, d->d_name); + seq_range_array_add(&ctx->primary_file_ids, file_id); + } + if (array_count(&ctx->primary_file_ids) > 0) { + const struct seq_range *range = + array_front(&ctx->primary_file_ids); + ctx->lowest_primary_file_id = range[0].seq1; + } + if (errno != 0) { + mail_storage_set_critical(storage, + "readdir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(storage, + "closedir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + return ret; +} + +static int uint32_t_cmp(const uint32_t *u1, const uint32_t *u2) +{ + if (*u1 < *u2) + return -1; + if (*u1 > *u2) + return 1; + return 0; +} + +static int mdbox_altmove_add_files(struct mdbox_purge_context *ctx) +{ + struct mdbox_storage *dstorage = ctx->storage; + const uint32_t *map_uids; + unsigned int i, count, alt_refcount = 0; + struct mdbox_map_mail_index_record cur_rec; + enum mdbox_msg_action action; + uint32_t cur_map_uid; + uint16_t cur_refcount = 0; + uoff_t offset; + int ret = 0; + + /* first add move-to-alt actions */ + if (array_is_created(&dstorage->move_to_alt_map_uids)) { + array_sort(&dstorage->move_to_alt_map_uids, uint32_t_cmp); + map_uids = array_get(&dstorage->move_to_alt_map_uids, &count); + } else { + map_uids = NULL; + count = 0; + } + cur_map_uid = 0; + for (i = 0; i < count; i++) { + if (cur_map_uid != map_uids[i]) { + cur_map_uid = map_uids[i]; + if (mdbox_map_lookup_full(dstorage->map, cur_map_uid, + &cur_rec, &cur_refcount) < 0) { + cur_refcount = (uint16_t)-1; + ret = -1; + } + alt_refcount = 1; + } else { + alt_refcount++; + } + + if (alt_refcount == cur_refcount && + seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) { + /* all instances marked as moved to alt storage */ + action = MDBOX_MSG_ACTION_MOVE_TO_ALT; + hash_table_insert(ctx->altmoves, + POINTER_CAST(cur_map_uid), + POINTER_CAST(action)); + seq_range_array_add(&ctx->purge_file_ids, + cur_rec.file_id); + } + } + + /* next add move-from-alt actions. they override move-to-alt actions + in case there happen to be any conflicts (shouldn't). only a single + move-from-alt record is needed to do the move. */ + if (array_is_created(&dstorage->move_from_alt_map_uids)) + map_uids = array_get(&dstorage->move_from_alt_map_uids, &count); + else { + map_uids = NULL; + count = 0; + } + cur_map_uid = 0; + for (i = 0; i < count; i++) { + if (cur_map_uid == map_uids[i]) + continue; + cur_map_uid = map_uids[i]; + + if (mdbox_map_lookup(dstorage->map, cur_map_uid, + &cur_rec.file_id, &offset) < 0) { + ret = -1; + continue; + } + if (seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) { + /* already in primary storage */ + continue; + } + + action = MDBOX_MSG_ACTION_MOVE_FROM_ALT; + hash_table_update(ctx->altmoves, POINTER_CAST(cur_map_uid), + POINTER_CAST(action)); + seq_range_array_add(&ctx->purge_file_ids, cur_rec.file_id); + } + ctx->have_altmoves = hash_table_count(ctx->altmoves) > 0; + return ret; +} + +int mdbox_purge(struct mail_storage *_storage) +{ + struct mdbox_storage *storage = (struct mdbox_storage *)_storage; + struct mdbox_purge_context *ctx; + struct dbox_file *file; + struct seq_range_iter iter; + unsigned int i = 0; + uint32_t file_id; + bool deleted; + int ret; + + ctx = mdbox_purge_alloc(storage); + ret = mdbox_map_get_zero_ref_files(storage->map, &ctx->purge_file_ids); + if (storage->alt_storage_dir != NULL) { + if (mdbox_purge_get_primary_files(ctx) < 0) + ret = -1; + else { + /* add files that can be altmoved */ + if (mdbox_altmove_add_files(ctx) < 0) + ret = -1; + } + } + + seq_range_array_iter_init(&iter, &ctx->purge_file_ids); i = 0; + while (ret == 0 && + seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN { + file = mdbox_file_init(storage, file_id); + if (dbox_file_open(file, &deleted) > 0 && !deleted) { + if (mdbox_file_purge(ctx, file, file_id) < 0) + ret = -1; + } else { + if (mdbox_map_remove_file_id(storage->map, file_id) < 0) + ret = -1; + } + dbox_file_unref(&file); + } T_END; + mdbox_purge_free(&ctx); + + if (storage->corrupted) { + /* purging found corrupted files */ + (void)mdbox_storage_rebuild(storage); + ret = -1; + } + return ret; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c new file mode 100644 index 0000000..268f77c --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c @@ -0,0 +1,493 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "fdatasync-path.h" +#include "hex-binary.h" +#include "hex-dec.h" +#include "str.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "write-full.h" +#include "index-mail.h" +#include "index-pop3-uidl.h" +#include "mail-copy.h" +#include "dbox-save.h" +#include "mdbox-storage.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" + + +struct dbox_save_mail { + struct dbox_file_append_context *file_append; + uint32_t seq; + uint32_t append_offset; + time_t save_date; + bool written_to_disk; +}; + +struct mdbox_save_context { + struct dbox_save_context ctx; + + struct mdbox_mailbox *mbox; + struct mdbox_sync_context *sync_ctx; + + struct dbox_file *cur_file; + struct dbox_file_append_context *cur_file_append; + struct mdbox_map_append_context *append_ctx; + + ARRAY_TYPE(uint32_t) copy_map_uids; + struct mdbox_map_atomic_context *atomic; + struct mdbox_map_transaction_context *map_trans; + + ARRAY(struct dbox_save_mail) mails; +}; + +#define MDBOX_SAVECTX(s) container_of(DBOX_SAVECTX(s), struct mdbox_save_context, ctx) + +static struct dbox_file * +mdbox_copy_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(t->save_ctx); + const struct mdbox_mail_index_record *rec; + const void *data; + uint32_t file_id; + + mail_index_lookup_ext(t->view, seq, ctx->mbox->ext_id, &data, NULL); + rec = data; + + if (mdbox_map_lookup(ctx->mbox->storage->map, rec->map_uid, + &file_id, offset_r) < 0) + i_unreached(); + + return mdbox_file_init(ctx->mbox->storage, file_id); +} + +struct dbox_file * +mdbox_save_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(t->save_ctx); + const struct dbox_save_mail *mails, *mail; + unsigned int count; + + mails = array_get(&ctx->mails, &count); + i_assert(count > 0); + i_assert(seq >= mails[0].seq); + + mail = &mails[seq - mails[0].seq]; + i_assert(mail->seq == seq); + + if (mail->file_append == NULL) { + /* copied mail */ + return mdbox_copy_file_get_file(t, seq, offset_r); + } + + /* saved mail */ + i_assert(mail->written_to_disk); + if (dbox_file_append_flush(mail->file_append) < 0) + ctx->ctx.failed = TRUE; + + mail->file_append->file->refcount++; + *offset_r = mail->append_offset; + return mail->file_append->file; +} + +struct mail_save_context * +mdbox_save_alloc(struct mailbox_transaction_context *t) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(t->box); + struct mdbox_save_context *ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx != NULL) { + /* use the existing allocated structure */ + ctx = MDBOX_SAVECTX(t->save_ctx); + ctx->cur_file = NULL; + ctx->ctx.failed = FALSE; + ctx->ctx.finished = FALSE; + ctx->ctx.dbox_output = NULL; + ctx->cur_file_append = NULL; + return &ctx->ctx.ctx; + } + + ctx = i_new(struct mdbox_save_context, 1); + ctx->ctx.ctx.transaction = t; + ctx->ctx.trans = t->itrans; + ctx->mbox = mbox; + ctx->atomic = mdbox_map_atomic_begin(mbox->storage->map); + ctx->append_ctx = mdbox_map_append_begin(ctx->atomic); + i_array_init(&ctx->mails, 32); + t->save_ctx = &ctx->ctx.ctx; + return t->save_ctx; +} + +int mdbox_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct dbox_save_mail *save_mail; + uoff_t mail_size, append_offset; + + /* get the size of the mail to be saved, if possible */ + if (i_stream_get_size(input, TRUE, &mail_size) <= 0) { + /* we couldn't find out the exact size. fallback to non-exact, + maybe it'll give something useful. the mail size is used + only to figure out if it's causing mdbox file to grow + too large. */ + if (i_stream_get_size(input, FALSE, &mail_size) <= 0) + mail_size = 0; + } + if (mdbox_map_append_next(ctx->append_ctx, mail_size, 0, + &ctx->cur_file_append, + &ctx->ctx.dbox_output) < 0) { + ctx->ctx.failed = TRUE; + return -1; + } + i_assert(ctx->ctx.dbox_output->offset <= (uint32_t)-1); + append_offset = ctx->ctx.dbox_output->offset; + + ctx->cur_file = ctx->cur_file_append->file; + dbox_save_begin(&ctx->ctx, input); + + save_mail = array_append_space(&ctx->mails); + save_mail->file_append = ctx->cur_file_append; + save_mail->seq = ctx->ctx.seq; + save_mail->append_offset = append_offset; + return ctx->ctx.failed ? -1 : 0; +} + +static int mdbox_save_mail_write_metadata(struct mdbox_save_context *ctx, + struct dbox_save_mail *mail) +{ + struct dbox_file *file = mail->file_append->file; + struct dbox_message_header dbox_msg_hdr; + uoff_t message_size; + guid_128_t guid_128; + + i_assert(file->msg_header_size == sizeof(dbox_msg_hdr)); + + message_size = ctx->ctx.dbox_output->offset - + mail->append_offset - mail->file_append->file->msg_header_size; + + dbox_save_write_metadata(&ctx->ctx.ctx, ctx->ctx.dbox_output, + message_size, ctx->mbox->box.name, guid_128); + /* save the 128bit GUID to index so if the map index gets corrupted + we can still find the message */ + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_128, NULL); + + dbox_msg_header_fill(&dbox_msg_hdr, message_size); + if (o_stream_pwrite(ctx->ctx.dbox_output, &dbox_msg_hdr, + sizeof(dbox_msg_hdr), mail->append_offset) < 0) { + dbox_file_set_syscall_error(file, "pwrite()"); + return -1; + } + mail->written_to_disk = TRUE; + mail->save_date = ctx->ctx.ctx.data.save_date; + return 0; +} + +static int mdbox_save_finish_write(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + struct dbox_save_mail *mail; + + ctx->ctx.finished = TRUE; + if (ctx->ctx.dbox_output == NULL) + return -1; + + dbox_save_end(&ctx->ctx); + + mail = array_back_modifiable(&ctx->mails); + if (!ctx->ctx.failed) T_BEGIN { + if (mdbox_save_mail_write_metadata(ctx, mail) < 0) + ctx->ctx.failed = TRUE; + else + mdbox_map_append_finish(ctx->append_ctx); + } T_END; + + if (mail->file_append->file->input != NULL) { + /* if we try to read the saved mail before unlocking file, + make sure the input stream doesn't have stale data */ + i_stream_sync(mail->file_append->file->input); + } + i_stream_unref(&ctx->ctx.input); + + if (ctx->ctx.failed) { + index_storage_save_abort_last(&ctx->ctx.ctx, ctx->ctx.seq); + mdbox_map_append_abort(ctx->append_ctx); + array_pop_back(&ctx->mails); + return -1; + } + return 0; +} + +int mdbox_save_finish(struct mail_save_context *ctx) +{ + int ret; + + ret = mdbox_save_finish_write(ctx); + index_save_context_free(ctx); + return ret; +} + +void mdbox_save_cancel(struct mail_save_context *_ctx) +{ + struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)mdbox_save_finish(_ctx); +} + +static void +mdbox_save_set_map_uids(struct mdbox_save_context *ctx, + uint32_t first_map_uid, uint32_t last_map_uid) +{ + struct mdbox_mailbox *mbox = ctx->mbox; + struct mail_index_view *view = ctx->ctx.ctx.transaction->view; + const struct mdbox_mail_index_record *old_rec; + struct mdbox_mail_index_record rec; + const struct dbox_save_mail *mails; + unsigned int i, count; + const void *data; + uint32_t next_map_uid = first_map_uid; + + mdbox_update_header(mbox, ctx->ctx.trans, NULL); + + i_zero(&rec); + mails = array_get(&ctx->mails, &count); + for (i = 0; i < count; i++) { + mail_index_lookup_ext(view, mails[i].seq, mbox->ext_id, + &data, NULL); + old_rec = data; + if (old_rec != NULL && old_rec->map_uid != 0) { + /* message was copied. keep the existing map uid */ + continue; + } + + if (mails[i].save_date > 0) + rec.save_date = mails[i].save_date; + else + rec.save_date = ioloop_time; + rec.map_uid = next_map_uid++; + mail_index_update_ext(ctx->ctx.trans, mails[i].seq, + mbox->ext_id, &rec, NULL); + } + i_assert(next_map_uid == last_map_uid + 1); +} + +int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + const struct mail_index_header *hdr; + uint32_t first_map_uid, last_map_uid; + + i_assert(ctx->ctx.finished); + + /* flush/fsync writes to m.* files before locking the map */ + if (mdbox_map_append_flush(ctx->append_ctx) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* make sure the map gets locked */ + if (mdbox_map_atomic_lock(ctx->atomic, "saving") < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + /* lock the mailbox after map to avoid deadlocks. if we've noticed + any corruption, deal with it later, otherwise we won't have + up-to-date atomic->sync_view */ + if (mdbox_sync_begin(ctx->mbox, MDBOX_SYNC_FLAG_NO_PURGE | + MDBOX_SYNC_FLAG_FORCE | + MDBOX_SYNC_FLAG_FSYNC | + MDBOX_SYNC_FLAG_NO_REBUILD, ctx->atomic, + &ctx->sync_ctx) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + i_assert(ctx->sync_ctx != NULL); + + /* assign map UIDs for newly saved messages after we've successfully + acquired all the locks. the transaction is now very unlikely to + fail. the UIDs are written to the transaction log immediately within + this function, but the map is left locked. */ + if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* update dbox header flags */ + dbox_save_update_header_flags(&ctx->ctx, ctx->sync_ctx->sync_view, + ctx->mbox->hdr_ext_id, offsetof(struct mdbox_index_header, flags)); + + /* assign UIDs for new messages */ + hdr = mail_index_get_header(ctx->sync_ctx->sync_view); + mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, + &_t->changes->saved_uids); + + if (ctx->ctx.highest_pop3_uidl_seq != 0) { + const struct dbox_save_mail *mails; + struct seq_range_iter iter; + unsigned int highest_pop3_uidl_idx; + uint32_t uid; + + mails = array_front(&ctx->mails); + highest_pop3_uidl_idx = + ctx->ctx.highest_pop3_uidl_seq - mails[0].seq; + i_assert(mails[highest_pop3_uidl_idx].seq == ctx->ctx.highest_pop3_uidl_seq); + + seq_range_array_iter_init(&iter, &_t->changes->saved_uids); + if (!seq_range_array_iter_nth(&iter, highest_pop3_uidl_idx, &uid)) + i_unreached(); + index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid); + } + + /* save map UIDs to mailbox index */ + if (first_map_uid != 0) + mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid); + + /* increase map's refcount for copied mails */ + if (array_is_created(&ctx->copy_map_uids)) { + ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE); + if (mdbox_map_update_refcounts(ctx->map_trans, + &ctx->copy_map_uids, 1) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + mail_index_sync_set_reason(ctx->sync_ctx->index_sync_ctx, "copying"); + } else { + mail_index_sync_set_reason(ctx->sync_ctx->index_sync_ctx, "saving"); + } + + _t->changes->uid_validity = hdr->uid_validity; + return 0; +} + +void mdbox_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct mailbox *box = _ctx->transaction->box; + struct mail_storage *_storage = box->storage; + struct mdbox_storage *storage = + container_of(_storage, struct mdbox_storage, storage.storage); + + _ctx->transaction = NULL; /* transaction is already freed */ + + mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx, + result); + + /* finish writing the mailbox APPENDs */ + if (mdbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) { + /* commit refcount increases for copied mails */ + if (ctx->map_trans != NULL) { + if (mdbox_map_transaction_commit(ctx->map_trans, "copy refcount updates") < 0) + mdbox_map_atomic_set_failed(ctx->atomic); + } + /* flush file append writes */ + if (mdbox_map_append_commit(ctx->append_ctx) < 0) + mdbox_map_atomic_set_failed(ctx->atomic); + } + mdbox_map_append_free(&ctx->append_ctx); + /* update the sync tail offset, everything else + was already written at this point. */ + (void)mdbox_map_atomic_finish(&ctx->atomic); + + if (_storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync_path(storage->storage_dir) < 0) { + mailbox_set_critical(box, + "fdatasync_path(%s) failed: %m", + storage->storage_dir); + } + } + mdbox_transaction_save_rollback(_ctx); +} + +void mdbox_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + + if (!ctx->ctx.finished) + mdbox_save_cancel(&ctx->ctx.ctx); + if (ctx->append_ctx != NULL) + mdbox_map_append_free(&ctx->append_ctx); + if (ctx->map_trans != NULL) + mdbox_map_transaction_free(&ctx->map_trans); + if (ctx->atomic != NULL) + (void)mdbox_map_atomic_finish(&ctx->atomic); + if (array_is_created(&ctx->copy_map_uids)) + array_free(&ctx->copy_map_uids); + + if (ctx->sync_ctx != NULL) + (void)mdbox_sync_finish(&ctx->sync_ctx, FALSE); + + array_free(&ctx->mails); + i_free(ctx); +} + +int mdbox_copy(struct mail_save_context *_ctx, struct mail *mail) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct dbox_save_mail *save_mail; + struct mdbox_mailbox *src_mbox; + struct mdbox_mail_index_record rec; + const void *guid_data; + guid_128_t wanted_guid; + + ctx->ctx.finished = TRUE; + + if (mail->box->storage != _ctx->transaction->box->storage || + _ctx->transaction->box->disable_reflink_copy_to) + return mail_storage_copy(_ctx, mail); + src_mbox = MDBOX_MAILBOX(mail->box); + + i_zero(&rec); + rec.save_date = ioloop_time; + if (mdbox_mail_lookup(src_mbox, mail->transaction->view, mail->seq, + &rec.map_uid) < 0) { + index_save_context_free(_ctx); + return -1; + } + + mail_index_lookup_ext(mail->transaction->view, mail->seq, + src_mbox->guid_ext_id, &guid_data, NULL); + if (guid_data == NULL || guid_128_is_empty(guid_data)) { + /* missing GUID, something's broken. don't copy using + refcounting. */ + return mail_storage_copy(_ctx, mail); + } else if (_ctx->data.guid != NULL && + (guid_128_from_string(_ctx->data.guid, wanted_guid) < 0 || + memcmp(guid_data, wanted_guid, sizeof(wanted_guid)) != 0)) { + /* GUID change requested. we can't do it with refcount + copying */ + return mail_storage_copy(_ctx, mail); + } + + /* remember the map_uid so we can later increase its refcount */ + if (!array_is_created(&ctx->copy_map_uids)) + i_array_init(&ctx->copy_map_uids, 32); + array_push_back(&ctx->copy_map_uids, &rec.map_uid); + + /* add message to mailbox index */ + dbox_save_add_to_index(&ctx->ctx); + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->ext_id, &rec, NULL); + + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_data, NULL); + index_copy_cache_fields(_ctx, mail, ctx->ctx.seq); + + save_mail = array_append_space(&ctx->mails); + save_mail->seq = ctx->ctx.seq; + + mail_set_seq_saving(_ctx->dest_mail, ctx->ctx.seq); + index_save_context_free(_ctx); + return 0; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-settings.c b/src/lib-storage/index/dbox-multi/mdbox-settings.c new file mode 100644 index 0000000..dc9e989 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.c @@ -0,0 +1,43 @@ +/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "mdbox-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct mdbox_settings) + +static const struct setting_define mdbox_setting_defines[] = { + DEF(BOOL, mdbox_preallocate_space), + DEF(SIZE, mdbox_rotate_size), + DEF(TIME, mdbox_rotate_interval), + + SETTING_DEFINE_LIST_END +}; + +static const struct mdbox_settings mdbox_default_settings = { + .mdbox_preallocate_space = FALSE, + .mdbox_rotate_size = 10*1024*1024, + .mdbox_rotate_interval = 0 +}; + +static const struct setting_parser_info mdbox_setting_parser_info = { + .module_name = "mdbox", + .defines = mdbox_setting_defines, + .defaults = &mdbox_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct mdbox_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info +}; + +const struct setting_parser_info *mdbox_get_setting_parser_info(void) +{ + return &mdbox_setting_parser_info; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-settings.h b/src/lib-storage/index/dbox-multi/mdbox-settings.h new file mode 100644 index 0000000..353da69 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.h @@ -0,0 +1,12 @@ +#ifndef MDBOX_SETTINGS_H +#define MDBOX_SETTINGS_H + +struct mdbox_settings { + bool mdbox_preallocate_space; + uoff_t mdbox_rotate_size; + unsigned int mdbox_rotate_interval; +}; + +const struct setting_parser_info *mdbox_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c new file mode 100644 index 0000000..4277a1a --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c @@ -0,0 +1,1005 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "hash.h" +#include "str.h" +#include "mail-cache.h" +#include "index-rebuild.h" +#include "mail-namespace.h" +#include "mailbox-list-private.h" +#include "mdbox-storage.h" +#include "mdbox-file.h" +#include "mdbox-map-private.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" + +#include <dirent.h> +#include <unistd.h> + +#define REBUILD_MAX_REFCOUNT 32768 + +struct mdbox_rebuild_msg { + struct mdbox_rebuild_msg *guid_hash_next; + + guid_128_t guid_128; + uint32_t file_id; + uint32_t offset; + uint32_t rec_size; + uoff_t mail_size; + uint32_t map_uid; + + uint16_t refcount; + bool seen_zero_ref_in_map:1; +}; + +struct rebuild_msg_mailbox { + struct mailbox *box; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + uint32_t next_uid; +}; + +struct mdbox_storage_rebuild_context { + struct mdbox_storage *storage; + struct mdbox_map_atomic_context *atomic; + pool_t pool; + + struct mdbox_map_mail_index_header orig_map_hdr; + HASH_TABLE(uint8_t *, struct mdbox_rebuild_msg *) guid_hash; + ARRAY(struct mdbox_rebuild_msg *) msgs; + ARRAY_TYPE(seq_range) seen_file_ids; + + uint32_t rebuild_count; + uint32_t highest_file_id; + + struct mailbox_list *default_list; + + struct rebuild_msg_mailbox prev_msg; + + bool have_pop3_uidls:1; + bool have_pop3_orders:1; +}; + +static struct mdbox_storage_rebuild_context * +mdbox_storage_rebuild_init(struct mdbox_storage *storage, + struct mdbox_map_atomic_context *atomic) +{ + struct mdbox_storage_rebuild_context *ctx; + + i_assert(!storage->rebuilding_storage); + + ctx = i_new(struct mdbox_storage_rebuild_context, 1); + ctx->storage = storage; + ctx->atomic = atomic; + ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256); + hash_table_create(&ctx->guid_hash, ctx->pool, 0, + guid_128_hash, guid_128_cmp); + i_array_init(&ctx->msgs, 512); + i_array_init(&ctx->seen_file_ids, 128); + + ctx->storage->rebuilding_storage = TRUE; + return ctx; +} + +static void +mdbox_storage_rebuild_deinit(struct mdbox_storage_rebuild_context *ctx) +{ + i_assert(ctx->storage->rebuilding_storage); + + ctx->storage->rebuilding_storage = FALSE; + + hash_table_destroy(&ctx->guid_hash); + pool_unref(&ctx->pool); + array_free(&ctx->seen_file_ids); + array_free(&ctx->msgs); + i_free(ctx); +} + +static int +mdbox_rebuild_msg_offset_cmp(struct mdbox_rebuild_msg *const *m1, + struct mdbox_rebuild_msg *const *m2) +{ + if ((*m1)->file_id < (*m2)->file_id) + return -1; + if ((*m1)->file_id > (*m2)->file_id) + return 1; + + if ((*m1)->offset < (*m2)->offset) + return -1; + if ((*m1)->offset > (*m2)->offset) + return 1; + + if ((*m1)->rec_size < (*m2)->rec_size) + return -1; + if ((*m1)->rec_size > (*m2)->rec_size) + return 1; + return 0; +} + +static int mdbox_rebuild_msg_uid_cmp(struct mdbox_rebuild_msg *const *m1, + struct mdbox_rebuild_msg *const *m2) +{ + if ((*m1)->map_uid < (*m2)->map_uid) + return -1; + if ((*m1)->map_uid > (*m2)->map_uid) + return 1; + return 0; +} + +static void rebuild_scan_metadata(struct mdbox_storage_rebuild_context *ctx, + struct dbox_file *file) +{ + if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_UIDL) != NULL) + ctx->have_pop3_uidls = TRUE; + if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_ORDER) != NULL) + ctx->have_pop3_orders = TRUE; +} + +static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx, + struct dbox_file *file, uint32_t file_id) +{ + const char *guid; + uint8_t *guid_p; + struct mdbox_rebuild_msg *rec, *old_rec; + uoff_t offset, prev_offset; + bool last, first, fixed = FALSE; + int ret; + + dbox_file_seek_rewind(file); + prev_offset = 0; + while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) { + if (ret > 0) { + if ((ret = dbox_file_metadata_read(file)) < 0) + break; + } + + if (ret == 0) { + /* file is corrupted. fix it and retry. */ + if (fixed || last) + break; + first = prev_offset == 0; + if (prev_offset == 0) { + /* use existing file header if it was ok */ + prev_offset = offset; + } + if ((ret = dbox_file_fix(file, prev_offset)) < 0) + break; + if (ret == 0) { + /* file was deleted */ + return 1; + } + fixed = TRUE; + if (!first) { + /* seek to the offset where we last left off */ + ret = dbox_file_seek(file, prev_offset); + if (ret <= 0) + break; + } + continue; + } + prev_offset = offset; + + guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID); + if (guid == NULL || *guid == '\0') { + dbox_file_set_corrupted(file, + "Message is missing GUID"); + ret = 0; + break; + } + rebuild_scan_metadata(ctx, file); + + rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1); + rec->file_id = file_id; + rec->offset = offset; + rec->rec_size = file->input->v_offset - offset; + rec->mail_size = dbox_file_get_plaintext_size(file); + mail_generate_guid_128_hash(guid, rec->guid_128); + i_assert(!guid_128_is_empty(rec->guid_128)); + array_push_back(&ctx->msgs, &rec); + + guid_p = rec->guid_128; + old_rec = hash_table_lookup(ctx->guid_hash, guid_p); + if (old_rec == NULL) + hash_table_insert(ctx->guid_hash, guid_p, rec); + else if (rec->mail_size == old_rec->mail_size) { + /* two mails' GUID and size are the same, which quite + likely means that their contents are the same as + well. we'll compare the mail sizes instead of the + record sizes, because the records' metadata may + differ. + + save this duplicate mail with refcount=0 to the map, + so it will eventually be purged. */ + rec->seen_zero_ref_in_map = TRUE; + } else { + /* duplicate GUID, but not a duplicate message. */ + i_error("mdbox %s: Duplicate GUID %s in " + "m.%u:%u (size=%"PRIuUOFF_T") and m.%u:%u " + "(size=%"PRIuUOFF_T")", + ctx->storage->storage_dir, guid, + old_rec->file_id, old_rec->offset, old_rec->mail_size, + rec->file_id, rec->offset, rec->mail_size); + rec->guid_hash_next = old_rec->guid_hash_next; + old_rec->guid_hash_next = rec; + } + } + if (ret < 0) + return -1; + else if (ret == 0 && !last) + return 0; + else + return 1; +} + +static int +rebuild_rename_file(struct mdbox_storage_rebuild_context *ctx, + const char *dir, const char **fname_p, uint32_t *file_id_r) +{ + const char *old_path, *new_path, *fname = *fname_p; + + old_path = t_strconcat(dir, "/", fname, NULL); + do { + new_path = t_strdup_printf("%s/"MDBOX_MAIL_FILE_FORMAT, + dir, ++ctx->highest_file_id); + /* use link()+unlink() instead of rename() to make sure we + don't overwrite any files. */ + if (link(old_path, new_path) == 0) { + i_unlink(old_path); + *fname_p = strrchr(new_path, '/') + 1; + *file_id_r = ctx->highest_file_id; + return 0; + } + } while (errno == EEXIST); + + i_error("link(%s, %s) failed: %m", old_path, new_path); + return -1; +} + +static int rebuild_add_file(struct mdbox_storage_rebuild_context *ctx, + const char *dir, const char *fname) +{ + struct dbox_file *file; + uint32_t file_id; + const char *id_str, *ext; + bool deleted; + int ret = 0; + + id_str = fname + strlen(MDBOX_MAIL_FILE_PREFIX); + if (str_to_uint32(id_str, &file_id) < 0 || file_id == 0) { + /* m.*.broken files are created by file fixing + m.*.lock files are created if flock() isn't available */ + ext = strrchr(id_str, '.'); + if (ext == NULL || (strcmp(ext, ".broken") != 0 && + strcmp(ext, ".lock") != 0)) { + i_warning("mdbox rebuild: " + "Skipping file with missing ID: %s/%s", + dir, fname); + } + return 0; + } + if (!seq_range_exists(&ctx->seen_file_ids, file_id)) { + if (ctx->highest_file_id < file_id) + ctx->highest_file_id = file_id; + } else { + /* duplicate file. either readdir() returned it twice + (unlikely) or it exists in both alt and primary storage. + to make sure we don't lose any mails from either of the + files, give this file a new ID and rename it. */ + if (rebuild_rename_file(ctx, dir, &fname, &file_id) < 0) + return -1; + } + seq_range_array_add(&ctx->seen_file_ids, file_id); + + file = mdbox_file_init(ctx->storage, file_id); + if ((ret = dbox_file_open(file, &deleted)) > 0 && !deleted) + ret = rebuild_file_mails(ctx, file, file_id); + if (ret == 0) + i_error("mdbox rebuild: Failed to fix file %s/%s", dir, fname); + dbox_file_unref(&file); + return ret < 0 ? -1 : 0; +} + +static void +rebuild_add_missing_map_uids(struct mdbox_storage_rebuild_context *ctx, + uint32_t next_uid) +{ + struct mdbox_rebuild_msg **msgs; + struct mdbox_map_mail_index_record rec; + unsigned int i, count; + uint32_t seq; + + i_zero(&rec); + msgs = array_get_modifiable(&ctx->msgs, &count); + for (i = 0; i < count; i++) { + if (msgs[i]->map_uid != 0) + continue; + + rec.file_id = msgs[i]->file_id; + rec.offset = msgs[i]->offset; + rec.size = msgs[i]->rec_size; + + msgs[i]->map_uid = next_uid++; + mail_index_append(ctx->atomic->sync_trans, + msgs[i]->map_uid, &seq); + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->storage->map->map_ext_id, + &rec, NULL); + } +} + +static void rebuild_apply_map(struct mdbox_storage_rebuild_context *ctx) +{ + struct mdbox_map *map = ctx->storage->map; + const struct mail_index_header *hdr; + struct mdbox_rebuild_msg **pos; + struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg; + struct dbox_mail_lookup_rec rec; + uint32_t seq; + + array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp); + /* msgs now contains a list of all messages that exists in m.* files, + sorted by file_id,offset */ + + hdr = mail_index_get_header(ctx->atomic->sync_view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + if (mdbox_map_view_lookup_rec(map, ctx->atomic->sync_view, + seq, &rec) < 0) { + /* map or ref extension is missing from the index. + Just ignore the file entirely. (Don't try to + continue with other records, since they'll fail + as well, and each failure logs the same error.) */ + i_assert(seq == 1); + break; + } + + /* look up the rebuild msg record for this message based on + the (file_id, offset, size) triplet */ + search_msg.file_id = rec.rec.file_id; + search_msg.offset = rec.rec.offset; + search_msg.rec_size = rec.rec.size; + pos = array_bsearch(&ctx->msgs, &search_msgp, + mdbox_rebuild_msg_offset_cmp); + if (pos == NULL || (*pos)->map_uid != 0) { + /* map record points to nonexistent or + a duplicate message. */ + mail_index_expunge(ctx->atomic->sync_trans, seq); + } else { + /* remember this message's map_uid */ + (*pos)->map_uid = rec.map_uid; + if (rec.refcount == 0) + (*pos)->seen_zero_ref_in_map = TRUE; + } + } + rebuild_add_missing_map_uids(ctx, hdr->next_uid); + + /* afterwards we're interested in looking up map_uids. + re-sort the messages to make it easier. */ + array_sort(&ctx->msgs, mdbox_rebuild_msg_uid_cmp); +} + +static struct mdbox_rebuild_msg * +rebuild_lookup_map_uid(struct mdbox_storage_rebuild_context *ctx, + uint32_t map_uid) +{ + struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg; + struct mdbox_rebuild_msg **pos; + + search_msg.map_uid = map_uid; + pos = array_bsearch(&ctx->msgs, &search_msgp, + mdbox_rebuild_msg_uid_cmp); + return pos == NULL ? NULL : *pos; +} + +static bool +guid_hash_have_map_uid(struct mdbox_rebuild_msg **recp, uint32_t map_uid) +{ + struct mdbox_rebuild_msg *rec; + + for (rec = *recp; rec != NULL; rec = rec->guid_hash_next) { + if (rec->map_uid == map_uid) { + *recp = rec; + return TRUE; + } + } + return FALSE; +} + +static void +rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx, + struct index_rebuild_context *rebuild_ctx, + struct mdbox_mailbox *mbox, + struct mail_index_view *view, + struct mail_index_transaction *trans) +{ + struct mdbox_mail_index_record new_dbox_rec; + const struct mail_index_header *hdr; + struct mdbox_rebuild_msg *rec; + const void *data; + const uint8_t *guid_p; + uint32_t old_seq, new_seq, uid, map_uid; + + /* Rebuild the mailbox's index. Note that index is reset at this point, + so although we can still access the old messages, we'll need to + append anything we want to keep as new messages. */ + hdr = mail_index_get_header(view); + for (old_seq = 1; old_seq <= hdr->messages_count; old_seq++) { + mail_index_lookup_ext(view, old_seq, mbox->ext_id, + &data, NULL); + if (data == NULL) { + i_zero(&new_dbox_rec); + map_uid = 0; + } else { + memcpy(&new_dbox_rec, data, sizeof(new_dbox_rec)); + map_uid = new_dbox_rec.map_uid; + } + + mail_index_lookup_ext(view, old_seq, mbox->guid_ext_id, + &data, NULL); + guid_p = data; + + /* see if we can find this message based on + 1) GUID, 2) map_uid */ + rec = guid_p == NULL ? NULL : + hash_table_lookup(ctx->guid_hash, guid_p); + if (rec == NULL) { + /* multi-dbox message that wasn't found with GUID. + either it's lost or GUID has been corrupted. we can + still try to look it up using map_uid. */ + rec = map_uid == 0 ? NULL : + rebuild_lookup_map_uid(ctx, map_uid); + map_uid = rec == NULL ? 0 : rec->map_uid; + } else if (!guid_hash_have_map_uid(&rec, map_uid)) { + /* message's GUID and map_uid point to different + physical messages. assume that GUID is correct and + map_uid is wrong. */ + map_uid = rec->map_uid; + } else { + /* everything was ok. use this specific record's + map_uid to avoid duplicating mails in case the same + GUID exists multiple times */ + } + + if (rec != NULL && + rec->refcount < REBUILD_MAX_REFCOUNT) T_BEGIN { + /* keep this message. add it to mailbox index. */ + i_assert(map_uid != 0); + rec->refcount++; + + mail_index_lookup_uid(view, old_seq, &uid); + mail_index_append(trans, uid, &new_seq); + index_rebuild_index_metadata(rebuild_ctx, + new_seq, uid); + + new_dbox_rec.map_uid = map_uid; + mail_index_update_ext(trans, new_seq, mbox->ext_id, + &new_dbox_rec, NULL); + mail_index_update_ext(trans, new_seq, mbox->guid_ext_id, + rec->guid_128, NULL); + } T_END; + } +} + +static void +mdbox_rebuild_get_header(struct mail_index_view *view, uint32_t hdr_ext_id, + struct mdbox_index_header *hdr_r, bool *need_resize_r) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(view, hdr_ext_id, &data, &data_size); + i_zero(hdr_r); + memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r))); + *need_resize_r = data_size < sizeof(*hdr_r); +} + +static void mdbox_header_update(struct mdbox_storage_rebuild_context *ctx, + struct index_rebuild_context *rebuild_ctx, + struct mdbox_mailbox *mbox) +{ + struct mdbox_index_header hdr, backup_hdr; + bool need_resize, need_resize_backup; + + mdbox_rebuild_get_header(rebuild_ctx->view, mbox->hdr_ext_id, + &hdr, &need_resize); + if (rebuild_ctx->backup_view == NULL) { + i_zero(&backup_hdr); + need_resize = TRUE; + } else { + mdbox_rebuild_get_header(rebuild_ctx->backup_view, + mbox->hdr_ext_id, &backup_hdr, + &need_resize_backup); + } + + /* make sure we have valid mailbox guid */ + if (guid_128_is_empty(hdr.mailbox_guid)) { + if (!guid_128_is_empty(backup_hdr.mailbox_guid)) { + memcpy(hdr.mailbox_guid, backup_hdr.mailbox_guid, + sizeof(hdr.mailbox_guid)); + } else { + guid_128_generate(hdr.mailbox_guid); + } + } + + /* update map's uid-validity */ + hdr.map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map); + + if (ctx->have_pop3_uidls) + hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS; + if (ctx->have_pop3_orders) + hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS; + + /* and write changes */ + if (need_resize) { + mail_index_ext_resize_hdr(rebuild_ctx->trans, mbox->hdr_ext_id, + sizeof(hdr)); + } + mail_index_update_header_ext(rebuild_ctx->trans, mbox->hdr_ext_id, 0, + &hdr, sizeof(hdr)); +} + +static int +rebuild_mailbox(struct mdbox_storage_rebuild_context *ctx, + struct mail_namespace *ns, const char *vname) +{ + struct mailbox *box; + struct mdbox_mailbox *mbox; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + struct index_rebuild_context *rebuild_ctx; + enum mail_error error; + int ret; + + box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY | + MAILBOX_FLAG_IGNORE_ACLS); + if (box->storage != &ctx->storage->storage.storage) { + /* the namespace has multiple storages. */ + mailbox_free(&box); + return 0; + } + if (mailbox_open(box) < 0) { + error = mailbox_get_last_mail_error(box); + i_error("Couldn't open mailbox '%s': %s", + vname, mailbox_get_last_internal_error(box, NULL)); + mailbox_free(&box); + if (error == MAIL_ERROR_TEMP) + return -1; + /* non-temporary error, ignore */ + return 0; + } + mbox = MDBOX_MAILBOX(box); + + ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans, + MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES); + if (ret <= 0) { + i_assert(ret != 0); + mailbox_set_index_error(box); + mailbox_free(&box); + return -1; + } + + rebuild_ctx = index_index_rebuild_init(&mbox->box, view, trans); + mdbox_header_update(ctx, rebuild_ctx, mbox); + rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans); + index_index_rebuild_deinit(&rebuild_ctx, dbox_get_uidvalidity_next); + mail_index_unset_fscked(trans); + + mail_index_sync_set_reason(sync_ctx, "mdbox storage rebuild"); + if (mail_index_sync_commit(&sync_ctx) < 0) { + mailbox_set_index_error(box); + ret = -1; + } + + mailbox_free(&box); + return ret < 0 ? -1 : 0; +} + +static int +rebuild_namespace_mailboxes(struct mdbox_storage_rebuild_context *ctx, + struct mail_namespace *ns) +{ + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + int ret = 0; + + if (ctx->default_list == NULL || + (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) + ctx->default_list = ns->list; + + iter = mailbox_list_iter_init(ns->list, "*", + MAILBOX_LIST_ITER_RAW_LIST | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & (MAILBOX_NONEXISTENT | + MAILBOX_NOSELECT)) == 0) { + T_BEGIN { + ret = rebuild_mailbox(ctx, ns, info->vname); + } T_END; + if (ret < 0) { + ret = -1; + break; + } + } + } + if (mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int rebuild_mailboxes(struct mdbox_storage_rebuild_context *ctx) +{ + struct mail_storage *storage = &ctx->storage->storage.storage; + struct mail_namespace *ns; + + for (ns = storage->user->namespaces; ns != NULL; ns = ns->next) { + if (ns->storage == storage && ns->alias_for == NULL) { + if (rebuild_namespace_mailboxes(ctx, ns) < 0) + return -1; + } + } + if (ctx->default_list == NULL) + i_panic("No namespace found for storage=%s", storage->name); + return 0; +} + +static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg) +{ + mail_index_sync_set_reason(msg->sync_ctx, "mdbox storage rebuild"); + if (mail_index_sync_commit(&msg->sync_ctx) < 0) + return -1; + mailbox_free(&msg->box); + i_zero(msg); + return 0; +} + +static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx, + struct mdbox_rebuild_msg *msg) +{ + struct mail_storage *storage = &ctx->storage->storage.storage; + struct dbox_file *file; + const struct mail_index_header *hdr; + struct mdbox_mail_index_record dbox_rec; + const char *mailbox = NULL; + struct mailbox *box; + struct mdbox_mailbox *mbox; + enum mail_error error; + bool deleted, created; + int ret; + uint32_t seq; + + /* first see if message contains the mailbox it was originally + saved to */ + file = mdbox_file_init(ctx->storage, msg->file_id); + ret = dbox_file_open(file, &deleted); + if (ret > 0 && !deleted) + ret = dbox_file_seek(file, msg->offset); + if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) { + mailbox = dbox_file_metadata_get(file, + DBOX_METADATA_ORIG_MAILBOX); + if (mailbox != NULL) { + mailbox = mailbox_list_get_vname(ctx->default_list, mailbox); + mailbox = t_strdup(mailbox); + } + rebuild_scan_metadata(ctx, file); + } + dbox_file_unref(&file); + if (ret <= 0 || deleted) { + if (ret < 0) + return -1; + /* we shouldn't get here, so apparently we couldn't fix + something. just ignore the mail.. */ + return 0; + } + + if (mailbox == NULL) + mailbox = "INBOX"; + + /* we have the destination mailbox. now open it and add the message + there. */ + created = FALSE; + box = ctx->prev_msg.box != NULL && + strcmp(mailbox, ctx->prev_msg.box->vname) == 0 ? + ctx->prev_msg.box : NULL; + while (box == NULL) { + box = mailbox_alloc(ctx->default_list, mailbox, + MAILBOX_FLAG_READONLY | + MAILBOX_FLAG_IGNORE_ACLS); + i_assert(box->storage == storage); + if (mailbox_open(box) == 0) + break; + + error = mailbox_get_last_mail_error(box); + if (error == MAIL_ERROR_NOTFOUND && !created) { + /* mailbox doesn't exist currently? see if creating + it helps. */ + created = TRUE; + (void)mailbox_create(box, NULL, FALSE); + mailbox_free(&box); + continue; + } + + mailbox_free(&box); + if (error == MAIL_ERROR_TEMP) + return -1; + + if (strcmp(mailbox, "INBOX") != 0) { + /* see if we can save to INBOX instead. */ + mailbox = "INBOX"; + } else { + /* this shouldn't happen */ + return -1; + } + } + mbox = MDBOX_MAILBOX(box); + + /* switch the mailbox cache if necessary */ + if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) { + if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) + return -1; + } + if (ctx->prev_msg.box == NULL) { + ret = mail_index_sync_begin(box->index, + &ctx->prev_msg.sync_ctx, + &ctx->prev_msg.view, + &ctx->prev_msg.trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mailbox_set_index_error(box); + mailbox_free(&box); + return -1; + } + ctx->prev_msg.box = box; + hdr = mail_index_get_header(ctx->prev_msg.view); + ctx->prev_msg.next_uid = hdr->next_uid; + } + + /* add the new message */ + i_zero(&dbox_rec); + dbox_rec.map_uid = msg->map_uid; + dbox_rec.save_date = ioloop_time; + mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq); + mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->ext_id, + &dbox_rec, NULL); + mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id, + msg->guid_128, NULL); + + i_assert(msg->refcount == 0); + msg->refcount++; + return 0; +} + +static int rebuild_handle_zero_refs(struct mdbox_storage_rebuild_context *ctx) +{ + struct mdbox_rebuild_msg **msgs; + unsigned int i, count; + + /* if we have messages at this point which have refcount=0, they're + either already expunged or they were somehow lost for some reason. + we'll need to figure out what to do about them. */ + msgs = array_get_modifiable(&ctx->msgs, &count); + for (i = 0; i < count; i++) { + if (msgs[i]->refcount != 0) + continue; + + if (msgs[i]->seen_zero_ref_in_map) { + /* we've seen the map record, trust it. */ + continue; + } + /* either map record was lost for this message or the message + was lost from its mailbox. safest way to handle this is to + restore the message. */ + if (rebuild_restore_msg(ctx, msgs[i]) < 0) + return -1; + } + if (ctx->prev_msg.box != NULL) { + if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) + return -1; + } + return 0; +} + +static void rebuild_update_refcounts(struct mdbox_storage_rebuild_context *ctx) +{ + const struct mail_index_header *hdr; + const void *data; + struct mdbox_rebuild_msg **msgs; + const uint16_t *ref16_p; + uint32_t seq, map_uid; + unsigned int i, count; + + /* update refcounts for existing map records */ + msgs = array_get_modifiable(&ctx->msgs, &count); + hdr = mail_index_get_header(ctx->atomic->sync_view); + for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) { + mail_index_lookup_uid(ctx->atomic->sync_view, seq, &map_uid); + if (map_uid != msgs[i]->map_uid) { + /* we've already expunged this map record */ + i_assert(map_uid < msgs[i]->map_uid); + continue; + } + + mail_index_lookup_ext(ctx->atomic->sync_view, seq, + ctx->storage->map->ref_ext_id, + &data, NULL); + ref16_p = data; + if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) { + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->storage->map->ref_ext_id, + &msgs[i]->refcount, NULL); + } + i++; + } + + /* update refcounts for newly created map records */ + for (; i < count; i++, seq++) { + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->storage->map->ref_ext_id, + &msgs[i]->refcount, NULL); + } +} + +static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx) +{ + struct mdbox_map_mail_index_header map_hdr; + + i_assert(ctx->default_list != NULL); + + if (rebuild_handle_zero_refs(ctx) < 0) + return -1; + rebuild_update_refcounts(ctx); + + /* update map header */ + map_hdr = ctx->orig_map_hdr; + map_hdr.highest_file_id = ctx->highest_file_id; + map_hdr.rebuild_count = ++ctx->rebuild_count; + + mail_index_update_header_ext(ctx->atomic->sync_trans, + ctx->storage->map->map_ext_id, + 0, &map_hdr, sizeof(map_hdr)); + return 0; +} + +static int +mdbox_storage_rebuild_scan_dir(struct mdbox_storage_rebuild_context *ctx, + const char *storage_dir, bool alt) +{ + DIR *dir; + struct dirent *d; + int ret = 0; + + dir = opendir(storage_dir); + if (dir == NULL) { + if (alt && errno == ENOENT) + return 0; + + mail_storage_set_critical(&ctx->storage->storage.storage, + "opendir(%s) failed: %m", storage_dir); + return -1; + } + for (errno = 0; (d = readdir(dir)) != NULL && ret == 0; errno = 0) { + if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, + strlen(MDBOX_MAIL_FILE_PREFIX)) == 0) T_BEGIN { + ret = rebuild_add_file(ctx, storage_dir, d->d_name); + } T_END; + } + if (ret == 0 && errno != 0) { + mail_storage_set_critical(&ctx->storage->storage.storage, + "readdir(%s) failed: %m", storage_dir); + ret = -1; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(&ctx->storage->storage.storage, + "closedir(%s) failed: %m", storage_dir); + ret = -1; + } + return ret; +} + +static int +mdbox_storage_rebuild_scan_prepare(struct mdbox_storage_rebuild_context *ctx) +{ + const void *data; + size_t data_size; + + if (mdbox_map_open_or_create(ctx->storage->map) < 0) + return -1; + + /* begin by locking the map, so that other processes can't try to + rebuild at the same time. */ + if (mdbox_map_atomic_lock(ctx->atomic, "mdbox storage rebuild") < 0) + return -1; + + /* fsck the map just in case its UIDs are broken */ + if (mail_index_fsck(ctx->storage->map->index) < 0) { + mail_storage_set_index_error(&ctx->storage->storage.storage, + ctx->storage->map->index); + return -1; + } + + /* get old map header */ + mail_index_get_header_ext(ctx->atomic->sync_view, + ctx->storage->map->map_ext_id, + &data, &data_size); + i_zero(&ctx->orig_map_hdr); + memcpy(&ctx->orig_map_hdr, data, + I_MIN(data_size, sizeof(ctx->orig_map_hdr))); + ctx->highest_file_id = ctx->orig_map_hdr.highest_file_id; + + /* get storage rebuild counter after locking */ + ctx->rebuild_count = mdbox_map_get_rebuild_count(ctx->storage->map); + if (ctx->rebuild_count != ctx->storage->corrupted_rebuild_count && + ctx->storage->corrupted) { + /* storage was already rebuilt by someone else */ + return 0; + } + return 1; +} + +static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx) +{ + i_warning("mdbox %s: rebuilding indexes", ctx->storage->storage_dir); + + if (mdbox_storage_rebuild_scan_dir(ctx, ctx->storage->storage_dir, + FALSE) < 0) + return -1; + if (ctx->storage->alt_storage_dir != NULL) { + if (mdbox_storage_rebuild_scan_dir(ctx, + ctx->storage->alt_storage_dir, TRUE) < 0) + return -1; + } + + rebuild_apply_map(ctx); + if (rebuild_mailboxes(ctx) < 0 || + rebuild_finish(ctx) < 0) { + mdbox_map_atomic_set_failed(ctx->atomic); + return -1; + } + return 0; +} + +int mdbox_storage_rebuild_in_context(struct mdbox_storage *storage, + struct mdbox_map_atomic_context *atomic) +{ + struct mdbox_storage_rebuild_context *ctx; + int ret; + + if (dbox_verify_alt_storage(storage->map->root_list) < 0) { + mail_storage_set_critical(&storage->storage.storage, + "mdbox rebuild: Alt storage %s not mounted, aborting", + storage->alt_storage_dir); + mdbox_map_atomic_set_failed(atomic); + return -1; + } + + ctx = mdbox_storage_rebuild_init(storage, atomic); + if ((ret = mdbox_storage_rebuild_scan_prepare(ctx)) > 0) { + struct event_reason *reason = event_reason_begin("mdbox:rebuild"); + ret = mdbox_storage_rebuild_scan(ctx); + event_reason_end(&reason); + } + mdbox_storage_rebuild_deinit(ctx); + + if (ret == 0) { + storage->corrupted = FALSE; + storage->corrupted_rebuild_count = 0; + } + return ret; +} + +int mdbox_storage_rebuild(struct mdbox_storage *storage) +{ + struct mdbox_map_atomic_context *atomic; + int ret; + + atomic = mdbox_map_atomic_begin(storage->map); + ret = mdbox_storage_rebuild_in_context(storage, atomic); + mdbox_map_atomic_set_success(atomic); + mdbox_map_atomic_unset_fscked(atomic); + if (mdbox_map_atomic_finish(&atomic) < 0) + ret = -1; + return ret; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h new file mode 100644 index 0000000..1194d29 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h @@ -0,0 +1,10 @@ +#ifndef MDBOX_STORAGE_REBUILD_H +#define MDBOX_STORAGE_REBUILD_H + +struct mdbox_map_atomic_context; + +int mdbox_storage_rebuild_in_context(struct mdbox_storage *storage, + struct mdbox_map_atomic_context *atomic); +int mdbox_storage_rebuild(struct mdbox_storage *storage); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.c b/src/lib-storage/index/dbox-multi/mdbox-storage.c new file mode 100644 index 0000000..d1f2d15 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c @@ -0,0 +1,529 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "master-service.h" +#include "mail-index-modseq.h" +#include "mail-index-alloc-cache.h" +#include "mailbox-log.h" +#include "mailbox-list-private.h" +#include "index-pop3-uidl.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-storage.h" + +extern struct mail_storage mdbox_storage; +extern struct mailbox mdbox_mailbox; + +static struct event_category event_category_mdbox = { + .name = "mdbox", + .parent = &event_category_storage, +}; + +static struct mail_storage *mdbox_storage_alloc(void) +{ + struct mdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mdbox storage", 2048); + storage = p_new(pool, struct mdbox_storage, 1); + storage->storage.v = mdbox_dbox_storage_vfuncs; + storage->storage.storage = mdbox_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +int mdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, const char **error_r) +{ + struct mdbox_storage *storage = MDBOX_STORAGE(_storage); + const char *dir; + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + storage->preallocate_space = storage->set->mdbox_preallocate_space; + + if (*ns->list->set.mailbox_dir_name == '\0') { + *error_r = "mdbox: MAILBOXDIR must not be empty"; + return -1; + } + + _storage->unique_root_dir = + p_strdup(_storage->pool, ns->list->set.root_dir); + + dir = mailbox_list_get_root_forced(ns->list, MAILBOX_LIST_PATH_TYPE_DIR); + storage->storage_dir = p_strconcat(_storage->pool, dir, + "/"MDBOX_GLOBAL_DIR_NAME, NULL); + if (ns->list->set.alt_dir != NULL) { + storage->alt_storage_dir = p_strconcat(_storage->pool, + ns->list->set.alt_dir, + "/"MDBOX_GLOBAL_DIR_NAME, NULL); + } + i_array_init(&storage->open_files, 64); + + storage->map = mdbox_map_init(storage, ns->list); + return dbox_storage_create(_storage, ns, error_r); +} + +void mdbox_storage_destroy(struct mail_storage *_storage) +{ + struct mdbox_storage *storage = MDBOX_STORAGE(_storage); + + mdbox_files_free(storage); + mdbox_map_deinit(&storage->map); + timeout_remove(&storage->to_close_unused_files); + + if (array_is_created(&storage->move_from_alt_map_uids)) + array_free(&storage->move_from_alt_map_uids); + if (array_is_created(&storage->move_to_alt_map_uids)) + array_free(&storage->move_to_alt_map_uids); + array_free(&storage->open_files); + dbox_storage_destroy(_storage); +} + +static const char * +mdbox_storage_find_root_dir(const struct mail_namespace *ns) +{ + bool debug = ns->mail_set->mail_debug; + const char *home, *path; + + if (ns->owner != NULL && + mail_user_get_home(ns->owner, &home) > 0) { + path = t_strconcat(home, "/mdbox", NULL); + if (access(path, R_OK|W_OK|X_OK) == 0) { + if (debug) + i_debug("mdbox: root exists (%s)", path); + return path; + } + if (debug) + i_debug("mdbox: access(%s, rwx): failed: %m", path); + } + return NULL; +} + +static bool mdbox_storage_autodetect(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + bool debug = ns->mail_set->mail_debug; + struct stat st; + const char *path, *root_dir; + + if (set->root_dir != NULL) + root_dir = set->root_dir; + else { + root_dir = mdbox_storage_find_root_dir(ns); + if (root_dir == NULL) { + if (debug) + i_debug("mdbox: couldn't find root dir"); + return FALSE; + } + } + + path = t_strconcat(root_dir, "/"MDBOX_GLOBAL_DIR_NAME, NULL); + if (stat(path, &st) < 0) { + if (debug) + i_debug("mdbox autodetect: stat(%s) failed: %m", path); + return FALSE; + } + + if (!S_ISDIR(st.st_mode)) { + if (debug) + i_debug("mdbox autodetect: %s not a directory", path); + return FALSE; + } + + set->root_dir = root_dir; + dbox_storage_get_list_settings(ns, set); + return TRUE; +} + +static struct mailbox * +mdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mdbox_mailbox *mbox; + struct index_mailbox_context *ibox; + pool_t pool; + + /* dbox can't work without index files */ + flags &= ENUM_NEGATE(MAILBOX_FLAG_NO_INDEX_FILES); + + pool = pool_alloconly_create("mdbox mailbox", 1024*3); + mbox = p_new(pool, struct mdbox_mailbox, 1); + mbox->box = mdbox_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mdbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + ibox = INDEX_STORAGE_CONTEXT(&mbox->box); + ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY; + + mbox->storage = MDBOX_STORAGE(storage); + return &mbox->box; +} + +int mdbox_mailbox_open(struct mailbox *box) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + + if (dbox_mailbox_check_existence(box) < 0) + return -1; + if (dbox_mailbox_open(box) < 0) + return -1; + + mbox->ext_id = + mail_index_ext_register(mbox->box.index, "mdbox", 0, + sizeof(struct mdbox_mail_index_record), + sizeof(uint32_t)); + mbox->hdr_ext_id = + mail_index_ext_register(mbox->box.index, "mdbox-hdr", + sizeof(struct mdbox_index_header), 0, 0); + mbox->guid_ext_id = + mail_index_ext_register(mbox->box.index, "guid", + 0, GUID_128_SIZE, 1); + return 0; +} + +static void mdbox_mailbox_close(struct mailbox *box) +{ + struct mdbox_storage *mstorage = MDBOX_STORAGE(box->storage); + + if (mstorage->corrupted && !mstorage->rebuilding_storage) + (void)mdbox_storage_rebuild(mstorage); + + dbox_mailbox_close(box); +} + +int mdbox_read_header(struct mdbox_mailbox *mbox, + struct mdbox_index_header *hdr, bool *need_resize_r) +{ + const void *data; + size_t data_size; + + i_assert(mbox->box.opened); + + mail_index_get_header_ext(mbox->box.view, mbox->hdr_ext_id, + &data, &data_size); + if (data_size < MDBOX_INDEX_HEADER_MIN_SIZE && + (!mbox->creating || data_size != 0)) { + mailbox_set_critical(&mbox->box, + "mdbox: Invalid dbox header size: %zu", + data_size); + mdbox_storage_set_corrupted(mbox->storage); + return -1; + } + i_zero(hdr); + if (data_size > 0) + memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr))); + *need_resize_r = data_size < sizeof(*hdr); + return 0; +} + +void mdbox_update_header(struct mdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update) +{ + struct mdbox_index_header hdr, new_hdr; + bool need_resize; + + if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) { + i_zero(&hdr); + need_resize = TRUE; + } + + new_hdr = hdr; + + if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) { + memcpy(new_hdr.mailbox_guid, update->mailbox_guid, + sizeof(new_hdr.mailbox_guid)); + } else if (guid_128_is_empty(new_hdr.mailbox_guid)) { + guid_128_generate(new_hdr.mailbox_guid); + } + + new_hdr.map_uid_validity = + mdbox_map_get_uid_validity(mbox->storage->map); + if (need_resize) { + mail_index_ext_resize_hdr(trans, mbox->hdr_ext_id, + sizeof(new_hdr)); + } + if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) { + mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0, + &new_hdr, sizeof(new_hdr)); + } +} + +static int ATTR_NULL(2, 3) +mdbox_write_index_header(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + struct mail_index_transaction *new_trans = NULL; + struct mail_index_view *view; + const struct mail_index_header *hdr; + uint32_t uid_validity, uid_next; + + if (mdbox_map_open_or_create(mbox->storage->map) < 0) + return -1; + + if (trans == NULL) { + new_trans = mail_index_transaction_begin(box->view, 0); + trans = new_trans; + } + + view = mail_index_view_open(box->index); + hdr = mail_index_get_header(view); + uid_validity = hdr->uid_validity; + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + else if (uid_validity == 0) { + /* set uidvalidity */ + uid_validity = dbox_get_uidvalidity_next(box->list); + } + + if (hdr->uid_validity != uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + if (update != NULL && hdr->next_uid < update->min_next_uid) { + uid_next = update->min_next_uid; + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + } + if (update != NULL && update->min_first_recent_uid != 0 && + hdr->first_recent_uid < update->min_first_recent_uid) { + uint32_t first_recent_uid = update->min_first_recent_uid; + + mail_index_update_header(trans, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + if (update != NULL && update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(view) < update->min_highest_modseq) { + mail_index_modseq_enable(box->index); + mail_index_update_highest_modseq(trans, + update->min_highest_modseq); + } + mail_index_view_close(&view); + + if (box->inbox_user && box->creating) { + /* initialize pop3-uidl header when creating mailbox + (not on mailbox_update()) */ + index_pop3_uidl_set_max_uid(box, trans, 0); + } + + mdbox_update_header(mbox, trans, update); + if (new_trans != NULL) { + if (mail_index_transaction_commit(&new_trans) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + return 0; +} + +int mdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + int ret; + + mbox->creating = TRUE; + ret = mdbox_write_index_header(box, update, trans); + mbox->creating = FALSE; + return ret; +} + +void mdbox_storage_set_corrupted(struct mdbox_storage *storage) +{ + if (storage->corrupted) { + /* already set it corrupted (possibly recursing back here) */ + return; + } + + storage->corrupted = TRUE; + storage->corrupted_rebuild_count = (uint32_t)-1; + + if (mdbox_map_open(storage->map) > 0 && + mdbox_map_refresh(storage->map) == 0) { + storage->corrupted_rebuild_count = + mdbox_map_get_rebuild_count(storage->map); + } +} + +static const char * +mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) +{ + return ""; +} + +void mdbox_set_mailbox_corrupted(struct mailbox *box) +{ + struct mdbox_storage *mstorage = MDBOX_STORAGE(box->storage); + + mdbox_storage_set_corrupted(mstorage); +} + +void mdbox_set_file_corrupted(struct dbox_file *file) +{ + struct mdbox_storage *mstorage = MDBOX_DBOX_STORAGE(file->storage); + + mdbox_storage_set_corrupted(mstorage); +} + +static int +mdbox_mailbox_get_guid(struct mdbox_mailbox *mbox, guid_128_t guid_r) +{ + const struct mail_index_header *idx_hdr; + struct mdbox_index_header hdr; + bool need_resize; + int ret = 0; + + i_assert(!mbox->creating); + + /* there's a race condition between mkdir and getting the mailbox GUID. + normally this is handled by mdbox syncing, but GUID can be looked up + without syncing. when we detect this situation we'll try to finish + creating the indexes first, which usually means just waiting for + the sync lock to get unlocked by the other process creating them. */ + idx_hdr = mail_index_get_header(mbox->box.view); + if (idx_hdr->uid_validity == 0 && idx_hdr->next_uid == 1) { + if (dbox_mailbox_create_indexes(&mbox->box, NULL) < 0) + return -1; + } + + if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) + i_zero(&hdr); + + if (guid_128_is_empty(hdr.mailbox_guid)) { + /* regenerate it */ + if (mdbox_write_index_header(&mbox->box, NULL, NULL) < 0 || + mdbox_read_header(mbox, &hdr, &need_resize) < 0) + ret = -1; + } + if (ret == 0) + memcpy(guid_r, hdr.mailbox_guid, GUID_128_SIZE); + return ret; +} + +static int +mdbox_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + if ((items & MAILBOX_METADATA_GUID) != 0) { + if (mdbox_mailbox_get_guid(mbox, metadata_r->guid) < 0) + return -1; + } + return 0; +} + +static int +mdbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + if (mdbox_write_index_header(box, update, NULL) < 0) + return -1; + return index_storage_mailbox_update_common(box, update); +} + +struct mail_storage mdbox_storage = { + .name = MDBOX_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + .event_category = &event_category_mdbox, + + .v = { + mdbox_get_setting_parser_info, + mdbox_storage_alloc, + mdbox_storage_create, + mdbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + mdbox_storage_autodetect, + mdbox_mailbox_alloc, + mdbox_purge, + mail_storage_list_index_rebuild, + } +}; + +struct mailbox mdbox_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mdbox_mailbox_open, + mdbox_mailbox_close, + index_storage_mailbox_free, + dbox_mailbox_create, + mdbox_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mdbox_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + mdbox_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + dbox_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + mdbox_save_alloc, + mdbox_save_begin, + dbox_save_continue, + mdbox_save_finish, + mdbox_save_cancel, + mdbox_copy, + mdbox_transaction_save_commit_pre, + mdbox_transaction_save_commit_post, + mdbox_transaction_save_rollback, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs = { + mdbox_file_unrefed, + mdbox_file_create_fd, + mdbox_mail_open, + mdbox_mailbox_create_indexes, + mdbox_get_attachment_path_suffix, + mdbox_set_mailbox_corrupted, + mdbox_set_file_corrupted +}; diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.h b/src/lib-storage/index/dbox-multi/mdbox-storage.h new file mode 100644 index 0000000..ea99532 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h @@ -0,0 +1,118 @@ +#ifndef MDBOX_STORAGE_H +#define MDBOX_STORAGE_H + +#include "index-storage.h" +#include "dbox-storage.h" +#include "mdbox-settings.h" + +#define MDBOX_STORAGE_NAME "mdbox" +#define MDBOX_DELETED_STORAGE_NAME "mdbox_deleted" +#define MDBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index" +#define MDBOX_GLOBAL_DIR_NAME "storage" +#define MDBOX_MAIL_FILE_PREFIX "m." +#define MDBOX_MAIL_FILE_FORMAT MDBOX_MAIL_FILE_PREFIX"%u" +#define MDBOX_MAX_OPEN_UNUSED_FILES 2 +#define MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS 30 + +#define MDBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t)) +struct mdbox_index_header { + uint32_t map_uid_validity; + guid_128_t mailbox_guid; + uint8_t flags; /* enum dbox_index_header_flags */ + uint8_t unused[3]; +}; + +struct mdbox_storage { + struct dbox_storage storage; + const struct mdbox_settings *set; + + /* paths for storage directories */ + const char *storage_dir, *alt_storage_dir; + struct mdbox_map *map; + + ARRAY(struct mdbox_file *) open_files; + struct timeout *to_close_unused_files; + + ARRAY_TYPE(uint32_t) move_to_alt_map_uids; + ARRAY_TYPE(uint32_t) move_from_alt_map_uids; + + /* if non-zero, storage should be rebuilt (except if rebuild_count + has changed from this value) */ + uint32_t corrupted_rebuild_count; + + bool corrupted:1; + bool rebuilding_storage:1; + bool preallocate_space:1; +}; + +struct mdbox_mail_index_record { + uint32_t map_uid; + /* UNIX timestamp of when the message was saved/copied to this + mailbox */ + uint32_t save_date; +}; + +struct mdbox_mailbox { + struct mailbox box; + struct mdbox_storage *storage; + + uint32_t map_uid_validity; + uint32_t ext_id, hdr_ext_id, guid_ext_id; + + bool mdbox_deleted_synced:1; + bool creating:1; +}; + +#define MDBOX_DBOX_STORAGE(s) container_of(s, struct mdbox_storage, storage) +#define MDBOX_STORAGE(s) MDBOX_DBOX_STORAGE(DBOX_STORAGE(s)) +#define MDBOX_MAILBOX(s) container_of(s, struct mdbox_mailbox, box) + +extern struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs; +extern struct mail_vfuncs mdbox_mail_vfuncs; + +int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r); + +/* Get map_uid for wanted message. */ +int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, uint32_t *map_uid_r); +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); +int mdbox_read_header(struct mdbox_mailbox *mbox, + struct mdbox_index_header *hdr, bool *need_resize_r); +void mdbox_update_header(struct mdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update) ATTR_NULL(3); +int mdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans); + +struct mail_save_context * +mdbox_save_alloc(struct mailbox_transaction_context *_t); +int mdbox_save_begin(struct mail_save_context *ctx, struct istream *input); +int mdbox_save_finish(struct mail_save_context *ctx); +void mdbox_save_cancel(struct mail_save_context *ctx); + +struct dbox_file * +mdbox_save_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r); + +int mdbox_transaction_save_commit_pre(struct mail_save_context *ctx); +void mdbox_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void mdbox_transaction_save_rollback(struct mail_save_context *ctx); + +int mdbox_copy(struct mail_save_context *ctx, struct mail *mail); + +void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt); +int mdbox_purge(struct mail_storage *storage); + +int mdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, const char **error_r); +void mdbox_storage_destroy(struct mail_storage *_storage); +int mdbox_mailbox_open(struct mailbox *box); + +void mdbox_storage_set_corrupted(struct mdbox_storage *storage); +void mdbox_set_mailbox_corrupted(struct mailbox *box); +void mdbox_set_file_corrupted(struct dbox_file *file); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.c b/src/lib-storage/index/dbox-multi/mdbox-sync.c new file mode 100644 index 0000000..0c2d2d4 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c @@ -0,0 +1,377 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +/* + Expunging works like: + + 1. Lock map index by beginning a map sync. + 2. Write map UID refcount changes to map index (=> tail != head). + 3. Expunge messages from mailbox index. + 4. Finish map sync, which updates tail=head and unlocks map index. + + If something crashes after 2 but before 4 is finished, tail != head and + reader can do a full resync to figure out what got broken. +*/ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "str.h" +#include "mdbox-storage.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mailbox-recent-flags.h" + + +/* returns -1 on error, 1 on success, 0 if guid is empty/missing */ +static int +dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq, + const guid_128_t guid_128) +{ + const void *data; + uint32_t uid; + + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mail_index_lookup_ext(ctx->sync_view, seq, + ctx->mbox->guid_ext_id, &data, NULL); + + if ((data == NULL) || guid_128_is_empty(data)) + return 0; + + if (guid_128_is_empty(guid_128) || + memcmp(data, guid_128, GUID_128_SIZE) == 0) + return 1; + + mailbox_set_critical(&ctx->mbox->box, + "Expunged GUID mismatch for UID %u: %s vs %s", + uid, guid_128_to_string(data), guid_128_to_string(guid_128)); + mdbox_storage_set_corrupted(ctx->mbox->storage); + return -1; +} + +static int mdbox_sync_expunge(struct mdbox_sync_context *ctx, uint32_t seq, + const guid_128_t guid_128) +{ + uint32_t map_uid; + int ret; + + if (seq_range_array_add(&ctx->expunged_seqs, seq)) { + /* already marked as expunged in this sync */ + return 0; + } + + ret = dbox_sync_verify_expunge_guid(ctx, seq, guid_128); + if (ret <= 0) + return ret; + if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0) + return -1; + if (mdbox_map_update_refcount(ctx->map_trans, map_uid, -1) < 0) + return -1; + return 0; +} + +static int mdbox_sync_rec(struct mdbox_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec) +{ + uint32_t seq, seq1, seq2; + + if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + /* not interested */ + return 0; + } + + if (!mail_index_lookup_seq_range(ctx->sync_view, + sync_rec->uid1, sync_rec->uid2, + &seq1, &seq2)) { + /* already expunged everything. nothing to do. */ + return 0; + } + + for (seq = seq1; seq <= seq2; seq++) { + if (mdbox_sync_expunge(ctx, seq, sync_rec->guid_128) < 0) + return -1; + } + return 0; +} + +static int dbox_sync_mark_expunges(struct mdbox_sync_context *ctx) +{ + enum mail_index_transaction_flags flags = + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + struct mailbox *box = &ctx->mbox->box; + struct mail_index_transaction *trans; + struct seq_range_iter iter; + unsigned int n; + const void *data; + uint32_t seq, uid; + + /* use a separate transaction here so that we can commit the changes + during map transaction */ + trans = mail_index_transaction_begin(ctx->sync_view, flags); + seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &seq)) { + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mail_index_lookup_ext(ctx->sync_view, seq, + ctx->mbox->guid_ext_id, &data, NULL); + if ((data == NULL) || guid_128_is_empty(data)) + mail_index_expunge(trans, seq); + else + mail_index_expunge_guid(trans, seq, data); + } + if (mail_index_transaction_commit(&trans) < 0) + return -1; + + box->tmp_sync_view = ctx->sync_view; + seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &seq)) { + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mailbox_sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE); + } + box->tmp_sync_view = NULL; + return 0; +} + +static int mdbox_sync_index(struct mdbox_sync_context *ctx) +{ + struct mailbox *box = &ctx->mbox->box; + const struct mail_index_header *hdr; + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + int ret = 0; + + hdr = mail_index_get_header(ctx->sync_view); + if (hdr->uid_validity == 0) { + /* newly created index file */ + if (hdr->next_uid == 1) { + /* could be just a race condition where we opened the + mailbox between mkdir and index creation. fix this + silently. */ + if (mdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0) + return -1; + return 1; + } + mailbox_set_critical(box, "Broken index: missing UIDVALIDITY"); + return 0; + } + + /* mark the newly seen messages as recent */ + if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, + hdr->next_uid, &seq1, &seq2)) { + mailbox_recent_flags_set_seqs(&ctx->mbox->box, ctx->sync_view, + seq1, seq2); + } + + /* handle syncing records without map being locked. */ + if (mdbox_map_atomic_is_locked(ctx->atomic)) { + ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE); + i_array_init(&ctx->expunged_seqs, 64); + } + while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) { + if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0) + break; + } + + /* write refcount changes to map index. transaction commit updates the + log head, while tail is left behind. */ + if (mdbox_map_atomic_is_locked(ctx->atomic)) { + if (ret == 0) + ret = mdbox_map_transaction_commit(ctx->map_trans, "mdbox syncing"); + /* write changes to mailbox index */ + if (ret == 0) + ret = dbox_sync_mark_expunges(ctx); + + /* finish the map changes and unlock the map. this also updates + map's tail -> head. */ + if (ret < 0) + mdbox_map_atomic_set_failed(ctx->atomic); + mdbox_map_transaction_free(&ctx->map_trans); + ctx->expunged_count = seq_range_count(&ctx->expunged_seqs); + array_free(&ctx->expunged_seqs); + } + + mailbox_sync_notify(box, 0, 0); + + return ret == 0 ? 1 : + (ctx->mbox->storage->corrupted ? 0 : -1); +} + +static int mdbox_sync_try_begin(struct mdbox_sync_context *ctx, + enum mail_index_sync_flags sync_flags) +{ + struct mdbox_mailbox *mbox = ctx->mbox; + int ret; + + ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, sync_flags); + if (mail_index_reset_fscked(mbox->box.index)) + mdbox_storage_set_corrupted(mbox->storage); + if (ret <= 0) + return ret; /* error / nothing to do */ + + if (!mdbox_map_atomic_is_locked(ctx->atomic) && + mail_index_sync_has_expunges(ctx->index_sync_ctx)) { + /* we have expunges, so we need to write to map. + it needs to be locked before mailbox index. */ + mail_index_sync_set_reason(ctx->index_sync_ctx, "mdbox expunge check"); + mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&ctx->mbox->box); + + if (mdbox_map_atomic_lock(ctx->atomic, "mdbox syncing with expunges") < 0) + return -1; + return mdbox_sync_try_begin(ctx, sync_flags); + } + return 1; +} + +int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, + struct mdbox_map_atomic_context *atomic, + struct mdbox_sync_context **ctx_r) +{ + const struct mail_index_header *hdr = + mail_index_get_header(mbox->box.view); + struct mdbox_sync_context *ctx; + const char *reason; + enum mail_index_sync_flags sync_flags; + int ret; + bool rebuild, storage_rebuilt = FALSE; + + *ctx_r = NULL; + + /* avoid race conditions with mailbox creation, don't check for dbox + headers until syncing has locked the mailbox */ + rebuild = mbox->storage->corrupted || + (hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0 || + mdbox_map_is_fscked(mbox->storage->map) || + (flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0; + if (rebuild && (flags & MDBOX_SYNC_FLAG_NO_REBUILD) == 0) { + if (mdbox_storage_rebuild_in_context(mbox->storage, atomic) < 0) + return -1; + mailbox_recent_flags_reset(&mbox->box); + storage_rebuilt = TRUE; + } + + ctx = i_new(struct mdbox_sync_context, 1); + ctx->mbox = mbox; + ctx->flags = flags; + ctx->atomic = atomic; + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if (!rebuild && (flags & MDBOX_SYNC_FLAG_FORCE) == 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + if ((flags & MDBOX_SYNC_FLAG_FSYNC) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC; + /* don't write unnecessary dirty flag updates */ + sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; + + ret = mdbox_sync_try_begin(ctx, sync_flags); + if (ret <= 0) { + /* failed / nothing to do */ + index_storage_expunging_deinit(&mbox->box); + i_free(ctx); + return ret; + } + + if ((ret = mdbox_sync_index(ctx)) <= 0) { + mail_index_sync_set_reason(ctx->index_sync_ctx, + ret < 0 ? "mdbox syncing failed" : + "mdbox syncing found corruption"); + mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&mbox->box); + i_free_and_null(ctx); + + if (ret < 0) + return -1; + + /* corrupted */ + if (storage_rebuilt) { + mailbox_set_critical(&mbox->box, + "mdbox: Storage keeps breaking"); + return -1; + } + + /* we'll need to rebuild storage. + try again from the beginning. */ + mdbox_storage_set_corrupted(mbox->storage); + if ((flags & MDBOX_SYNC_FLAG_NO_REBUILD) != 0) { + mailbox_set_critical(&mbox->box, + "mdbox: Can't rebuild storage"); + return -1; + } + return mdbox_sync_begin(mbox, flags, atomic, ctx_r); + } + index_storage_expunging_deinit(&mbox->box); + + if (!mdbox_map_atomic_is_locked(ctx->atomic)) + reason = "mdbox synced"; + else { + /* may be 0 msgs, but that still informs that the map + was locked */ + reason = t_strdup_printf("mdbox synced - %u msgs expunged", + ctx->expunged_count); + } + mail_index_sync_set_reason(ctx->index_sync_ctx, reason); + + *ctx_r = ctx; + return 0; +} + +int mdbox_sync_finish(struct mdbox_sync_context **_ctx, bool success) +{ + struct mdbox_sync_context *ctx = *_ctx; + struct mail_storage *storage = &ctx->mbox->storage->storage.storage; + int ret = success ? 0 : -1; + + *_ctx = NULL; + + if (success) { + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { + mailbox_set_index_error(&ctx->mbox->box); + ret = -1; + } + } else { + mail_index_sync_rollback(&ctx->index_sync_ctx); + } + + if (storage->rebuild_list_index) + ret = mail_storage_list_index_rebuild_and_set_uncorrupted(storage); + + i_free(ctx); + return ret; +} + +int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags) +{ + struct mdbox_sync_context *sync_ctx; + struct mdbox_map_atomic_context *atomic; + int ret; + + atomic = mdbox_map_atomic_begin(mbox->storage->map); + ret = mdbox_sync_begin(mbox, flags, atomic, &sync_ctx); + if (ret == 0 && sync_ctx != NULL) + ret = mdbox_sync_finish(&sync_ctx, TRUE); + if (ret == 0) + mdbox_map_atomic_set_success(atomic); + if (mdbox_map_atomic_finish(&atomic) < 0) + ret = -1; + return ret; +} + +struct mailbox_sync_context * +mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + enum mdbox_sync_flags mdbox_sync_flags = 0; + int ret = 0; + + if (mail_index_reset_fscked(box->index)) + mdbox_storage_set_corrupted(mbox->storage); + if (index_mailbox_want_full_sync(&mbox->box, flags) || + mbox->storage->corrupted) { + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) + mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD; + ret = mdbox_sync(mbox, mdbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.h b/src/lib-storage/index/dbox-multi/mdbox-sync.h new file mode 100644 index 0000000..a9bc565 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.h @@ -0,0 +1,37 @@ +#ifndef MDBOX_SYNC_H +#define MDBOX_SYNC_H + +struct mailbox; +struct mdbox_mailbox; + +enum mdbox_sync_flags { + MDBOX_SYNC_FLAG_FORCE = 0x01, + MDBOX_SYNC_FLAG_FSYNC = 0x02, + MDBOX_SYNC_FLAG_FORCE_REBUILD = 0x04, + MDBOX_SYNC_FLAG_NO_PURGE = 0x08, + MDBOX_SYNC_FLAG_NO_REBUILD = 0x10 +}; + +struct mdbox_sync_context { + struct mdbox_mailbox *mbox; + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mdbox_map_transaction_context *map_trans; + struct mdbox_map_atomic_context *atomic; + enum mdbox_sync_flags flags; + + ARRAY_TYPE(seq_range) expunged_seqs; + unsigned int expunged_count; +}; + +int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, + struct mdbox_map_atomic_context *atomic, + struct mdbox_sync_context **ctx_r); +int mdbox_sync_finish(struct mdbox_sync_context **ctx, bool success); +int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags); + +struct mailbox_sync_context * +mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +#endif diff --git a/src/lib-storage/index/dbox-single/Makefile.am b/src/lib-storage/index/dbox-single/Makefile.am new file mode 100644 index 0000000..b6f59bd --- /dev/null +++ b/src/lib-storage/index/dbox-single/Makefile.am @@ -0,0 +1,30 @@ +noinst_LTLIBRARIES = libstorage_dbox_single.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_single_la_SOURCES = \ + sdbox-copy.c \ + sdbox-file.c \ + sdbox-mail.c \ + sdbox-save.c \ + sdbox-sync.c \ + sdbox-sync-rebuild.c \ + sdbox-storage.c + +headers = \ + sdbox-file.h \ + sdbox-storage.h \ + sdbox-sync.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/dbox-single/Makefile.in b/src/lib-storage/index/dbox-single/Makefile.in new file mode 100644 index 0000000..03b6f53 --- /dev/null +++ b/src/lib-storage/index/dbox-single/Makefile.in @@ -0,0 +1,848 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/dbox-single +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_dbox_single_la_LIBADD = +am_libstorage_dbox_single_la_OBJECTS = sdbox-copy.lo sdbox-file.lo \ + sdbox-mail.lo sdbox-save.lo sdbox-sync.lo \ + sdbox-sync-rebuild.lo sdbox-storage.lo +libstorage_dbox_single_la_OBJECTS = \ + $(am_libstorage_dbox_single_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/sdbox-copy.Plo \ + ./$(DEPDIR)/sdbox-file.Plo ./$(DEPDIR)/sdbox-mail.Plo \ + ./$(DEPDIR)/sdbox-save.Plo ./$(DEPDIR)/sdbox-storage.Plo \ + ./$(DEPDIR)/sdbox-sync-rebuild.Plo ./$(DEPDIR)/sdbox-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_dbox_single_la_SOURCES) +DIST_SOURCES = $(libstorage_dbox_single_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_dbox_single.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_single_la_SOURCES = \ + sdbox-copy.c \ + sdbox-file.c \ + sdbox-mail.c \ + sdbox-save.c \ + sdbox-sync.c \ + sdbox-sync-rebuild.c \ + sdbox-storage.c + +headers = \ + sdbox-file.h \ + sdbox-storage.h \ + sdbox-sync.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/dbox-single/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/dbox-single/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_dbox_single.la: $(libstorage_dbox_single_la_OBJECTS) $(libstorage_dbox_single_la_DEPENDENCIES) $(EXTRA_libstorage_dbox_single_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_dbox_single_la_OBJECTS) $(libstorage_dbox_single_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-copy.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-sync-rebuild.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/sdbox-copy.Plo + -rm -f ./$(DEPDIR)/sdbox-file.Plo + -rm -f ./$(DEPDIR)/sdbox-mail.Plo + -rm -f ./$(DEPDIR)/sdbox-save.Plo + -rm -f ./$(DEPDIR)/sdbox-storage.Plo + -rm -f ./$(DEPDIR)/sdbox-sync-rebuild.Plo + -rm -f ./$(DEPDIR)/sdbox-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/sdbox-copy.Plo + -rm -f ./$(DEPDIR)/sdbox-file.Plo + -rm -f ./$(DEPDIR)/sdbox-mail.Plo + -rm -f ./$(DEPDIR)/sdbox-save.Plo + -rm -f ./$(DEPDIR)/sdbox-storage.Plo + -rm -f ./$(DEPDIR)/sdbox-sync-rebuild.Plo + -rm -f ./$(DEPDIR)/sdbox-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/dbox-single/sdbox-copy.c b/src/lib-storage/index/dbox-single/sdbox-copy.c new file mode 100644 index 0000000..48a3f59 --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-copy.c @@ -0,0 +1,185 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "nfs-workarounds.h" +#include "fs-api.h" +#include "dbox-save.h" +#include "dbox-attachment.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "mail-copy.h" + +static int +sdbox_file_copy_attachments(struct sdbox_file *src_file, + struct sdbox_file *dest_file) +{ + struct dbox_storage *src_storage = src_file->file.storage; + struct dbox_storage *dest_storage = dest_file->file.storage; + struct fs_file *src_fsfile, *dest_fsfile; + ARRAY_TYPE(mail_attachment_extref) extrefs; + const struct mail_attachment_extref *extref; + const char *extrefs_line, *src, *dest, *dest_relpath; + pool_t pool; + int ret; + + if (src_storage->attachment_dir == NULL) { + /* no attachments in source storage */ + return 1; + } + if (dest_storage->attachment_dir == NULL || + strcmp(src_storage->attachment_dir, + dest_storage->attachment_dir) != 0 || + strcmp(src_storage->storage.set->mail_attachment_fs, + dest_storage->storage.set->mail_attachment_fs) != 0 || + strcmp(src_storage->storage.set->mail_attachment_hash, + dest_storage->storage.set->mail_attachment_hash) != 0) { + /* different attachment dirs/settings between storages. + have to copy the slow way. */ + return 0; + } + + if ((ret = sdbox_file_get_attachments(&src_file->file, + &extrefs_line)) <= 0) + return ret < 0 ? -1 : 1; + + pool = pool_alloconly_create("sdbox attachments copy", 1024); + p_array_init(&extrefs, pool, 16); + if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) { + mailbox_set_critical(&dest_file->mbox->box, + "Can't copy %s with corrupted extref metadata: %s", + src_file->file.cur_path, extrefs_line); + pool_unref(&pool); + return -1; + } + + dest_file->attachment_pool = + pool_alloconly_create("sdbox attachment copy paths", 512); + p_array_init(&dest_file->attachment_paths, dest_file->attachment_pool, + array_count(&extrefs)); + + ret = 1; + array_foreach(&extrefs, extref) T_BEGIN { + src = t_strdup_printf("%s/%s", dest_storage->attachment_dir, + sdbox_file_attachment_relpath(src_file, extref->path)); + dest_relpath = p_strconcat(dest_file->attachment_pool, + extref->path, "-", + guid_generate(), NULL); + dest = t_strdup_printf("%s/%s", dest_storage->attachment_dir, + dest_relpath); + /* we verified above that attachment_fs is compatible for + src and dest, so it doesn't matter which storage's + attachment_fs we use. in any case we need to use the same + one or fs_copy() will crash with assert. */ + src_fsfile = fs_file_init(dest_storage->attachment_fs, src, + FS_OPEN_MODE_READONLY); + dest_fsfile = fs_file_init(dest_storage->attachment_fs, dest, + FS_OPEN_MODE_READONLY); + if (fs_copy(src_fsfile, dest_fsfile) < 0) { + mailbox_set_critical(&dest_file->mbox->box, "%s", + fs_file_last_error(dest_fsfile)); + ret = -1; + } else { + array_push_back(&dest_file->attachment_paths, + &dest_relpath); + } + fs_file_deinit(&src_fsfile); + fs_file_deinit(&dest_fsfile); + } T_END; + pool_unref(&pool); + return ret; +} + +static int +sdbox_copy_hardlink(struct mail_save_context *_ctx, struct mail *mail) +{ + struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx); + struct sdbox_mailbox *dest_mbox = SDBOX_MAILBOX(_ctx->transaction->box); + struct sdbox_mailbox *src_mbox; + struct dbox_file *src_file, *dest_file; + const char *src_path, *dest_path; + int ret; + + if (strcmp(mail->box->storage->name, SDBOX_STORAGE_NAME) == 0) + src_mbox = SDBOX_MAILBOX(mail->box); + else { + /* Source storage isn't sdbox, can't hard link */ + return 0; + } + + src_file = sdbox_file_init(src_mbox, mail->uid); + dest_file = sdbox_file_init(dest_mbox, 0); + + ctx->ctx.data.flags &= ENUM_NEGATE(DBOX_INDEX_FLAG_ALT); + + src_path = src_file->primary_path; + dest_path = dest_file->primary_path; + ret = nfs_safe_link(src_path, dest_path, FALSE); + if (ret < 0 && errno == ENOENT && src_file->alt_path != NULL) { + src_path = src_file->alt_path; + if (dest_file->alt_path != NULL) { + dest_path = dest_file->cur_path = dest_file->alt_path; + ctx->ctx.data.flags |= DBOX_INDEX_FLAG_ALT; + } + ret = nfs_safe_link(src_path, dest_path, FALSE); + } + if (ret < 0) { + if (ECANTLINK(errno)) + ret = 0; + else if (errno == ENOENT) { + /* try if the fallback copying code can still + read the file (the mail could still have the + stream open) */ + ret = 0; + } else { + mail_set_critical(mail, "link(%s, %s) failed: %m", + src_path, dest_path); + } + dbox_file_unref(&src_file); + dbox_file_unref(&dest_file); + return ret; + } + + ret = sdbox_file_copy_attachments((struct sdbox_file *)src_file, + (struct sdbox_file *)dest_file); + if (ret <= 0) { + (void)sdbox_file_unlink_aborted_save((struct sdbox_file *)dest_file); + dbox_file_unref(&src_file); + dbox_file_unref(&dest_file); + return ret; + } + ((struct sdbox_file *)dest_file)->written_to_disk = TRUE; + + dbox_save_add_to_index(ctx); + index_copy_cache_fields(_ctx, mail, ctx->seq); + + sdbox_save_add_file(_ctx, dest_file); + mail_set_seq_saving(_ctx->dest_mail, ctx->seq); + dbox_file_unref(&src_file); + return 1; +} + +int sdbox_copy(struct mail_save_context *_ctx, struct mail *mail) +{ + struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; + struct mailbox_transaction_context *_t = _ctx->transaction; + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)_t->box; + int ret; + + i_assert((_t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + ctx->finished = TRUE; + if (mail_storage_copy_can_use_hardlink(mail->box, &mbox->box) && + _ctx->data.guid == NULL) { + T_BEGIN { + ret = sdbox_copy_hardlink(_ctx, mail); + } T_END; + + if (ret != 0) { + index_save_context_free(_ctx); + return ret > 0 ? 0 : -1; + } + + /* non-fatal hardlinking failure, try the slow way */ + } + return mail_storage_copy(_ctx, mail); +} diff --git a/src/lib-storage/index/dbox-single/sdbox-file.c b/src/lib-storage/index/dbox-single/sdbox-file.c new file mode 100644 index 0000000..fed111a --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-file.c @@ -0,0 +1,447 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "eacces-error.h" +#include "fdatasync-path.h" +#include "mkdir-parents.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "fs-api.h" +#include "dbox-attachment.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" + +#include <stdio.h> +#include <utime.h> + +static void sdbox_file_init_paths(struct sdbox_file *file, const char *fname) +{ + struct mailbox *box = &file->mbox->box; + const char *alt_path; + + i_free(file->file.primary_path); + i_free(file->file.alt_path); + file->file.primary_path = + i_strdup_printf("%s/%s", mailbox_get_path(box), fname); + file->file.cur_path = file->file.primary_path; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX, + &alt_path) > 0) + file->file.alt_path = i_strdup_printf("%s/%s", alt_path, fname); +} + +struct dbox_file *sdbox_file_init(struct sdbox_mailbox *mbox, uint32_t uid) +{ + struct sdbox_file *file; + const char *fname; + + file = i_new(struct sdbox_file, 1); + file->file.storage = &mbox->storage->storage; + file->mbox = mbox; + T_BEGIN { + if (uid != 0) { + fname = t_strdup_printf(SDBOX_MAIL_FILE_FORMAT, uid); + sdbox_file_init_paths(file, fname); + file->uid = uid; + } else { + sdbox_file_init_paths(file, dbox_generate_tmp_filename()); + } + } T_END; + dbox_file_init(&file->file); + return &file->file; +} + +struct dbox_file *sdbox_file_create(struct sdbox_mailbox *mbox) +{ + struct dbox_file *file; + + file = sdbox_file_init(mbox, 0); + file->fd = file->storage->v. + file_create_fd(file, file->primary_path, FALSE); + return file; +} + +void sdbox_file_free(struct dbox_file *file) +{ + struct sdbox_file *sfile = (struct sdbox_file *)file; + + pool_unref(&sfile->attachment_pool); + dbox_file_free(file); +} + +int sdbox_file_get_attachments(struct dbox_file *file, const char **extrefs_r) +{ + const char *line; + bool deleted; + int ret; + + *extrefs_r = NULL; + + /* read the metadata */ + ret = dbox_file_open(file, &deleted); + if (ret > 0) { + if (deleted) + return 0; + if ((ret = dbox_file_seek(file, 0)) > 0) + ret = dbox_file_metadata_read(file); + } + if (ret <= 0) { + if (ret < 0) + return -1; + /* corrupted file. we're deleting it anyway. */ + line = NULL; + } else { + line = dbox_file_metadata_get(file, DBOX_METADATA_EXT_REF); + } + if (line == NULL) { + /* no attachments */ + return 0; + } + *extrefs_r = line; + return 1; +} + +const char * +sdbox_file_attachment_relpath(struct sdbox_file *file, const char *srcpath) +{ + const char *p; + + p = strchr(srcpath, '-'); + if (p == NULL) { + mailbox_set_critical(&file->mbox->box, + "sdbox attachment path in invalid format: %s", srcpath); + } else { + p = strchr(p+1, '-'); + } + return t_strdup_printf("%s-%s-%u", + p == NULL ? srcpath : t_strdup_until(srcpath, p), + guid_128_to_string(file->mbox->mailbox_guid), + file->uid); +} + +static int sdbox_file_rename_attachments(struct sdbox_file *file) +{ + struct dbox_storage *storage = file->file.storage; + struct fs_file *src_file, *dest_file; + const char *path, *src, *dest; + int ret = 0; + + array_foreach_elem(&file->attachment_paths, path) T_BEGIN { + src = t_strdup_printf("%s/%s", storage->attachment_dir, path); + dest = t_strdup_printf("%s/%s", storage->attachment_dir, + sdbox_file_attachment_relpath(file, path)); + src_file = fs_file_init(storage->attachment_fs, src, + FS_OPEN_MODE_READONLY); + dest_file = fs_file_init(storage->attachment_fs, dest, + FS_OPEN_MODE_READONLY); + if (fs_rename(src_file, dest_file) < 0) { + mailbox_set_critical(&file->mbox->box, "%s", + fs_file_last_error(dest_file)); + ret = -1; + } + fs_file_deinit(&src_file); + fs_file_deinit(&dest_file); + } T_END; + return ret; +} + +int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid) +{ + const char *p, *old_path, *dir, *new_fname, *new_path; + struct stat st; + + i_assert(file->uid == 0); + i_assert(uid != 0); + + old_path = file->file.cur_path; + p = strrchr(old_path, '/'); + i_assert(p != NULL); + dir = t_strdup_until(old_path, p); + + new_fname = t_strdup_printf(SDBOX_MAIL_FILE_FORMAT, uid); + new_path = t_strdup_printf("%s/%s", dir, new_fname); + + if (stat(new_path, &st) == 0) { + mailbox_set_critical(&file->mbox->box, + "sdbox: %s already exists, rebuilding index", new_path); + sdbox_set_mailbox_corrupted(&file->mbox->box); + return -1; + } + if (rename(old_path, new_path) < 0) { + mailbox_set_critical(&file->mbox->box, + "rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + sdbox_file_init_paths(file, new_fname); + file->uid = uid; + + if (array_is_created(&file->attachment_paths)) { + if (sdbox_file_rename_attachments(file) < 0) + return -1; + } + return 0; +} + +static int sdbox_file_unlink_aborted_save_attachments(struct sdbox_file *file) +{ + struct dbox_storage *storage = file->file.storage; + struct fs *fs = storage->attachment_fs; + struct fs_file *fs_file; + const char *path, *att_path; + int ret = 0; + + array_foreach_elem(&file->attachment_paths, att_path) T_BEGIN { + /* we don't know if we aborted before renaming this attachment, + so try deleting both source and dest path. the source paths + point to temporary files (not to source messages' + attachment paths), so it's safe to delete them. */ + path = t_strdup_printf("%s/%s", storage->attachment_dir, + att_path); + fs_file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + if (fs_delete(fs_file) < 0 && + errno != ENOENT) { + mailbox_set_critical(&file->mbox->box, "%s", + fs_file_last_error(fs_file)); + ret = -1; + } + fs_file_deinit(&fs_file); + + path = t_strdup_printf("%s/%s", storage->attachment_dir, + sdbox_file_attachment_relpath(file, att_path)); + fs_file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + if (fs_delete(fs_file) < 0 && + errno != ENOENT) { + mailbox_set_critical(&file->mbox->box, "%s", + fs_file_last_error(fs_file)); + ret = -1; + } + fs_file_deinit(&fs_file); + } T_END; + return ret; +} + +int sdbox_file_unlink_aborted_save(struct sdbox_file *file) +{ + int ret = 0; + + if (unlink(file->file.cur_path) < 0) { + mailbox_set_critical(&file->mbox->box, + "unlink(%s) failed: %m", file->file.cur_path); + ret = -1; + } + if (array_is_created(&file->attachment_paths)) { + if (sdbox_file_unlink_aborted_save_attachments(file) < 0) + ret = -1; + } + return ret; +} + +int sdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents) +{ + struct sdbox_file *sfile = (struct sdbox_file *)file; + struct mailbox *box = &sfile->mbox->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *p, *dir; + mode_t old_mask; + int fd; + + old_mask = umask(0666 & ~perm->file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + if (fd == -1 && errno == ENOENT && parents && + (p = strrchr(path, '/')) != NULL) { + dir = t_strdup_until(path, p); + if (mkdir_parents_chgrp(dir, perm->dir_create_mode, + perm->file_create_gid, + perm->file_create_gid_origin) < 0 && + errno != EEXIST) { + mailbox_set_critical(box, + "mkdir_parents(%s) failed: %m", dir); + return -1; + } + /* try again */ + old_mask = umask(0666 & ~perm->file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + } + if (fd == -1) { + mailbox_set_critical(box, "open(%s, O_CREAT) failed: %m", path); + } else if (perm->file_create_gid == (gid_t)-1) { + /* no group change */ + } else if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, + "fchown(%s, -1, %ld) failed: %m", + path, (long)perm->file_create_gid); + } + /* continue anyway */ + } + return fd; +} + +int sdbox_file_move(struct dbox_file *file, bool alt_path) +{ + struct mail_storage *storage = &file->storage->storage; + struct ostream *output; + const char *dest_dir, *temp_path, *dest_path, *p; + struct stat st; + struct utimbuf ut; + bool deleted; + int out_fd, ret = 0; + + i_assert(file->input != NULL); + + if (dbox_file_is_in_alt(file) == alt_path) + return 0; + if (file->alt_path == NULL) + return 0; + + if (stat(file->cur_path, &st) < 0 && errno == ENOENT) { + /* already expunged/moved by another session */ + return 0; + } + + dest_path = !alt_path ? file->primary_path : file->alt_path; + + i_assert(dest_path != NULL); + + p = strrchr(dest_path, '/'); + i_assert(p != NULL); + dest_dir = t_strdup_until(dest_path, p); + temp_path = t_strdup_printf("%s/%s", dest_dir, + dbox_generate_tmp_filename()); + + /* first copy the file. make sure to catch every possible error + since we really don't want to break the file. */ + out_fd = file->storage->v.file_create_fd(file, temp_path, TRUE); + if (out_fd == -1) + return -1; + + output = o_stream_create_fd_file(out_fd, 0, FALSE); + i_stream_seek(file->input, 0); + o_stream_nsend_istream(output, file->input); + if (o_stream_finish(output) < 0) { + mail_storage_set_critical(storage, "write(%s) failed: %s", + temp_path, o_stream_get_error(output)); + ret = -1; + } + o_stream_unref(&output); + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER && ret == 0) { + if (fsync(out_fd) < 0) { + mail_storage_set_critical(storage, + "fsync(%s) failed: %m", temp_path); + ret = -1; + } + } + if (close(out_fd) < 0) { + mail_storage_set_critical(storage, + "close(%s) failed: %m", temp_path); + ret = -1; + } + if (ret < 0) { + i_unlink(temp_path); + return -1; + } + /* preserve the original atime/mtime. this isn't necessary for Dovecot, + but could be useful for external reasons. */ + ut.actime = st.st_atime; + ut.modtime = st.st_mtime; + if (utime(temp_path, &ut) < 0) { + mail_storage_set_critical(storage, + "utime(%s) failed: %m", temp_path); + } + + /* the temp file was successfully written. rename it now to the + destination file. the destination shouldn't exist, but if it does + its contents should be the same (except for maybe older metadata) */ + if (rename(temp_path, dest_path) < 0) { + mail_storage_set_critical(storage, + "rename(%s, %s) failed: %m", temp_path, dest_path); + i_unlink_if_exists(temp_path); + return -1; + } + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync_path(dest_dir) < 0) { + mail_storage_set_critical(storage, + "fdatasync(%s) failed: %m", dest_dir); + i_unlink(dest_path); + return -1; + } + } + if (unlink(file->cur_path) < 0) { + dbox_file_set_syscall_error(file, "unlink()"); + if (errno == EACCES) { + /* configuration problem? revert the write */ + i_unlink(dest_path); + } + /* who knows what happened to the file. keep both just to be + sure both won't get deleted. */ + return -1; + } + + /* file was successfully moved - reopen it */ + dbox_file_close(file); + if (dbox_file_open(file, &deleted) <= 0) { + mail_storage_set_critical(storage, + "dbox_file_move(%s): reopening file failed", dest_path); + return -1; + } + return 0; +} + +static int +sdbox_unlink_attachments(struct sdbox_file *sfile, + const ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + struct dbox_storage *storage = sfile->file.storage; + const struct mail_attachment_extref *extref; + const char *path; + int ret = 0; + + array_foreach(extrefs, extref) T_BEGIN { + path = sdbox_file_attachment_relpath(sfile, extref->path); + if (index_attachment_delete(&storage->storage, + storage->attachment_fs, path) < 0) + ret = -1; + } T_END; + return ret; +} + +int sdbox_file_unlink_with_attachments(struct sdbox_file *sfile) +{ + ARRAY_TYPE(mail_attachment_extref) extrefs; + const char *extrefs_line; + pool_t pool; + int ret; + + ret = sdbox_file_get_attachments(&sfile->file, &extrefs_line); + if (ret < 0) + return -1; + if (ret == 0) { + /* no attachments */ + return dbox_file_unlink(&sfile->file); + } + + pool = pool_alloconly_create("sdbox attachments unlink", 1024); + p_array_init(&extrefs, pool, 16); + if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) { + i_warning("%s: Ignoring corrupted extref: %s", + sfile->file.cur_path, extrefs_line); + array_clear(&extrefs); + } + + /* try to delete the file first, so if it fails we don't have + missing attachments */ + if ((ret = dbox_file_unlink(&sfile->file)) >= 0) + (void)sdbox_unlink_attachments(sfile, &extrefs); + pool_unref(&pool); + return ret; +} diff --git a/src/lib-storage/index/dbox-single/sdbox-file.h b/src/lib-storage/index/dbox-single/sdbox-file.h new file mode 100644 index 0000000..ba5a7f9 --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-file.h @@ -0,0 +1,43 @@ +#ifndef SDBOX_FILE_H +#define SDBOX_FILE_H + +#include "dbox-file.h" + +struct sdbox_file { + struct dbox_file file; + struct sdbox_mailbox *mbox; + + /* 0 while file is being created */ + uint32_t uid; + + /* list of attachment paths while saving/copying message */ + pool_t attachment_pool; + ARRAY_TYPE(const_string) attachment_paths; + bool written_to_disk; +}; + +struct dbox_file *sdbox_file_init(struct sdbox_mailbox *mbox, uint32_t uid); +struct dbox_file *sdbox_file_create(struct sdbox_mailbox *mbox); +void sdbox_file_free(struct dbox_file *file); + +/* Get file's extrefs metadata. */ +int sdbox_file_get_attachments(struct dbox_file *file, const char **extrefs_r); +/* Returns attachment path for this file, given the source path. The result is + always <hash>-<guid>-<mailbox_guid>-<uid>. The source path is expected to + contain <hash>-<guid>[-*]. */ +const char * +sdbox_file_attachment_relpath(struct sdbox_file *file, const char *srcpath); + +/* Assign UID for a newly created file (by renaming it) */ +int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid); + +int sdbox_file_create_fd(struct dbox_file *file, const char *path, + bool parents); +/* Move the file to alt path or back. */ +int sdbox_file_move(struct dbox_file *file, bool alt_path); +/* Unlink file and all of its referenced attachments. */ +int sdbox_file_unlink_with_attachments(struct sdbox_file *sfile); +/* Unlink file and its attachments when rolling back a saved message. */ +int sdbox_file_unlink_aborted_save(struct sdbox_file *file); + +#endif diff --git a/src/lib-storage/index/dbox-single/sdbox-mail.c b/src/lib-storage/index/dbox-single/sdbox-mail.c new file mode 100644 index 0000000..3b0352c --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-mail.c @@ -0,0 +1,182 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "str.h" +#include "index-mail.h" +#include "dbox-mail.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" + +#include <sys/stat.h> + +static void sdbox_mail_set_expunged(struct dbox_mail *mail) +{ + struct mail *_mail = &mail->imail.mail.mail; + + mail_index_refresh(_mail->box->index); + if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) { + mail_set_expunged(_mail); + return; + } + + mail_set_critical(_mail, "dbox: Unexpectedly lost uid"); + sdbox_set_mailbox_corrupted(_mail->box); +} + +static int sdbox_mail_file_set(struct dbox_mail *mail) +{ + struct mail *_mail = &mail->imail.mail.mail; + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(_mail->box); + bool deleted; + int ret; + + if (mail->open_file != NULL) { + /* already set */ + return 0; + } else if (!_mail->saving) { + mail->open_file = sdbox_file_init(mbox, _mail->uid); + return 0; + } else { + /* mail is being saved in this transaction */ + mail->open_file = + sdbox_save_file_get_file(_mail->transaction, + _mail->seq); + mail->open_file->refcount++; + + /* it doesn't have input stream yet */ + ret = dbox_file_open(mail->open_file, &deleted); + if (ret <= 0) { + mail_set_critical(_mail, + "dbox: Unexpectedly lost mail being saved"); + sdbox_set_mailbox_corrupted(_mail->box); + return -1; + } + return 1; + } +} + +static int +sdbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(_mail->box); + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct stat st; + + switch (field) { + case MAIL_FETCH_REFCOUNT: + if (sdbox_mail_file_set(mail) < 0) + return -1; + + _mail->transaction->stats.fstat_lookup_count++; + if (dbox_file_stat(mail->open_file, &st) < 0) { + if (errno == ENOENT) + mail_set_expunged(_mail); + return -1; + } + *value_r = p_strdup_printf(mail->imail.mail.data_pool, "%lu", + (unsigned long)st.st_nlink); + return 0; + case MAIL_FETCH_REFCOUNT_ID: + if (sdbox_mail_file_set(mail) < 0) + return -1; + + _mail->transaction->stats.fstat_lookup_count++; + if (dbox_file_stat(mail->open_file, &st) < 0) { + if (errno == ENOENT) + mail_set_expunged(_mail); + return -1; + } + *value_r = p_strdup_printf(mail->imail.mail.data_pool, "%llu", + (unsigned long long)st.st_ino); + return 0; + case MAIL_FETCH_UIDL_BACKEND: + if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id, + offsetof(struct sdbox_index_header, flags), + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) { + *value_r = ""; + return 0; + } + break; + case MAIL_FETCH_POP3_ORDER: + if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id, + offsetof(struct sdbox_index_header, flags), + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) { + *value_r = ""; + return 0; + } + break; + default: + break; + } + return dbox_mail_get_special(_mail, field, value_r); +} + +int sdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r) +{ + struct mail *_mail = &mail->imail.mail.mail; + bool deleted; + int ret; + + if (!mail_stream_access_start(_mail)) + return -1; + + ret = sdbox_mail_file_set(mail); + if (ret < 0) + return -1; + if (ret == 0) { + if (!dbox_file_is_open(mail->open_file)) + _mail->transaction->stats.open_lookup_count++; + if (dbox_file_open(mail->open_file, &deleted) <= 0) + return -1; + if (deleted) { + sdbox_mail_set_expunged(mail); + return -1; + } + } + + *file_r = mail->open_file; + *offset_r = 0; + return 0; +} + +struct mail_vfuncs sdbox_mail_vfuncs = { + dbox_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + dbox_mail_get_received_date, + dbox_mail_get_save_date, + dbox_mail_get_virtual_size, + dbox_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + dbox_mail_get_stream, + index_mail_get_binary_stream, + sdbox_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/dbox-single/sdbox-save.c b/src/lib-storage/index/dbox-single/sdbox-save.c new file mode 100644 index 0000000..03e24ab --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-save.c @@ -0,0 +1,359 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "fdatasync-path.h" +#include "hex-binary.h" +#include "hex-dec.h" +#include "str.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "write-full.h" +#include "index-mail.h" +#include "mail-copy.h" +#include "index-pop3-uidl.h" +#include "dbox-attachment.h" +#include "dbox-save.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" + + +struct sdbox_save_context { + struct dbox_save_context ctx; + + struct sdbox_mailbox *mbox; + struct sdbox_sync_context *sync_ctx; + + struct dbox_file *cur_file; + struct dbox_file_append_context *append_ctx; + + uint32_t first_saved_seq; + ARRAY(struct dbox_file *) files; +}; + +#define SDBOX_SAVECTX(s) container_of(DBOX_SAVECTX(s), struct sdbox_save_context, ctx) + +struct dbox_file * +sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq) +{ + struct sdbox_save_context *ctx = SDBOX_SAVECTX(t->save_ctx); + struct dbox_file *const *files, *file; + unsigned int count; + + i_assert(seq >= ctx->first_saved_seq); + + files = array_get(&ctx->files, &count); + i_assert(count > 0); + i_assert(seq - ctx->first_saved_seq < count); + + file = files[seq - ctx->first_saved_seq]; + i_assert(((struct sdbox_file *)file)->written_to_disk); + return file; +} + +struct mail_save_context * +sdbox_save_alloc(struct mailbox_transaction_context *t) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(t->box); + struct sdbox_save_context *ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx != NULL) { + /* use the existing allocated structure */ + ctx = SDBOX_SAVECTX(t->save_ctx); + ctx->cur_file = NULL; + ctx->ctx.failed = FALSE; + ctx->ctx.finished = FALSE; + ctx->ctx.dbox_output = NULL; + return &ctx->ctx.ctx; + } + + ctx = i_new(struct sdbox_save_context, 1); + ctx->ctx.ctx.transaction = t; + ctx->ctx.trans = t->itrans; + ctx->mbox = mbox; + i_array_init(&ctx->files, 32); + t->save_ctx = &ctx->ctx.ctx; + return t->save_ctx; +} + +void sdbox_save_add_file(struct mail_save_context *_ctx, struct dbox_file *file) +{ + struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx); + struct dbox_file *const *files; + unsigned int count; + + if (ctx->first_saved_seq == 0) + ctx->first_saved_seq = ctx->ctx.seq; + + files = array_get(&ctx->files, &count); + if (count > 0) { + /* a plugin may leave a previously saved file open. + we'll close it here to avoid eating too many fds. */ + dbox_file_close(files[count-1]); + } + array_push_back(&ctx->files, &file); +} + +int sdbox_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx); + struct dbox_file *file; + int ret; + + file = sdbox_file_create(ctx->mbox); + ctx->append_ctx = dbox_file_append_init(file); + ret = dbox_file_get_append_stream(ctx->append_ctx, + &ctx->ctx.dbox_output); + if (ret <= 0) { + i_assert(ret != 0); + dbox_file_append_rollback(&ctx->append_ctx); + dbox_file_unref(&file); + ctx->ctx.failed = TRUE; + return -1; + } + ctx->cur_file = file; + dbox_save_begin(&ctx->ctx, input); + + sdbox_save_add_file(_ctx, file); + return ctx->ctx.failed ? -1 : 0; +} + +static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx, + struct dbox_file *file) +{ + struct sdbox_file *sfile = (struct sdbox_file *)file; + const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr; + const struct mail_attachment_extref *extrefs; + struct dbox_message_header dbox_msg_hdr; + uoff_t message_size; + guid_128_t guid_128; + unsigned int i, count; + + i_assert(file->msg_header_size == sizeof(dbox_msg_hdr)); + + message_size = ctx->dbox_output->offset - + file->msg_header_size - file->file_header_size; + + dbox_save_write_metadata(&ctx->ctx, ctx->dbox_output, + message_size, NULL, guid_128); + dbox_msg_header_fill(&dbox_msg_hdr, message_size); + if (o_stream_pwrite(ctx->dbox_output, &dbox_msg_hdr, + sizeof(dbox_msg_hdr), + file->file_header_size) < 0) { + dbox_file_set_syscall_error(file, "pwrite()"); + return -1; + } + sfile->written_to_disk = TRUE; + + /* remember the attachment paths until commit time */ + extrefs_arr = index_attachment_save_get_extrefs(&ctx->ctx); + if (extrefs_arr != NULL) + extrefs = array_get(extrefs_arr, &count); + else { + extrefs = NULL; + count = 0; + } + if (count > 0) { + sfile->attachment_pool = + pool_alloconly_create("sdbox attachment paths", 512); + p_array_init(&sfile->attachment_paths, + sfile->attachment_pool, count); + for (i = 0; i < count; i++) { + const char *path = p_strdup(sfile->attachment_pool, + extrefs[i].path); + array_push_back(&sfile->attachment_paths, &path); + } + } + return 0; +} + +static int dbox_save_finish_write(struct mail_save_context *_ctx) +{ + struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; + struct dbox_file **files; + + ctx->ctx.finished = TRUE; + if (ctx->ctx.dbox_output == NULL) + return -1; + + if (_ctx->data.save_date != (time_t)-1) { + /* we can't change ctime, but we can add the date to cache */ + struct index_mail *mail = (struct index_mail *)_ctx->dest_mail; + uint32_t t = _ctx->data.save_date; + + index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t)); + } + dbox_save_end(&ctx->ctx); + + files = array_back_modifiable(&ctx->files); + if (!ctx->ctx.failed) T_BEGIN { + if (dbox_save_mail_write_metadata(&ctx->ctx, *files) < 0) + ctx->ctx.failed = TRUE; + } T_END; + + if (ctx->ctx.failed) { + index_storage_save_abort_last(&ctx->ctx.ctx, ctx->ctx.seq); + dbox_file_append_rollback(&ctx->append_ctx); + dbox_file_unlink(*files); + dbox_file_unref(files); + array_pop_back(&ctx->files); + } else { + dbox_file_append_checkpoint(ctx->append_ctx); + if (dbox_file_append_commit(&ctx->append_ctx) < 0) + ctx->ctx.failed = TRUE; + dbox_file_close(*files); + } + + i_stream_unref(&ctx->ctx.input); + ctx->ctx.dbox_output = NULL; + + return ctx->ctx.failed ? -1 : 0; +} + +int sdbox_save_finish(struct mail_save_context *ctx) +{ + int ret; + + ret = dbox_save_finish_write(ctx); + index_save_context_free(ctx); + return ret; +} + +void sdbox_save_cancel(struct mail_save_context *_ctx) +{ + struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)sdbox_save_finish(_ctx); +} + +static int dbox_save_assign_uids(struct sdbox_save_context *ctx, + const ARRAY_TYPE(seq_range) *uids) +{ + struct dbox_file *const *files; + struct seq_range_iter iter; + unsigned int i, count, n = 0; + uint32_t uid; + bool ret; + + seq_range_array_iter_init(&iter, uids); + files = array_get(&ctx->files, &count); + for (i = 0; i < count; i++) { + struct sdbox_file *sfile = (struct sdbox_file *)files[i]; + + ret = seq_range_array_iter_nth(&iter, n++, &uid); + i_assert(ret); + if (sdbox_file_assign_uid(sfile, uid) < 0) + return -1; + if (ctx->ctx.highest_pop3_uidl_seq == i+1) { + index_pop3_uidl_set_max_uid(&ctx->mbox->box, + ctx->ctx.trans, uid); + } + } + i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); + return 0; +} + +static void dbox_save_unref_files(struct sdbox_save_context *ctx) +{ + struct dbox_file **files; + unsigned int i, count; + + files = array_get_modifiable(&ctx->files, &count); + for (i = 0; i < count; i++) { + if (ctx->ctx.failed) { + struct sdbox_file *sfile = + (struct sdbox_file *)files[i]; + + (void)sdbox_file_unlink_aborted_save(sfile); + } + dbox_file_unref(&files[i]); + } + array_free(&ctx->files); +} + +int sdbox_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + const struct mail_index_header *hdr; + + i_assert(ctx->ctx.finished); + + if (array_count(&ctx->files) == 0) { + /* the mail must be freed in the commit_pre() */ + return 0; + } + + if (sdbox_sync_begin(ctx->mbox, SDBOX_SYNC_FLAG_FORCE | + SDBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) { + sdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* update dbox header flags */ + dbox_save_update_header_flags(&ctx->ctx, ctx->sync_ctx->sync_view, + ctx->mbox->hdr_ext_id, offsetof(struct sdbox_index_header, flags)); + + /* assign UIDs for new messages */ + hdr = mail_index_get_header(ctx->sync_ctx->sync_view); + mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, + &_t->changes->saved_uids); + if (dbox_save_assign_uids(ctx, &_t->changes->saved_uids) < 0) { + sdbox_transaction_save_rollback(_ctx); + return -1; + } + + _t->changes->uid_validity = hdr->uid_validity; + return 0; +} + +void sdbox_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result) +{ + struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx); + struct mail_storage *storage = _ctx->transaction->box->storage; + + _ctx->transaction = NULL; /* transaction is already freed */ + + if (array_count(&ctx->files) == 0) { + sdbox_transaction_save_rollback(_ctx); + return; + } + + mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx, + result); + + if (sdbox_sync_finish(&ctx->sync_ctx, TRUE) < 0) + ctx->ctx.failed = TRUE; + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + const char *box_path = mailbox_get_path(&ctx->mbox->box); + + if (fdatasync_path(box_path) < 0) { + mail_set_critical(_ctx->dest_mail, + "fdatasync_path(%s) failed: %m", box_path); + } + } + i_assert(ctx->ctx.finished); + dbox_save_unref_files(ctx); + i_free(ctx); +} + +void sdbox_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx); + + ctx->ctx.failed = TRUE; + if (!ctx->ctx.finished) + sdbox_save_cancel(_ctx); + dbox_save_unref_files(ctx); + + if (ctx->sync_ctx != NULL) + (void)sdbox_sync_finish(&ctx->sync_ctx, FALSE); + i_free(ctx); +} diff --git a/src/lib-storage/index/dbox-single/sdbox-storage.c b/src/lib-storage/index/dbox-single/sdbox-storage.c new file mode 100644 index 0000000..a25c325 --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c @@ -0,0 +1,535 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "fs-api.h" +#include "master-service.h" +#include "mail-index-modseq.h" +#include "mail-search-build.h" +#include "mailbox-list-private.h" +#include "index-pop3-uidl.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" +#include "sdbox-storage.h" + +extern struct mail_storage dbox_storage, sdbox_storage; +extern struct mailbox sdbox_mailbox; +extern struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs; + +static struct event_category event_category_sdbox = { + .name = "sdbox", + .parent = &event_category_storage, +}; + +static struct mail_storage *sdbox_storage_alloc(void) +{ + struct sdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("sdbox storage", 512+256); + storage = p_new(pool, struct sdbox_storage, 1); + storage->storage.v = sdbox_dbox_storage_vfuncs; + storage->storage.storage = sdbox_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +static int sdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, + const char **error_r) +{ + struct dbox_storage *storage = DBOX_STORAGE(_storage); + enum fs_properties props; + + if (dbox_storage_create(_storage, ns, error_r) < 0) + return -1; + + if (storage->attachment_fs != NULL) { + props = fs_get_properties(storage->attachment_fs); + if ((props & FS_PROPERTY_RENAME) == 0) { + *error_r = "mail_attachment_fs: " + "Backend doesn't support renaming"; + return -1; + } + } + return 0; +} + +static const char * +sdbox_storage_find_root_dir(const struct mail_namespace *ns) +{ + bool debug = ns->mail_set->mail_debug; + const char *home, *path; + + if (ns->owner != NULL && + mail_user_get_home(ns->owner, &home) > 0) { + path = t_strconcat(home, "/sdbox", NULL); + if (access(path, R_OK|W_OK|X_OK) == 0) { + if (debug) + i_debug("sdbox: root exists (%s)", path); + return path; + } + if (debug) + i_debug("sdbox: access(%s, rwx): failed: %m", path); + } + return NULL; +} + +static bool sdbox_storage_autodetect(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + bool debug = ns->mail_set->mail_debug; + struct stat st; + const char *path, *root_dir; + + if (set->root_dir != NULL) + root_dir = set->root_dir; + else { + root_dir = sdbox_storage_find_root_dir(ns); + if (root_dir == NULL) { + if (debug) + i_debug("sdbox: couldn't find root dir"); + return FALSE; + } + } + + /* NOTE: this check works for mdbox as well. we'll rely on the + autodetect ordering to catch mdbox before we get here. */ + path = t_strconcat(root_dir, "/"DBOX_MAILBOX_DIR_NAME, NULL); + if (stat(path, &st) < 0) { + if (debug) + i_debug("sdbox autodetect: stat(%s) failed: %m", path); + return FALSE; + } + + if (!S_ISDIR(st.st_mode)) { + if (debug) + i_debug("sdbox autodetect: %s not a directory", path); + return FALSE; + } + + set->root_dir = root_dir; + dbox_storage_get_list_settings(ns, set); + return TRUE; +} + +static struct mailbox * +sdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct sdbox_mailbox *mbox; + struct index_mailbox_context *ibox; + pool_t pool; + + /* dbox can't work without index files */ + flags &= ENUM_NEGATE(MAILBOX_FLAG_NO_INDEX_FILES); + + pool = pool_alloconly_create("sdbox mailbox", 1024*3); + mbox = p_new(pool, struct sdbox_mailbox, 1); + mbox->box = sdbox_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &sdbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + ibox = INDEX_STORAGE_CONTEXT(&mbox->box); + ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY; + + mbox->storage = SDBOX_STORAGE(storage); + return &mbox->box; +} + +int sdbox_read_header(struct sdbox_mailbox *mbox, + struct sdbox_index_header *hdr, bool log_error, + bool *need_resize_r) +{ + struct mail_index_view *view; + const void *data; + size_t data_size; + int ret = 0; + + i_assert(mbox->box.opened); + + view = mail_index_view_open(mbox->box.index); + mail_index_get_header_ext(view, mbox->hdr_ext_id, + &data, &data_size); + if (data_size < SDBOX_INDEX_HEADER_MIN_SIZE && + (!mbox->box.creating || data_size != 0)) { + if (log_error) { + mailbox_set_critical(&mbox->box, + "sdbox: Invalid dbox header size"); + } + ret = -1; + } else { + i_zero(hdr); + memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr))); + if (guid_128_is_empty(hdr->mailbox_guid)) + ret = -1; + else { + /* data is valid. remember it in case mailbox + is being reset */ + mail_index_set_ext_init_data(mbox->box.index, + mbox->hdr_ext_id, + hdr, sizeof(*hdr)); + } + } + mail_index_view_close(&view); + *need_resize_r = data_size < sizeof(*hdr); + return ret; +} + +static void sdbox_update_header(struct sdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update) +{ + struct sdbox_index_header hdr, new_hdr; + bool need_resize; + + if (sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0) { + i_zero(&hdr); + need_resize = TRUE; + } + + new_hdr = hdr; + + if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) { + memcpy(new_hdr.mailbox_guid, update->mailbox_guid, + sizeof(new_hdr.mailbox_guid)); + } else if (guid_128_is_empty(new_hdr.mailbox_guid)) { + guid_128_generate(new_hdr.mailbox_guid); + } + + if (need_resize) { + mail_index_ext_resize_hdr(trans, mbox->hdr_ext_id, + sizeof(new_hdr)); + } + if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) { + mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0, + &new_hdr, sizeof(new_hdr)); + } + memcpy(mbox->mailbox_guid, new_hdr.mailbox_guid, + sizeof(mbox->mailbox_guid)); +} + +int sdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box); + struct mail_index_transaction *new_trans = NULL; + const struct mail_index_header *hdr; + uint32_t uid_validity, uid_next; + + if (trans == NULL) { + new_trans = mail_index_transaction_begin(box->view, 0); + trans = new_trans; + } + + hdr = mail_index_get_header(box->view); + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + else if (hdr->uid_validity != 0) + uid_validity = hdr->uid_validity; + else { + /* set uidvalidity */ + uid_validity = dbox_get_uidvalidity_next(box->list); + } + + if (hdr->uid_validity != uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + if (update != NULL && hdr->next_uid < update->min_next_uid) { + uid_next = update->min_next_uid; + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + } + if (update != NULL && update->min_first_recent_uid != 0 && + hdr->first_recent_uid < update->min_first_recent_uid) { + uint32_t first_recent_uid = update->min_first_recent_uid; + + mail_index_update_header(trans, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + if (update != NULL && update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(box->view) < + update->min_highest_modseq) { + mail_index_modseq_enable(box->index); + mail_index_update_highest_modseq(trans, + update->min_highest_modseq); + } + + if (box->inbox_user && box->creating) { + /* initialize pop3-uidl header when creating mailbox + (not on mailbox_update()) */ + index_pop3_uidl_set_max_uid(box, trans, 0); + } + + sdbox_update_header(mbox, trans, update); + if (new_trans != NULL) { + if (mail_index_transaction_commit(&new_trans) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + return 0; +} + +static const char * +sdbox_get_attachment_path_suffix(struct dbox_file *_file) +{ + struct sdbox_file *file = (struct sdbox_file *)_file; + + return t_strdup_printf("-%s-%u", + guid_128_to_string(file->mbox->mailbox_guid), + file->uid); +} + +void sdbox_set_mailbox_corrupted(struct mailbox *box) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box); + struct sdbox_index_header hdr; + bool need_resize; + + if (sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0 || + hdr.rebuild_count == 0) + mbox->corrupted_rebuild_count = 1; + else + mbox->corrupted_rebuild_count = hdr.rebuild_count; +} + +static void sdbox_set_file_corrupted(struct dbox_file *_file) +{ + struct sdbox_file *file = (struct sdbox_file *)_file; + + sdbox_set_mailbox_corrupted(&file->mbox->box); +} + +static int sdbox_mailbox_alloc_index(struct sdbox_mailbox *mbox) +{ + struct sdbox_index_header hdr; + + if (index_storage_mailbox_alloc_index(&mbox->box) < 0) + return -1; + + mbox->hdr_ext_id = + mail_index_ext_register(mbox->box.index, "dbox-hdr", + sizeof(struct sdbox_index_header), 0, 0); + /* set the initialization data in case the mailbox is created */ + i_zero(&hdr); + guid_128_generate(hdr.mailbox_guid); + mail_index_set_ext_init_data(mbox->box.index, mbox->hdr_ext_id, + &hdr, sizeof(hdr)); + return 0; +} + +static int sdbox_mailbox_open(struct mailbox *box) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box); + struct sdbox_index_header hdr; + bool need_resize; + + if (dbox_mailbox_check_existence(box) < 0) + return -1; + + if (sdbox_mailbox_alloc_index(mbox) < 0) + return -1; + + if (dbox_mailbox_open(box) < 0) + return -1; + + if (box->creating) { + /* wait for mailbox creation to initialize the index */ + return 0; + } + + /* get/generate mailbox guid */ + if (sdbox_read_header(mbox, &hdr, FALSE, &need_resize) < 0) { + /* looks like the mailbox is corrupted */ + (void)sdbox_sync(mbox, SDBOX_SYNC_FLAG_FORCE); + if (sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0) + i_zero(&hdr); + } + + if (guid_128_is_empty(hdr.mailbox_guid)) { + /* regenerate it */ + if (sdbox_mailbox_create_indexes(box, NULL, NULL) < 0 || + sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0) + return -1; + } + memcpy(mbox->mailbox_guid, hdr.mailbox_guid, + sizeof(mbox->mailbox_guid)); + return 0; +} + +static void sdbox_mailbox_close(struct mailbox *box) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box); + + if (mbox->corrupted_rebuild_count != 0) + (void)sdbox_sync(mbox, 0); + + dbox_mailbox_close_cleanup(box); + dbox_mailbox_close(box); +} + +static int +sdbox_mailbox_create(struct mailbox *box, + const struct mailbox_update *update, bool directory) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box); + struct sdbox_index_header hdr; + bool need_resize; + + if (dbox_mailbox_create(box, update, directory) < 0) + return -1; + if (directory || !guid_128_is_empty(mbox->mailbox_guid)) + return 0; + + /* another process just created the mailbox. read the mailbox_guid. */ + if (sdbox_read_header(mbox, &hdr, FALSE, &need_resize) < 0) { + mailbox_set_critical(box, + "sdbox: Failed to read newly created dbox header"); + return -1; + } + memcpy(mbox->mailbox_guid, hdr.mailbox_guid, + sizeof(mbox->mailbox_guid)); + i_assert(!guid_128_is_empty(mbox->mailbox_guid)); + return 0; +} + +static int +sdbox_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box); + + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + if ((items & MAILBOX_METADATA_GUID) != 0) { + memcpy(metadata_r->guid, mbox->mailbox_guid, + sizeof(metadata_r->guid)); + } + return 0; +} + +static int +dbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + if (sdbox_mailbox_create_indexes(box, update, NULL) < 0) + return -1; + return index_storage_mailbox_update_common(box, update); +} + +struct mail_storage sdbox_storage = { + .name = SDBOX_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + .event_category = &event_category_sdbox, + + .v = { + NULL, + sdbox_storage_alloc, + sdbox_storage_create, + dbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + sdbox_storage_autodetect, + sdbox_mailbox_alloc, + NULL, + mail_storage_list_index_rebuild, + } +}; + +struct mail_storage dbox_storage = { + .name = "dbox", /* alias */ + .class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG, + .event_category = &event_category_sdbox, + + .v = { + NULL, + sdbox_storage_alloc, + sdbox_storage_create, + dbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + sdbox_storage_autodetect, + sdbox_mailbox_alloc, + NULL, + mail_storage_list_index_rebuild, + } +}; + +struct mailbox sdbox_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + sdbox_mailbox_open, + sdbox_mailbox_close, + index_storage_mailbox_free, + sdbox_mailbox_create, + dbox_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + sdbox_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + sdbox_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + dbox_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + sdbox_save_alloc, + sdbox_save_begin, + dbox_save_continue, + sdbox_save_finish, + sdbox_save_cancel, + sdbox_copy, + sdbox_transaction_save_commit_pre, + sdbox_transaction_save_commit_post, + sdbox_transaction_save_rollback, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs = { + sdbox_file_free, + sdbox_file_create_fd, + sdbox_mail_open, + sdbox_mailbox_create_indexes, + sdbox_get_attachment_path_suffix, + sdbox_set_mailbox_corrupted, + sdbox_set_file_corrupted +}; diff --git a/src/lib-storage/index/dbox-single/sdbox-storage.h b/src/lib-storage/index/dbox-single/sdbox-storage.h new file mode 100644 index 0000000..66d08da --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-storage.h @@ -0,0 +1,69 @@ +#ifndef SDBOX_STORAGE_H +#define SDBOX_STORAGE_H + +#include "index-storage.h" +#include "dbox-storage.h" + +#define SDBOX_STORAGE_NAME "sdbox" +#define SDBOX_MAIL_FILE_PREFIX "u." +#define SDBOX_MAIL_FILE_FORMAT SDBOX_MAIL_FILE_PREFIX"%u" + +#define SDBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t)) +struct sdbox_index_header { + /* increased every time a full mailbox rebuild is done */ + uint32_t rebuild_count; + guid_128_t mailbox_guid; + uint8_t flags; /* enum dbox_index_header_flags */ + uint8_t unused[3]; +}; + +struct sdbox_storage { + struct dbox_storage storage; +}; + +struct sdbox_mailbox { + struct mailbox box; + struct sdbox_storage *storage; + + uint32_t hdr_ext_id; + /* if non-zero, storage should be rebuilt (except if rebuild_count + has changed from this value) */ + uint32_t corrupted_rebuild_count; + + guid_128_t mailbox_guid; +}; + +#define SDBOX_STORAGE(s) container_of(DBOX_STORAGE(s), struct sdbox_storage, storage) +#define SDBOX_MAILBOX(s) container_of(s, struct sdbox_mailbox, box) + +extern struct mail_vfuncs sdbox_mail_vfuncs; + +int sdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r); + +int sdbox_read_header(struct sdbox_mailbox *mbox, + struct sdbox_index_header *hdr, bool log_error, + bool *need_resize_r); +int sdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans); +void sdbox_set_mailbox_corrupted(struct mailbox *box); + +struct mail_save_context * +sdbox_save_alloc(struct mailbox_transaction_context *_t); +int sdbox_save_begin(struct mail_save_context *ctx, struct istream *input); +int sdbox_save_finish(struct mail_save_context *ctx); +void sdbox_save_cancel(struct mail_save_context *ctx); + +struct dbox_file * +sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq); +void sdbox_save_add_file(struct mail_save_context *ctx, struct dbox_file *file); + +int sdbox_transaction_save_commit_pre(struct mail_save_context *ctx); +void sdbox_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void sdbox_transaction_save_rollback(struct mail_save_context *ctx); + +int sdbox_copy(struct mail_save_context *ctx, struct mail *mail); + +#endif diff --git a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c new file mode 100644 index 0000000..aa4832f --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c @@ -0,0 +1,218 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "index-rebuild.h" +#include "mail-cache.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" + +#include <dirent.h> + +static void sdbox_sync_set_uidvalidity(struct index_rebuild_context *ctx) +{ + uint32_t uid_validity; + + /* if uidvalidity is set in the old index, use it */ + uid_validity = mail_index_get_header(ctx->view)->uid_validity; + if (uid_validity == 0) + uid_validity = dbox_get_uidvalidity_next(ctx->box->list); + + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); +} + +static int +sdbox_sync_add_file_index(struct index_rebuild_context *ctx, + struct dbox_file *file, uint32_t uid, bool primary) +{ + uint32_t seq; + bool deleted; + int ret; + + if ((ret = dbox_file_open(file, &deleted)) > 0) { + if (deleted) + return 0; + ret = dbox_file_seek(file, 0); + } + if (ret == 0) { + if ((ret = dbox_file_fix(file, 0)) > 0) + ret = dbox_file_seek(file, 0); + } + + if (ret <= 0) { + if (ret < 0) + return -1; + + i_warning("sdbox: Skipping unfixable file: %s", file->cur_path); + return 0; + } + + if (!dbox_file_is_in_alt(file) && !primary) { + /* we were supposed to open the file in alt storage, but it + exists in primary storage as well. skip it to avoid adding + it twice. */ + return 0; + } + + mail_index_append(ctx->trans, uid, &seq); + T_BEGIN { + index_rebuild_index_metadata(ctx, seq, uid); + } T_END; + return 0; +} + +static int +sdbox_sync_add_file(struct index_rebuild_context *ctx, + const char *fname, bool primary) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(ctx->box); + struct dbox_file *file; + uint32_t uid; + int ret; + + if (!str_begins(fname, SDBOX_MAIL_FILE_PREFIX)) + return 0; + fname += strlen(SDBOX_MAIL_FILE_PREFIX); + + if (str_to_uint32(fname, &uid) < 0 || uid == 0) { + i_warning("sdbox %s: Ignoring invalid filename %s", + mailbox_get_path(ctx->box), fname); + return 0; + } + + file = sdbox_file_init(mbox, uid); + if (!primary) + file->cur_path = file->alt_path; + ret = sdbox_sync_add_file_index(ctx, file, uid, primary); + dbox_file_unref(&file); + return ret; +} + +static int sdbox_sync_index_rebuild_dir(struct index_rebuild_context *ctx, + const char *path, bool primary) +{ + DIR *dir; + struct dirent *d; + int ret = 0; + + dir = opendir(path); + if (dir == NULL) { + if (errno == ENOENT) { + if (!primary) { + /* alt directory doesn't exist, ignore */ + return 0; + } + return index_mailbox_fix_inconsistent_existence(ctx->box, path); + } + mailbox_set_critical(ctx->box, "opendir(%s) failed: %m", path); + return -1; + } + do { + errno = 0; + if ((d = readdir(dir)) == NULL) + break; + + ret = sdbox_sync_add_file(ctx, d->d_name, primary); + } while (ret >= 0); + if (errno != 0) { + mailbox_set_critical(ctx->box, "readdir(%s) failed: %m", path); + ret = -1; + } + + if (closedir(dir) < 0) { + mailbox_set_critical(ctx->box, "closedir(%s) failed: %m", path); + ret = -1; + } + return ret; +} + +static void sdbox_sync_update_header(struct index_rebuild_context *ctx) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(ctx->box); + struct sdbox_index_header hdr; + bool need_resize; + + if (sdbox_read_header(mbox, &hdr, FALSE, &need_resize) < 0) + i_zero(&hdr); + if (guid_128_is_empty(hdr.mailbox_guid)) + guid_128_generate(hdr.mailbox_guid); + if (++hdr.rebuild_count == 0) + hdr.rebuild_count = 1; + /* mailbox is being reset. this gets written directly there */ + mail_index_set_ext_init_data(ctx->box->index, mbox->hdr_ext_id, + &hdr, sizeof(hdr)); +} + +static int +sdbox_sync_index_rebuild_singles(struct index_rebuild_context *ctx) +{ + const char *path, *alt_path; + int ret = 0; + + path = mailbox_get_path(ctx->box); + if (mailbox_get_path_to(ctx->box, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX, + &alt_path) < 0) + return -1; + + sdbox_sync_set_uidvalidity(ctx); + if (sdbox_sync_index_rebuild_dir(ctx, path, TRUE) < 0) { + mailbox_set_critical(ctx->box, "sdbox: Rebuilding failed"); + ret = -1; + } else if (alt_path != NULL) { + if (sdbox_sync_index_rebuild_dir(ctx, alt_path, FALSE) < 0) { + mailbox_set_critical(ctx->box, + "sdbox: Rebuilding failed on alt path %s", + alt_path); + ret = -1; + } + } + sdbox_sync_update_header(ctx); + return ret; +} + +int sdbox_sync_index_rebuild(struct sdbox_mailbox *mbox, bool force) +{ + struct index_rebuild_context *ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + struct sdbox_index_header hdr; + bool need_resize; + int ret; + + if (!force && sdbox_read_header(mbox, &hdr, FALSE, &need_resize) == 0) { + if (hdr.rebuild_count != mbox->corrupted_rebuild_count && + hdr.rebuild_count != 0) { + /* already rebuilt by someone else */ + return 0; + } + } + i_warning("sdbox %s: Rebuilding index", mailbox_get_path(&mbox->box)); + + if (dbox_verify_alt_storage(mbox->box.list) < 0) { + mailbox_set_critical(&mbox->box, + "sdbox: Alt storage not mounted, " + "aborting index rebuild"); + return -1; + } + + view = mail_index_view_open(mbox->box.index); + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + + ctx = index_index_rebuild_init(&mbox->box, view, trans); + ret = sdbox_sync_index_rebuild_singles(ctx); + index_index_rebuild_deinit(&ctx, dbox_get_uidvalidity_next); + + if (ret < 0) + mail_index_transaction_rollback(&trans); + else { + mail_index_unset_fscked(trans); + ret = mail_index_transaction_commit(&trans); + } + mail_index_view_close(&view); + mbox->corrupted_rebuild_count = 0; + return ret; +} diff --git a/src/lib-storage/index/dbox-single/sdbox-sync.c b/src/lib-storage/index/dbox-single/sdbox-sync.c new file mode 100644 index 0000000..2698ea9 --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c @@ -0,0 +1,326 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dbox-attachment.h" +#include "sdbox-storage.h" +#include "sdbox-file.h" +#include "sdbox-sync.h" +#include "mailbox-recent-flags.h" + +#define SDBOX_REBUILD_COUNT 3 + +static void +dbox_sync_file_move_if_needed(struct dbox_file *file, + enum sdbox_sync_entry_type type) +{ + struct stat st; + bool move_to_alt = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT; + bool deleted; + + if (move_to_alt == dbox_file_is_in_alt(file) && + !move_to_alt) { + /* unopened dbox files default to primary dir. + stat the file to update its location. */ + (void)dbox_file_stat(file, &st); + + } + if (move_to_alt != dbox_file_is_in_alt(file)) { + /* move the file. if it fails, nothing broke so + don't worry about it. */ + if (dbox_file_open(file, &deleted) > 0 && !deleted) + (void)sdbox_file_move(file, move_to_alt); + } +} + +static void sdbox_sync_file(struct sdbox_sync_context *ctx, + uint32_t seq, uint32_t uid, + enum sdbox_sync_entry_type type) +{ + struct dbox_file *file; + enum modify_type modify_type; + + switch (type) { + case SDBOX_SYNC_ENTRY_TYPE_EXPUNGE: + if (!mail_index_transaction_is_expunged(ctx->trans, seq)) { + mail_index_expunge(ctx->trans, seq); + array_push_back(&ctx->expunged_uids, &uid); + } + break; + case SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT: + case SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT: + /* update flags in the sync transaction, mainly to make + sure that these alt changes get marked as synced + and won't be retried */ + modify_type = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT ? + MODIFY_ADD : MODIFY_REMOVE; + mail_index_update_flags(ctx->trans, seq, modify_type, + (enum mail_flags)DBOX_INDEX_FLAG_ALT); + file = sdbox_file_init(ctx->mbox, uid); + dbox_sync_file_move_if_needed(file, type); + dbox_file_unref(&file); + break; + } +} + +static void sdbox_sync_add(struct sdbox_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec) +{ + uint32_t uid; + enum sdbox_sync_entry_type type; + uint32_t seq, seq1, seq2; + + if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + /* we're interested */ + type = SDBOX_SYNC_ENTRY_TYPE_EXPUNGE; + } else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) { + /* we care only about alt flag changes */ + if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0) + type = SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT; + else if ((sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) != 0) + type = SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT; + else + return; + } else { + /* not interested */ + return; + } + + if (!mail_index_lookup_seq_range(ctx->sync_view, + sync_rec->uid1, sync_rec->uid2, + &seq1, &seq2)) { + /* already expunged everything. nothing to do. */ + return; + } + + for (seq = seq1; seq <= seq2; seq++) { + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + sdbox_sync_file(ctx, seq, uid, type); + } +} + +static int sdbox_sync_index(struct sdbox_sync_context *ctx) +{ + struct mailbox *box = &ctx->mbox->box; + const struct mail_index_header *hdr; + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + + hdr = mail_index_get_header(ctx->sync_view); + if (hdr->uid_validity == 0) { + /* newly created index file */ + if (hdr->next_uid == 1) { + /* could be just a race condition where we opened the + mailbox between mkdir and index creation. fix this + silently. */ + if (sdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0) + return -1; + return 1; + } + mailbox_set_critical(box, + "sdbox: Broken index: missing UIDVALIDITY"); + sdbox_set_mailbox_corrupted(box); + return 0; + } + + /* mark the newly seen messages as recent */ + if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, + hdr->next_uid, &seq1, &seq2)) + mailbox_recent_flags_set_seqs(box, ctx->sync_view, seq1, seq2); + + while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) + sdbox_sync_add(ctx, &sync_rec); + return 1; +} + +static void dbox_sync_file_expunge(struct sdbox_sync_context *ctx, + uint32_t uid) +{ + struct mailbox *box = &ctx->mbox->box; + struct dbox_file *file; + struct sdbox_file *sfile; + int ret; + + file = sdbox_file_init(ctx->mbox, uid); + sfile = (struct sdbox_file *)file; + if (file->storage->attachment_dir != NULL) + ret = sdbox_file_unlink_with_attachments(sfile); + else + ret = dbox_file_unlink(file); + + /* do sync_notify only when the file was unlinked by us */ + if (ret > 0) + mailbox_sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE); + dbox_file_unref(&file); +} + +static void dbox_sync_expunge_files(struct sdbox_sync_context *ctx) +{ + uint32_t uid; + + /* NOTE: Index is no longer locked. Multiple processes may be unlinking + the files at the same time. */ + ctx->mbox->box.tmp_sync_view = ctx->sync_view; + array_foreach_elem(&ctx->expunged_uids, uid) T_BEGIN { + dbox_sync_file_expunge(ctx, uid); + } T_END; + mailbox_sync_notify(&ctx->mbox->box, 0, 0); + ctx->mbox->box.tmp_sync_view = NULL; +} + +static int +sdbox_refresh_header(struct sdbox_mailbox *mbox, bool retry, bool log_error) +{ + struct mail_index_view *view; + struct sdbox_index_header hdr; + bool need_resize; + int ret; + + view = mail_index_view_open(mbox->box.index); + ret = sdbox_read_header(mbox, &hdr, log_error, &need_resize); + mail_index_view_close(&view); + + if (ret < 0 && retry) { + mail_index_refresh(mbox->box.index); + return sdbox_refresh_header(mbox, FALSE, log_error); + } + return ret; +} + +int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags, + struct sdbox_sync_context **ctx_r) +{ + const struct mail_index_header *hdr = + mail_index_get_header(mbox->box.view); + struct sdbox_sync_context *ctx; + enum mail_index_sync_flags sync_flags; + unsigned int i; + int ret; + bool rebuild, force_rebuild; + + force_rebuild = (flags & SDBOX_SYNC_FLAG_FORCE_REBUILD) != 0; + rebuild = force_rebuild || + (hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0 || + mbox->corrupted_rebuild_count != 0 || + sdbox_refresh_header(mbox, TRUE, FALSE) < 0; + + ctx = i_new(struct sdbox_sync_context, 1); + ctx->mbox = mbox; + ctx->flags = flags; + i_array_init(&ctx->expunged_uids, 32); + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if (!rebuild && (flags & SDBOX_SYNC_FLAG_FORCE) == 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + if ((flags & SDBOX_SYNC_FLAG_FSYNC) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC; + /* don't write unnecessary dirty flag updates */ + sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; + + for (i = 0;; i++) { + ret = index_storage_expunged_sync_begin(&mbox->box, + &ctx->index_sync_ctx, &ctx->sync_view, + &ctx->trans, sync_flags); + if (mail_index_reset_fscked(mbox->box.index)) + sdbox_set_mailbox_corrupted(&mbox->box); + if (ret <= 0) { + array_free(&ctx->expunged_uids); + i_free(ctx); + *ctx_r = NULL; + return ret; + } + + if (rebuild) + ret = 0; + else { + if ((ret = sdbox_sync_index(ctx)) > 0) + break; + } + + /* failure. keep the index locked while we're doing a + rebuild. */ + if (ret == 0) { + if (i >= SDBOX_REBUILD_COUNT) { + mailbox_set_critical(&ctx->mbox->box, + "sdbox: Index keeps breaking"); + ret = -1; + } else { + /* do a full resync and try again. */ + rebuild = FALSE; + ret = sdbox_sync_index_rebuild(mbox, + force_rebuild); + } + } + mail_index_sync_rollback(&ctx->index_sync_ctx); + if (ret < 0) { + index_storage_expunging_deinit(&ctx->mbox->box); + array_free(&ctx->expunged_uids); + i_free(ctx); + return -1; + } + } + + *ctx_r = ctx; + return 0; +} + +int sdbox_sync_finish(struct sdbox_sync_context **_ctx, bool success) +{ + struct sdbox_sync_context *ctx = *_ctx; + struct mail_storage *storage = &ctx->mbox->storage->storage.storage; + int ret = success ? 0 : -1; + + *_ctx = NULL; + + if (success) { + mail_index_view_ref(ctx->sync_view); + + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { + mailbox_set_index_error(&ctx->mbox->box); + ret = -1; + } else { + dbox_sync_expunge_files(ctx); + } + mail_index_view_close(&ctx->sync_view); + } else { + mail_index_sync_rollback(&ctx->index_sync_ctx); + } + + if (storage->rebuild_list_index) + ret = mail_storage_list_index_rebuild_and_set_uncorrupted(storage); + + index_storage_expunging_deinit(&ctx->mbox->box); + array_free(&ctx->expunged_uids); + i_free(ctx); + return ret; +} + +int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags) +{ + struct sdbox_sync_context *sync_ctx; + + if (sdbox_sync_begin(mbox, flags, &sync_ctx) < 0) + return -1; + + if (sync_ctx == NULL) + return 0; + return sdbox_sync_finish(&sync_ctx, TRUE); +} + +struct mailbox_sync_context * +sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box); + enum sdbox_sync_flags sdbox_sync_flags = 0; + int ret = 0; + + if (mail_index_reset_fscked(box->index)) + sdbox_set_mailbox_corrupted(box); + if (index_mailbox_want_full_sync(&mbox->box, flags) || + mbox->corrupted_rebuild_count != 0) { + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) + sdbox_sync_flags |= SDBOX_SYNC_FLAG_FORCE_REBUILD; + ret = sdbox_sync(mbox, sdbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +} diff --git a/src/lib-storage/index/dbox-single/sdbox-sync.h b/src/lib-storage/index/dbox-single/sdbox-sync.h new file mode 100644 index 0000000..70306c9 --- /dev/null +++ b/src/lib-storage/index/dbox-single/sdbox-sync.h @@ -0,0 +1,38 @@ +#ifndef SDBOX_SYNC_H +#define SDBOX_SYNC_H + +struct mailbox; +struct sdbox_mailbox; + +enum sdbox_sync_flags { + SDBOX_SYNC_FLAG_FORCE = 0x01, + SDBOX_SYNC_FLAG_FSYNC = 0x02, + SDBOX_SYNC_FLAG_FORCE_REBUILD = 0x04 +}; + +enum sdbox_sync_entry_type { + SDBOX_SYNC_ENTRY_TYPE_EXPUNGE, + SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT, + SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT +}; + +struct sdbox_sync_context { + struct sdbox_mailbox *mbox; + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + enum sdbox_sync_flags flags; + ARRAY_TYPE(uint32_t) expunged_uids; +}; + +int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags, + struct sdbox_sync_context **ctx_r); +int sdbox_sync_finish(struct sdbox_sync_context **ctx, bool success); +int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags); + +int sdbox_sync_index_rebuild(struct sdbox_mailbox *mbox, bool force); + +struct mailbox_sync_context * +sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +#endif diff --git a/src/lib-storage/index/imapc/Makefile.am b/src/lib-storage/index/imapc/Makefile.am new file mode 100644 index 0000000..72ee102 --- /dev/null +++ b/src/lib-storage/index/imapc/Makefile.am @@ -0,0 +1,36 @@ +noinst_LTLIBRARIES = libstorage_imapc.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-imap-client \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/list \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-ssl-iostream + +libstorage_imapc_la_SOURCES = \ + imapc-list.c \ + imapc-mail.c \ + imapc-mail-fetch.c \ + imapc-mailbox.c \ + imapc-save.c \ + imapc-search.c \ + imapc-settings.c \ + imapc-sync.c \ + imapc-storage.c + +headers = \ + imapc-list.h \ + imapc-mail.h \ + imapc-search.h \ + imapc-settings.h \ + imapc-storage.h \ + imapc-sync.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/imapc/Makefile.in b/src/lib-storage/index/imapc/Makefile.in new file mode 100644 index 0000000..4d73707 --- /dev/null +++ b/src/lib-storage/index/imapc/Makefile.in @@ -0,0 +1,861 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/imapc +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_imapc_la_LIBADD = +am_libstorage_imapc_la_OBJECTS = imapc-list.lo imapc-mail.lo \ + imapc-mail-fetch.lo imapc-mailbox.lo imapc-save.lo \ + imapc-search.lo imapc-settings.lo imapc-sync.lo \ + imapc-storage.lo +libstorage_imapc_la_OBJECTS = $(am_libstorage_imapc_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/imapc-list.Plo \ + ./$(DEPDIR)/imapc-mail-fetch.Plo ./$(DEPDIR)/imapc-mail.Plo \ + ./$(DEPDIR)/imapc-mailbox.Plo ./$(DEPDIR)/imapc-save.Plo \ + ./$(DEPDIR)/imapc-search.Plo ./$(DEPDIR)/imapc-settings.Plo \ + ./$(DEPDIR)/imapc-storage.Plo ./$(DEPDIR)/imapc-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_imapc_la_SOURCES) +DIST_SOURCES = $(libstorage_imapc_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_imapc.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-imap-client \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/list \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-ssl-iostream + +libstorage_imapc_la_SOURCES = \ + imapc-list.c \ + imapc-mail.c \ + imapc-mail-fetch.c \ + imapc-mailbox.c \ + imapc-save.c \ + imapc-search.c \ + imapc-settings.c \ + imapc-sync.c \ + imapc-storage.c + +headers = \ + imapc-list.h \ + imapc-mail.h \ + imapc-search.h \ + imapc-settings.h \ + imapc-storage.h \ + imapc-sync.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/imapc/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/imapc/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_imapc.la: $(libstorage_imapc_la_OBJECTS) $(libstorage_imapc_la_DEPENDENCIES) $(EXTRA_libstorage_imapc_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_imapc_la_OBJECTS) $(libstorage_imapc_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-list.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-mail-fetch.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-mailbox.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-search.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/imapc-list.Plo + -rm -f ./$(DEPDIR)/imapc-mail-fetch.Plo + -rm -f ./$(DEPDIR)/imapc-mail.Plo + -rm -f ./$(DEPDIR)/imapc-mailbox.Plo + -rm -f ./$(DEPDIR)/imapc-save.Plo + -rm -f ./$(DEPDIR)/imapc-search.Plo + -rm -f ./$(DEPDIR)/imapc-settings.Plo + -rm -f ./$(DEPDIR)/imapc-storage.Plo + -rm -f ./$(DEPDIR)/imapc-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/imapc-list.Plo + -rm -f ./$(DEPDIR)/imapc-mail-fetch.Plo + -rm -f ./$(DEPDIR)/imapc-mail.Plo + -rm -f ./$(DEPDIR)/imapc-mailbox.Plo + -rm -f ./$(DEPDIR)/imapc-save.Plo + -rm -f ./$(DEPDIR)/imapc-search.Plo + -rm -f ./$(DEPDIR)/imapc-settings.Plo + -rm -f ./$(DEPDIR)/imapc-storage.Plo + -rm -f ./$(DEPDIR)/imapc-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/imapc/imapc-list.c b/src/lib-storage/index/imapc/imapc-list.c new file mode 100644 index 0000000..d987538 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-list.c @@ -0,0 +1,1013 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +/* + There are various different mailbox names here. Here's an example assuming + - imapc_list_prefix = "prefix" + - remote imapc server separator = '/' + - mailbox_list separator = '^' (actually this is currently always the same + as remote separator, but this clarifies the example) + - namespace separator = ':' + - fs_list separator = '.' + - mailbox_list storage_name_escape_char = '+' + - mailbox_list vname_escape_char = '~' + - fs_list storage_name_escape_char = '%' + + remote_name = "prefix/~foo/bar^baz+_%_." + storage_name = "prefix^~foo^bar+5ebaz+2b_%_." + - separator is changed from / to ^ + - conflicting ^ separator in remote_name is escaped as +5e + - storage_name_escape character + is escaped as +2b + vname = "~7efoo:bar.baz+_%_." + - imapc_list_prefix is dropped + - vname_escape_character ~ is escaped into ~7e + - separator is changed from ^ to : + - storage_name_escape_characters are unescaped + fs_name = "prefix.~foo.bar^baz+_%25_%2e" + - this is generated from remote_name + - separator is changed from / to . + - storage_name_escape_character=% and fs_list separator . are escaped +*/ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "imap-arg.h" +#include "imap-match.h" +#include "imap-utf7.h" +#include "mailbox-tree.h" +#include "mailbox-list-subscriptions.h" +#include "imapc-storage.h" +#include "imapc-list.h" + +struct imapc_mailbox_list_iterate_context { + struct mailbox_list_iterate_context ctx; + struct mailbox_tree_context *tree; + struct mailbox_node *ns_root; + + struct mailbox_tree_iterate_context *iter; + struct mailbox_info info; + string_t *special_use; +}; + +static struct { + const char *str; + enum mailbox_info_flags flag; +} imap_list_flags[] = { + { "\\NoSelect", MAILBOX_NOSELECT }, + { "\\NonExistent", MAILBOX_NONEXISTENT }, + { "\\NoInferiors", MAILBOX_NOINFERIORS }, + { "\\Subscribed", MAILBOX_SUBSCRIBED }, + { "\\All", MAILBOX_SPECIALUSE_ALL }, + { "\\Archive", MAILBOX_SPECIALUSE_ARCHIVE }, + { "\\Drafts", MAILBOX_SPECIALUSE_DRAFTS }, + { "\\Flagged", MAILBOX_SPECIALUSE_FLAGGED }, + { "\\Junk", MAILBOX_SPECIALUSE_JUNK }, + { "\\Sent", MAILBOX_SPECIALUSE_SENT }, + { "\\Trash", MAILBOX_SPECIALUSE_TRASH }, + { "\\Important", MAILBOX_SPECIALUSE_IMPORTANT } +}; + +extern struct mailbox_list imapc_mailbox_list; + +static void imapc_list_send_hierarchy_sep_lookup(struct imapc_mailbox_list *list); +static void imapc_untagged_list(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client); +static void imapc_untagged_lsub(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client); + +static struct mailbox_list *imapc_list_alloc(void) +{ + struct imapc_mailbox_list *list; + pool_t pool; + + pool = pool_alloconly_create("imapc mailbox list", 1024); + list = p_new(pool, struct imapc_mailbox_list, 1); + list->list = imapc_mailbox_list; + list->list.pool = pool; + /* separator is set lazily */ + list->mailboxes = mailbox_tree_init('\0'); + mailbox_tree_set_parents_nonexistent(list->mailboxes); + return &list->list; +} + +static int imapc_list_init(struct mailbox_list *_list, const char **error_r) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + + list->set = mail_user_set_get_driver_settings(_list->ns->user->set_info, + _list->ns->user_set, + IMAPC_STORAGE_NAME); + if (imapc_storage_client_create(_list->ns, list->set, _list->mail_set, + &list->client, error_r) < 0) + return -1; + list->client->_list = list; + + imapc_storage_client_register_untagged(list->client, "LIST", + imapc_untagged_list); + imapc_storage_client_register_untagged(list->client, "LSUB", + imapc_untagged_lsub); + imapc_list_send_hierarchy_sep_lookup(list); + return 0; +} + +static void imapc_list_deinit(struct mailbox_list *_list) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + + /* make sure all pending commands are aborted before anything is + deinitialized */ + if (list->client != NULL) { + list->client->destroying = TRUE; + imapc_client_logout(list->client->client); + imapc_storage_client_unref(&list->client); + } + if (list->index_list != NULL) + mailbox_list_destroy(&list->index_list); + mailbox_tree_deinit(&list->mailboxes); + if (list->tmp_subscriptions != NULL) + mailbox_tree_deinit(&list->tmp_subscriptions); + pool_unref(&list->list.pool); +} + +static void +imapc_list_copy_error_from_reply(struct imapc_mailbox_list *list, + enum mail_error default_error, + const struct imapc_command_reply *reply) +{ + enum mail_error error; + + if (imapc_resp_text_code_parse(reply->resp_text_key, &error)) { + mailbox_list_set_error(&list->list, error, + reply->text_without_resp); + } else { + mailbox_list_set_error(&list->list, default_error, + reply->text_without_resp); + } +} + +static void imapc_list_simple_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_simple_context *ctx = context; + + if (reply->state == IMAPC_COMMAND_STATE_OK) + ctx->ret = 0; + else if (reply->state == IMAPC_COMMAND_STATE_NO) { + imapc_list_copy_error_from_reply(ctx->client->_list, + MAIL_ERROR_PARAMS, reply); + ctx->ret = -1; + } else if (imapc_storage_client_handle_auth_failure(ctx->client)) { + ctx->ret = -1; + } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + mailbox_list_set_internal_error(&ctx->client->_list->list); + ctx->ret = -1; + } else { + mailbox_list_set_critical(&ctx->client->_list->list, + "imapc: Command failed: %s", reply->text_full); + ctx->ret = -1; + } + imapc_client_stop(ctx->client->client); +} + +static bool +imap_list_flag_parse(const char *str, enum mailbox_info_flags *flag_r) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(imap_list_flags); i++) { + if (strcasecmp(str, imap_list_flags[i].str) == 0) { + *flag_r = imap_list_flags[i].flag; + return TRUE; + } + } + return FALSE; +} + +static const char * +imapc_list_remote_to_storage_name(struct imapc_mailbox_list *list, + const char *remote_name) +{ + /* typically mailbox_list_escape_name() is used to escape vname into + a list name. but we want to convert remote IMAP name to a list name, + so we need to use the remote IMAP separator. */ + return mailbox_list_escape_name_params(remote_name, "", + list->root_sep, + mailbox_list_get_hierarchy_sep(&list->list), + list->list.set.storage_name_escape_char, ""); +} + +static const char * +imapc_list_remote_to_vname(struct imapc_mailbox_list *list, + const char *remote_name) +{ + return mailbox_list_get_vname(&list->list, + imapc_list_remote_to_storage_name(list, remote_name)); +} + +const char * +imapc_list_storage_to_remote_name(struct imapc_mailbox_list *list, + const char *storage_name) +{ + return mailbox_list_unescape_name_params(storage_name, "", list->root_sep, + mailbox_list_get_hierarchy_sep(&list->list), + list->list.set.storage_name_escape_char); +} + +static struct mailbox_node * +imapc_list_update_tree(struct imapc_mailbox_list *list, + struct mailbox_tree_context *tree, + const struct imap_arg *args) +{ + struct mailbox_node *node; + const struct imap_arg *flags; + const char *remote_name, *flag; + enum mailbox_info_flags info_flag, info_flags = 0; + bool created; + + if (!imap_arg_get_list(&args[0], &flags) || + args[1].type == IMAP_ARG_EOL || + !imap_arg_get_astring(&args[2], &remote_name)) + return NULL; + + while (imap_arg_get_atom(flags, &flag)) { + if (imap_list_flag_parse(flag, &info_flag)) + info_flags |= info_flag; + flags++; + } + + T_BEGIN { + const char *vname = + imapc_list_remote_to_vname(list, remote_name); + + if ((info_flags & MAILBOX_NONEXISTENT) != 0) + node = mailbox_tree_lookup(tree, vname); + else + node = mailbox_tree_get(tree, vname, &created); + } T_END; + if (node != NULL) + node->flags = info_flags; + return node; +} + +static void imapc_untagged_list(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client) +{ + struct imapc_mailbox_list *list = client->_list; + const struct imap_arg *args = reply->args; + const char *sep, *remote_name; + + if (list->root_sep == '\0') { + /* we haven't asked for the separator yet. + lets see if this is the reply for its request. */ + if (args[0].type == IMAP_ARG_EOL || + !imap_arg_get_nstring(&args[1], &sep) || + !imap_arg_get_astring(&args[2], &remote_name)) + return; + + /* we can't handle NIL separator yet */ + list->root_sep = sep == NULL ? '/' : sep[0]; + mailbox_tree_set_separator(list->mailboxes, list->root_sep); + } else { + (void)imapc_list_update_tree(list, list->mailboxes, args); + } +} + +static void imapc_untagged_lsub(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client) +{ + struct imapc_mailbox_list *list = client->_list; + const struct imap_arg *args = reply->args; + struct mailbox_node *node; + + if (list->root_sep == '\0') { + /* we haven't asked for the separator yet */ + return; + } + node = imapc_list_update_tree(list, list->tmp_subscriptions != NULL ? + list->tmp_subscriptions : + list->list.subscriptions, args); + if (node != NULL) { + if ((node->flags & MAILBOX_NOSELECT) == 0) + node->flags |= MAILBOX_SUBSCRIBED; + else { + /* LSUB \Noselect means that the mailbox isn't + subscribed, but it has children that are */ + node->flags &= ENUM_NEGATE(MAILBOX_NOSELECT); + } + } +} + +static void imapc_list_sep_verify(struct imapc_mailbox_list *list) +{ + const char *imapc_list_prefix = list->set->imapc_list_prefix; + + if (list->root_sep == '\0') { + mailbox_list_set_critical(&list->list, + "imapc: LIST didn't return hierarchy separator"); + } else if (imapc_list_prefix[0] != '\0' && + imapc_list_prefix[strlen(imapc_list_prefix)-1] == list->root_sep) { + mailbox_list_set_critical(&list->list, + "imapc_list_prefix must not end with hierarchy separator"); + } +} + +static void imapc_storage_sep_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_mailbox_list *list = context; + + list->root_sep_pending = FALSE; + if (reply->state == IMAPC_COMMAND_STATE_OK) + imapc_list_sep_verify(list); + else if (reply->state == IMAPC_COMMAND_STATE_NO) + imapc_list_copy_error_from_reply(list, MAIL_ERROR_PARAMS, reply); + else if (imapc_storage_client_handle_auth_failure(list->client)) + ; + else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) + mailbox_list_set_internal_error(&list->list); + else if (!list->list.ns->user->deinitializing) { + mailbox_list_set_critical(&list->list, + "imapc: Command failed: %s", reply->text_full); + } + imapc_client_stop(list->client->client); +} + +static void imapc_list_send_hierarchy_sep_lookup(struct imapc_mailbox_list *list) +{ + struct imapc_command *cmd; + + if (list->root_sep_pending) + return; + list->root_sep_pending = TRUE; + + cmd = imapc_client_cmd(list->client->client, + imapc_storage_sep_callback, list); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_send(cmd, "LIST \"\" \"\""); +} + +int imapc_list_try_get_root_sep(struct imapc_mailbox_list *list, char *sep_r) +{ + if (list->root_sep == '\0') { + if (imapc_storage_client_handle_auth_failure(list->client)) + return -1; + imapc_list_send_hierarchy_sep_lookup(list); + while (list->root_sep_pending) + imapc_client_run(list->client->client); + if (list->root_sep == '\0') + return -1; + } + *sep_r = list->root_sep; + return 0; +} + +static char imapc_list_get_hierarchy_sep(struct mailbox_list *_list) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + char sep; + + if (imapc_list_try_get_root_sep(list, &sep) < 0) { + /* we can't really return a failure here. just return a common + separator and fail all the future list operations. */ + return '/'; + } + return sep; +} + +static const char * +imapc_list_get_storage_name(struct mailbox_list *_list, const char *vname) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + const char *prefix = list->set->imapc_list_prefix; + const char *storage_name; + + storage_name = mailbox_list_default_get_storage_name(_list, vname); + if (*prefix != '\0' && strcasecmp(storage_name, "INBOX") != 0) { + storage_name = storage_name[0] == '\0' ? prefix : + t_strdup_printf("%s%c%s", prefix, + mailbox_list_get_hierarchy_sep(_list), + storage_name); + } + return storage_name; +} + +static const char * +imapc_list_get_vname(struct mailbox_list *_list, const char *storage_name) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + const char *prefix = list->set->imapc_list_prefix; + size_t prefix_len; + + if (*storage_name == '\0') { + /* ACL plugin does these lookups */ + } else if (*prefix != '\0' && strcasecmp(storage_name, "INBOX") != 0) { + prefix_len = strlen(prefix); + i_assert(str_begins(storage_name, prefix)); + storage_name += prefix_len; + if (storage_name[0] == '\0') { + /* we're looking up the prefix itself */ + } else { + i_assert(storage_name[0] == + mailbox_list_get_hierarchy_sep(_list)); + storage_name++; + } + } + return mailbox_list_default_get_vname(_list, storage_name); +} + +static struct mailbox_list *imapc_list_get_fs(struct imapc_mailbox_list *list) +{ + struct mailbox_list_settings list_set; + const char *error, *dir; + + dir = list->list.set.index_dir; + if (dir == NULL) + dir = list->list.set.root_dir; + + if (dir == NULL || dir[0] == '\0') { + /* indexes disabled */ + } else if (list->index_list == NULL && !list->index_list_failed) { + mailbox_list_settings_init_defaults(&list_set); + list_set.layout = MAILBOX_LIST_NAME_MAILDIRPLUSPLUS; + list_set.root_dir = dir; + list_set.index_pvt_dir = p_strdup_empty(list->list.pool, list->list.set.index_pvt_dir); + /* Filesystem needs to be able to store any kind of a mailbox + name. */ + list_set.storage_name_escape_char = + IMAPC_LIST_FS_NAME_ESCAPE_CHAR; + + if (mailbox_list_create(list_set.layout, list->list.ns, + &list_set, MAILBOX_LIST_FLAG_SECONDARY, + &list->index_list, &error) < 0) { + i_error("imapc: Couldn't create %s mailbox list: %s", + list_set.layout, error); + list->index_list_failed = TRUE; + } + } + return list->index_list; +} + +static const char * +imapc_list_storage_to_fs_name(struct imapc_mailbox_list *list, + const char *storage_name) +{ + struct mailbox_list *fs_list = imapc_list_get_fs(list); + const char *remote_name; + + if (storage_name == NULL) + return NULL; + + remote_name = imapc_list_storage_to_remote_name(list, storage_name); + return mailbox_list_escape_name_params(remote_name, "", + list->root_sep, + mailbox_list_get_hierarchy_sep(fs_list), + fs_list->set.storage_name_escape_char, ""); +} + +static const char * +imapc_list_fs_to_storage_name(struct imapc_mailbox_list *list, + const char *fs_name) +{ + struct mailbox_list *fs_list = imapc_list_get_fs(list); + const char *remote_name; + + if (fs_name == NULL) + return NULL; + + remote_name = mailbox_list_unescape_name_params(fs_name, "", + list->root_sep, + mailbox_list_get_hierarchy_sep(fs_list), + fs_list->set.storage_name_escape_char); + return imapc_list_remote_to_storage_name(list, remote_name); +} + +static int +imapc_list_get_path(struct mailbox_list *_list, const char *name, + enum mailbox_list_path_type type, const char **path_r) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + struct mailbox_list *fs_list = imapc_list_get_fs(list); + const char *fs_name; + + if (fs_list != NULL) { + fs_name = imapc_list_storage_to_fs_name(list, name); + return mailbox_list_get_path(fs_list, fs_name, type, path_r); + } else { + *path_r = NULL; + return 0; + } +} + +static const char * +imapc_list_get_temp_prefix(struct mailbox_list *_list, bool global) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + struct mailbox_list *fs_list = imapc_list_get_fs(list); + + if (fs_list != NULL) { + return global ? + mailbox_list_get_global_temp_prefix(fs_list) : + mailbox_list_get_temp_prefix(fs_list); + } else { + i_panic("imapc: Can't return a temp prefix for '%s'", + _list->ns->prefix); + } +} + +static const char * +imapc_list_join_refpattern(struct mailbox_list *list ATTR_UNUSED, + const char *ref, const char *pattern) +{ + return t_strconcat(ref, pattern, NULL); +} + +static struct imapc_command * +imapc_list_simple_context_init(struct imapc_simple_context *ctx, + struct imapc_mailbox_list *list) +{ + imapc_simple_context_init(ctx, list->client); + return imapc_client_cmd(list->client->client, + imapc_list_simple_callback, ctx); +} + +static void imapc_list_delete_unused_indexes(struct imapc_mailbox_list *list) +{ + struct mailbox_list *fs_list = imapc_list_get_fs(list); + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + const char *fs_name, *storage_name, *vname; + + if (fs_list == NULL) + return; + + iter = mailbox_list_iter_init(fs_list, "*", + MAILBOX_LIST_ITER_RAW_LIST | + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + fs_name = mailbox_list_get_storage_name(fs_list, info->vname); + storage_name = imapc_list_fs_to_storage_name(list, fs_name); + vname = mailbox_list_get_vname(&list->list, storage_name); + + /* list->mailboxes contains proper vnames. fs_vname */ + if (mailbox_tree_lookup(list->mailboxes, vname) == NULL) + (void)fs_list->v.delete_mailbox(fs_list, fs_name); + } T_END; + (void)mailbox_list_iter_deinit(&iter); +} + +static int imapc_list_refresh(struct imapc_mailbox_list *list) +{ + struct imapc_command *cmd; + struct imapc_simple_context ctx; + struct mailbox_node *node; + const char *pattern; + char sep; + + if (imapc_list_try_get_root_sep(list, &sep) < 0) + return -1; + if (list->refreshed_mailboxes) + return 0; + + if (*list->set->imapc_list_prefix == '\0') + pattern = "*"; + else { + /* list "prefix*" instead of "prefix.*". this may return a bit + more than we want, but we're also interested in the flags + of the prefix itself. */ + pattern = t_strdup_printf("%s*", list->set->imapc_list_prefix); + } + + cmd = imapc_list_simple_context_init(&ctx, list); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_sendf(cmd, "LIST \"\" %s", pattern); + mailbox_tree_deinit(&list->mailboxes); + list->mailboxes = mailbox_tree_init(mail_namespace_get_sep(list->list.ns)); + mailbox_tree_set_parents_nonexistent(list->mailboxes); + imapc_simple_run(&ctx, &cmd); + + if ((list->list.ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { + /* INBOX always exists in IMAP server. since this namespace is + marked with inbox=yes, show the INBOX even if + imapc_list_prefix doesn't match it */ + bool created; + node = mailbox_tree_get(list->mailboxes, "INBOX", &created); + if (*list->set->imapc_list_prefix != '\0') { + /* this listing didn't include the INBOX itself, but + might have included its children. make sure there + aren't any extra flags in it (especially + \NonExistent) */ + node->flags &= MAILBOX_CHILDREN; + } + } + + if (ctx.ret == 0) { + list->refreshed_mailboxes = TRUE; + list->refreshed_mailboxes_recently = TRUE; + list->last_refreshed_mailboxes = ioloop_time; + imapc_list_delete_unused_indexes(list); + } + return ctx.ret; +} + +static void +imapc_list_build_match_tree(struct imapc_mailbox_list_iterate_context *ctx) +{ + struct imapc_mailbox_list *list = + (struct imapc_mailbox_list *)ctx->ctx.list; + struct mailbox_list_iter_update_context update_ctx; + struct mailbox_tree_iterate_context *iter; + struct mailbox_node *node; + const char *vname; + + i_zero(&update_ctx); + update_ctx.iter_ctx = &ctx->ctx; + update_ctx.tree_ctx = ctx->tree; + update_ctx.glob = ctx->ctx.glob; + update_ctx.match_parents = TRUE; + + iter = mailbox_tree_iterate_init(list->mailboxes, NULL, 0); + while ((node = mailbox_tree_iterate_next(iter, &vname)) != NULL) { + update_ctx.leaf_flags = node->flags; + mailbox_list_iter_update(&update_ctx, vname); + } + mailbox_tree_iterate_deinit(&iter); +} + +static struct mailbox_list_iterate_context * +imapc_list_iter_init(struct mailbox_list *_list, const char *const *patterns, + enum mailbox_list_iter_flags flags) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + struct mailbox_list_iterate_context *_ctx; + struct imapc_mailbox_list_iterate_context *ctx; + pool_t pool; + const char *ns_root_name; + char ns_sep; + int ret = 0; + + if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0 || + (flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) == 0) + ret = imapc_list_refresh(list); + + list->iter_count++; + + if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0) { + /* we're listing only subscriptions. just use the cached + subscriptions list. */ + _ctx = mailbox_list_subscriptions_iter_init(_list, patterns, + flags); + if (ret < 0) + _ctx->failed = TRUE; + return _ctx; + } + + /* if we've already failed, make sure we don't call + mailbox_list_get_hierarchy_sep(), since it clears the error */ + ns_sep = ret < 0 ? '/' : mail_namespace_get_sep(_list->ns); + + pool = pool_alloconly_create("mailbox list imapc iter", 1024); + ctx = p_new(pool, struct imapc_mailbox_list_iterate_context, 1); + ctx->ctx.pool = pool; + ctx->ctx.list = _list; + ctx->ctx.flags = flags; + ctx->ctx.glob = imap_match_init_multiple(pool, patterns, FALSE, ns_sep); + array_create(&ctx->ctx.module_contexts, pool, sizeof(void *), 5); + + ctx->info.ns = _list->ns; + + ctx->tree = mailbox_tree_init(ns_sep); + mailbox_tree_set_parents_nonexistent(ctx->tree); + if (ret == 0) + imapc_list_build_match_tree(ctx); + + if (list->list.ns->prefix_len > 0) { + ns_root_name = t_strndup(_list->ns->prefix, + _list->ns->prefix_len - 1); + ctx->ns_root = mailbox_tree_lookup(ctx->tree, ns_root_name); + } + + ctx->iter = mailbox_tree_iterate_init(ctx->tree, NULL, 0); + if (ret < 0) + ctx->ctx.failed = TRUE; + return &ctx->ctx; +} + +static void +imapc_list_write_special_use(struct imapc_mailbox_list_iterate_context *ctx, + struct mailbox_node *node) +{ + unsigned int i; + + if (ctx->special_use == NULL) + ctx->special_use = str_new(ctx->ctx.pool, 64); + str_truncate(ctx->special_use, 0); + + for (i = 0; i < N_ELEMENTS(imap_list_flags); i++) { + if ((node->flags & imap_list_flags[i].flag) != 0 && + (node->flags & MAILBOX_SPECIALUSE_MASK) != 0) { + str_append(ctx->special_use, imap_list_flags[i].str); + str_append_c(ctx->special_use, ' '); + } + } + + if (str_len(ctx->special_use) > 0) { + str_truncate(ctx->special_use, str_len(ctx->special_use) - 1); + ctx->info.special_use = str_c(ctx->special_use); + } else { + ctx->info.special_use = NULL; + } +} + +static bool +imapc_list_is_ns_root(struct imapc_mailbox_list_iterate_context *ctx, + struct mailbox_node *node) +{ + struct mailbox_node *root_node = ctx->ns_root; + + while (root_node != NULL) { + if (node == root_node) + return TRUE; + root_node = root_node->parent; + } + return FALSE; +} + +static const struct mailbox_info * +imapc_list_iter_next(struct mailbox_list_iterate_context *_ctx) +{ + struct imapc_mailbox_list_iterate_context *ctx = + (struct imapc_mailbox_list_iterate_context *)_ctx; + struct imapc_mailbox_list *list = + (struct imapc_mailbox_list *)_ctx->list; + struct mailbox_node *node; + const char *vname; + + if (_ctx->failed) + return NULL; + + if ((_ctx->flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0) + return mailbox_list_subscriptions_iter_next(_ctx); + + do { + node = mailbox_tree_iterate_next(ctx->iter, &vname); + if (node == NULL) + return mailbox_list_iter_default_next(_ctx); + } while ((node->flags & MAILBOX_MATCHED) == 0 || + (imapc_list_is_ns_root(ctx, node) && + (strcasecmp(vname, "INBOX") != 0 || + (ctx->info.ns->flags & NAMESPACE_FLAG_INBOX_ANY) == 0))); + + if (ctx->info.ns->prefix_len > 0 && + strcasecmp(vname, "INBOX") != 0 && + strncmp(vname, ctx->info.ns->prefix, ctx->info.ns->prefix_len-1) == 0 && + vname[ctx->info.ns->prefix_len] == '\0' && + list->set->imapc_list_prefix[0] == '\0') { + /* don't return "" name */ + return imapc_list_iter_next(_ctx); + } + + ctx->info.vname = vname; + ctx->info.flags = node->flags; + if ((_ctx->list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { + /* we're iterating the INBOX namespace. pass through the + SPECIAL-USE flags if they exist. */ + imapc_list_write_special_use(ctx, node); + } else { + ctx->info.special_use = NULL; + } + return &ctx->info; +} + +static int imapc_list_iter_deinit(struct mailbox_list_iterate_context *_ctx) +{ + struct imapc_mailbox_list_iterate_context *ctx = + (struct imapc_mailbox_list_iterate_context *)_ctx; + struct imapc_mailbox_list *list = + (struct imapc_mailbox_list *)_ctx->list; + int ret = _ctx->failed ? -1 : 0; + + i_assert(list->iter_count > 0); + + if (--list->iter_count == 0) { + list->refreshed_mailboxes = FALSE; + list->refreshed_subscriptions = FALSE; + } + + if ((_ctx->flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0) + return mailbox_list_subscriptions_iter_deinit(_ctx); + + mailbox_tree_iterate_deinit(&ctx->iter); + mailbox_tree_deinit(&ctx->tree); + pool_unref(&_ctx->pool); + return ret; +} + +static int +imapc_list_subscriptions_refresh(struct mailbox_list *_src_list, + struct mailbox_list *dest_list) +{ + struct imapc_mailbox_list *src_list = + (struct imapc_mailbox_list *)_src_list; + struct imapc_simple_context ctx; + struct imapc_command *cmd; + const char *pattern; + char list_sep, dest_sep = mail_namespace_get_sep(dest_list->ns); + + i_assert(src_list->tmp_subscriptions == NULL); + + if (imapc_list_try_get_root_sep(src_list, &list_sep) < 0) + return -1; + + if (src_list->refreshed_subscriptions) { + if (dest_list->subscriptions == NULL) + dest_list->subscriptions = mailbox_tree_init(dest_sep); + return 0; + } + + src_list->tmp_subscriptions = + mailbox_tree_init(mail_namespace_get_sep(_src_list->ns)); + + cmd = imapc_list_simple_context_init(&ctx, src_list); + if (*src_list->set->imapc_list_prefix == '\0') + pattern = "*"; + else + pattern = t_strdup_printf("%s*", src_list->set->imapc_list_prefix); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_sendf(cmd, "LSUB \"\" %s", pattern); + imapc_simple_run(&ctx, &cmd); + + if (ctx.ret < 0) + return -1; + + /* replace subscriptions tree in destination */ + if (dest_list->subscriptions != NULL) + mailbox_tree_deinit(&dest_list->subscriptions); + dest_list->subscriptions = src_list->tmp_subscriptions; + src_list->tmp_subscriptions = NULL; + mailbox_tree_set_separator(dest_list->subscriptions, dest_sep); + + src_list->refreshed_subscriptions = TRUE; + return 0; +} + +static int imapc_list_set_subscribed(struct mailbox_list *_list, + const char *name, bool set) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + struct imapc_command *cmd; + struct imapc_simple_context ctx; + + cmd = imapc_list_simple_context_init(&ctx, list); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_sendf(cmd, set ? "SUBSCRIBE %s" : "UNSUBSCRIBE %s", + imapc_list_storage_to_remote_name(list, name)); + imapc_simple_run(&ctx, &cmd); + return ctx.ret; +} + +static int +imapc_list_delete_mailbox(struct mailbox_list *_list, const char *name) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + struct mailbox_list *fs_list = imapc_list_get_fs(list); + enum imapc_capability capa; + struct imapc_command *cmd; + struct imapc_simple_context ctx; + + if (imapc_storage_client_handle_auth_failure(list->client)) + return -1; + if (imapc_client_get_capabilities(list->client->client, &capa) < 0) + return -1; + + cmd = imapc_list_simple_context_init(&ctx, list); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + if (!imapc_command_connection_is_selected(cmd)) + imapc_command_abort(&cmd); + else { + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT); + if ((capa & IMAPC_CAPABILITY_UNSELECT) != 0) + imapc_command_sendf(cmd, "UNSELECT"); + else + imapc_command_sendf(cmd, "SELECT \"~~~\""); + imapc_simple_run(&ctx, &cmd); + } + + cmd = imapc_list_simple_context_init(&ctx, list); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_sendf(cmd, "DELETE %s", imapc_list_storage_to_remote_name(list, name)); + imapc_simple_run(&ctx, &cmd); + + if (fs_list != NULL && ctx.ret == 0) { + const char *fs_name = imapc_list_storage_to_fs_name(list, name); + (void)fs_list->v.delete_mailbox(fs_list, fs_name); + } + return ctx.ret; +} + +static int +imapc_list_delete_dir(struct mailbox_list *_list, const char *name) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + struct mailbox_list *fs_list = imapc_list_get_fs(list); + + if (fs_list != NULL) { + const char *fs_name = imapc_list_storage_to_fs_name(list, name); + (void)mailbox_list_delete_dir(fs_list, fs_name); + } + return 0; +} + +static int +imapc_list_delete_symlink(struct mailbox_list *list, + const char *name ATTR_UNUSED) +{ + mailbox_list_set_error(list, MAIL_ERROR_NOTPOSSIBLE, "Not supported"); + return -1; +} + +static int +imapc_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname, + struct mailbox_list *newlist, const char *newname) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)oldlist; + struct mailbox_list *fs_list = imapc_list_get_fs(list); + struct imapc_command *cmd; + struct imapc_simple_context ctx; + + if (oldlist != newlist) { + mailbox_list_set_error(oldlist, MAIL_ERROR_NOTPOSSIBLE, + "Can't rename mailboxes across storages."); + return -1; + } + + cmd = imapc_list_simple_context_init(&ctx, list); + imapc_command_sendf(cmd, "RENAME %s %s", + imapc_list_storage_to_remote_name(list, oldname), + imapc_list_storage_to_remote_name(list, newname)); + imapc_simple_run(&ctx, &cmd); + if (ctx.ret == 0 && fs_list != NULL && oldlist == newlist) { + const char *old_fs_name = + imapc_list_storage_to_fs_name(list, oldname); + const char *new_fs_name = + imapc_list_storage_to_fs_name(list, newname); + (void)fs_list->v.rename_mailbox(fs_list, old_fs_name, + fs_list, new_fs_name); + } + return ctx.ret; +} + +int imapc_list_get_mailbox_flags(struct mailbox_list *_list, const char *name, + enum mailbox_info_flags *flags_r) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + struct mailbox_node *node; + const char *vname; + + vname = mailbox_list_get_vname(_list, name); + if (!list->refreshed_mailboxes_recently) { + if (imapc_list_refresh(list) < 0) + return -1; + i_assert(list->refreshed_mailboxes_recently); + } + + if (list->mailboxes == NULL) { + /* imapc list isn't used, but e.g. LAYOUT=none */ + *flags_r = 0; + return 0; + } + node = mailbox_tree_lookup(list->mailboxes, vname); + if (node == NULL) + *flags_r = MAILBOX_NONEXISTENT; + else + *flags_r = node->flags; + return 0; +} + +struct mailbox_list imapc_mailbox_list = { + .name = MAILBOX_LIST_NAME_IMAPC, + .props = MAILBOX_LIST_PROP_NO_ROOT | MAILBOX_LIST_PROP_AUTOCREATE_DIRS | + MAILBOX_LIST_PROP_NO_LIST_INDEX, + .mailbox_name_max_length = MAILBOX_LIST_NAME_MAX_LENGTH, + + .v = { + .alloc = imapc_list_alloc, + .init = imapc_list_init, + .deinit = imapc_list_deinit, + .get_hierarchy_sep = imapc_list_get_hierarchy_sep, + .get_vname = imapc_list_get_vname, + .get_storage_name = imapc_list_get_storage_name, + .get_path = imapc_list_get_path, + .get_temp_prefix = imapc_list_get_temp_prefix, + .join_refpattern = imapc_list_join_refpattern, + .iter_init = imapc_list_iter_init, + .iter_next = imapc_list_iter_next, + .iter_deinit = imapc_list_iter_deinit, + .subscriptions_refresh = imapc_list_subscriptions_refresh, + .set_subscribed = imapc_list_set_subscribed, + .delete_mailbox = imapc_list_delete_mailbox, + .delete_dir = imapc_list_delete_dir, + .delete_symlink = imapc_list_delete_symlink, + .rename_mailbox = imapc_list_rename_mailbox, + } +}; diff --git a/src/lib-storage/index/imapc/imapc-list.h b/src/lib-storage/index/imapc/imapc-list.h new file mode 100644 index 0000000..11df6b0 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-list.h @@ -0,0 +1,41 @@ +#ifndef IMAPC_LIST_H +#define IMAPC_LIST_H + +struct imap_arg; + +#include "mailbox-list-private.h" + +#define MAILBOX_LIST_NAME_IMAPC "imapc" + +struct imapc_mailbox_list { + struct mailbox_list list; + const struct imapc_settings *set; + struct imapc_storage_client *client; + struct mailbox_list *index_list; + + /* mailboxes are stored as vnames */ + struct mailbox_tree_context *mailboxes, *tmp_subscriptions; + char root_sep; + time_t last_refreshed_mailboxes; + + unsigned int iter_count; + + /* mailboxes/subscriptions are fully refreshed only during + mailbox list iteration. */ + bool refreshed_subscriptions:1; + bool refreshed_mailboxes:1; + /* mailbox list's "recently refreshed" state is reset by syncing a + mailbox. mainly we use this to cache mailboxes' existence to avoid + issuing a LIST command every time. */ + bool refreshed_mailboxes_recently:1; + bool index_list_failed:1; + bool root_sep_pending:1; +}; + +int imapc_list_get_mailbox_flags(struct mailbox_list *list, const char *name, + enum mailbox_info_flags *flags_r); +int imapc_list_try_get_root_sep(struct imapc_mailbox_list *list, char *sep_r); +const char *imapc_list_storage_to_remote_name(struct imapc_mailbox_list *list, + const char *storage_name); + +#endif diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c new file mode 100644 index 0000000..1b0eee7 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c @@ -0,0 +1,911 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-concat.h" +#include "istream-header-filter.h" +#include "message-header-parser.h" +#include "imap-arg.h" +#include "imap-util.h" +#include "imap-date.h" +#include "imap-quote.h" +#include "imap-bodystructure.h" +#include "imap-resp-code.h" +#include "imapc-mail.h" +#include "imapc-storage.h" + +static void imapc_mail_set_failure(struct imapc_mail *mail, + const struct imapc_command_reply *reply) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box); + + mail->last_fetch_reply = p_strdup(mail->imail.mail.pool, reply->text_full); + + switch (reply->state) { + case IMAPC_COMMAND_STATE_OK: + break; + case IMAPC_COMMAND_STATE_NO: + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS)) { + /* fetch-fix-broken-mails feature disabled - + fail any mails with missing replies */ + break; + } + if (reply->resp_text_key != NULL && + (strcasecmp(reply->resp_text_key, IMAP_RESP_CODE_SERVERBUG) == 0 || + strcasecmp(reply->resp_text_key, IMAP_RESP_CODE_LIMIT) == 0)) { + /* this is a temporary error, retrying should work. + Yahoo sends * BYE + + NO [LIMIT] UID FETCH Rate limit hit. */ + } else { + /* hopefully this is a permanent failure */ + mail->fetch_ignore_if_missing = TRUE; + } + break; + case IMAPC_COMMAND_STATE_BAD: + case IMAPC_COMMAND_STATE_DISCONNECTED: + case IMAPC_COMMAND_STATE_AUTH_FAILED: + mail->fetch_failed = TRUE; + break; + } +} + +static void +imapc_mail_fetch_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_fetch_request *request = context; + struct imapc_fetch_request *const *requests; + struct imapc_mail *mail; + struct imapc_mailbox *mbox = NULL; + unsigned int i, count; + + array_foreach_elem(&request->mails, mail) { + i_assert(mail->fetch_count > 0); + imapc_mail_set_failure(mail, reply); + if (--mail->fetch_count == 0) + mail->fetching_fields = 0; + mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box); + } + i_assert(mbox != NULL); + + requests = array_get(&mbox->fetch_requests, &count); + for (i = 0; i < count; i++) { + if (requests[i] == request) { + array_delete(&mbox->fetch_requests, i, 1); + break; + } + } + i_assert(i < count); + + array_free(&request->mails); + i_free(request); + + if (reply->state == IMAPC_COMMAND_STATE_OK) + ; + else if (reply->state == IMAPC_COMMAND_STATE_NO) { + imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS, + reply); + } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + /* The disconnection message was already logged */ + mail_storage_set_internal_error(&mbox->storage->storage); + } else { + mailbox_set_critical(&mbox->box, + "imapc: Mail FETCH failed: %s", reply->text_full); + } + imapc_client_stop(mbox->storage->client->client); +} + +static bool +headers_have_subset(const char *const *superset, const char *const *subset) +{ + unsigned int i; + + if (superset == NULL) + return FALSE; + if (subset != NULL) { + for (i = 0; subset[i] != NULL; i++) { + if (!str_array_icase_find(superset, subset[i])) + return FALSE; + } + } + return TRUE; +} + +static const char *const * +headers_merge(pool_t pool, const char *const *h1, const char *const *h2) +{ + ARRAY_TYPE(const_string) headers; + const char *value; + unsigned int i; + + p_array_init(&headers, pool, 16); + if (h1 != NULL) { + for (i = 0; h1[i] != NULL; i++) { + value = p_strdup(pool, h1[i]); + array_push_back(&headers, &value); + } + } + if (h2 != NULL) { + for (i = 0; h2[i] != NULL; i++) { + if (h1 == NULL || !str_array_icase_find(h1, h2[i])) { + value = p_strdup(pool, h2[i]); + array_push_back(&headers, &value); + } + } + } + array_append_zero(&headers); + return array_front(&headers); +} + +static bool +imapc_mail_try_merge_fetch(struct imapc_mailbox *mbox, string_t *str) +{ + const char *s1 = str_c(str); + const char *s2 = str_c(mbox->pending_fetch_cmd); + const char *p1, *p2; + + i_assert(str_begins(s1, "UID FETCH ")); + i_assert(str_begins(s2, "UID FETCH ")); + + /* skip over UID range */ + p1 = strchr(s1+10, ' '); + p2 = strchr(s2+10, ' '); + + if (null_strcmp(p1, p2) != 0) + return FALSE; + /* append the new UID to the pending FETCH UID range */ + str_truncate(str, p1-s1); + str_insert(mbox->pending_fetch_cmd, p2-s2, ","); + str_insert(mbox->pending_fetch_cmd, p2-s2+1, str_c(str) + 10); + return TRUE; +} + +static void +imapc_mail_delayed_send_or_merge(struct imapc_mail *mail, string_t *str) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box); + + if (mbox->pending_fetch_request != NULL && + !imapc_mail_try_merge_fetch(mbox, str)) { + /* send the previous FETCH and create a new one */ + imapc_mail_fetch_flush(mbox); + } + if (mbox->pending_fetch_request == NULL) { + mbox->pending_fetch_request = + i_new(struct imapc_fetch_request, 1); + i_array_init(&mbox->pending_fetch_request->mails, 4); + i_assert(mbox->pending_fetch_cmd->used == 0); + str_append_str(mbox->pending_fetch_cmd, str); + } + array_push_back(&mbox->pending_fetch_request->mails, &mail); + + if (mbox->to_pending_fetch_send == NULL && + array_count(&mbox->pending_fetch_request->mails) > + mbox->box.storage->set->mail_prefetch_count) { + /* we're now prefetching the maximum number of mails. this + most likely means that we need to flush out the command now + before sending anything else. delay it a little bit though + in case the sending code doesn't actually use + mail_prefetch_count and wants to fetch more. + + note that we don't want to add this timeout too early, + because we want to optimize the maximum number of messages + placed into a single FETCH. even without timeout the command + gets flushed by imapc_mail_fetch() call. */ + mbox->to_pending_fetch_send = + timeout_add_short(0, imapc_mail_fetch_flush, mbox); + } +} + +static int +imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields, + const char *const *headers) +{ + struct imapc_mail *mail = IMAPC_MAIL(_mail); + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct mail_index_view *view; + string_t *str; + uint32_t seq; + unsigned int i; + + i_assert(headers == NULL || + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)); + + if (!mbox->selected) { + mail_storage_set_error(_mail->box->storage, + MAIL_ERROR_NOTPOSSIBLE, "Can't fetch mails before selecting mailbox"); + return -1; + } + + if (!mail_stream_access_start(_mail)) + return -1; + + /* drop any fields that we may already be fetching currently */ + fields &= ENUM_NEGATE(mail->fetching_fields); + if (headers_have_subset(mail->fetching_headers, headers)) + headers = NULL; + if (fields == 0 && headers == NULL) + return mail->fetch_sent ? 0 : 1; + + if (!_mail->saving) { + /* if we already know that the mail is expunged, + don't try to FETCH it */ + view = mbox->delayed_sync_view != NULL ? + mbox->delayed_sync_view : mbox->box.view; + if (!mail_index_lookup_seq(view, _mail->uid, &seq) || + mail_index_is_expunged(view, seq)) { + mail_set_expunged(_mail); + return -1; + } + } else if (mbox->client_box == NULL) { + /* opened as save-only. we'll need to fetch the mail, + so actually SELECT/EXAMINE the mailbox */ + i_assert(mbox->box.opened); + + if (imapc_mailbox_select(mbox) < 0) + return -1; + } + + if ((fields & MAIL_FETCH_STREAM_BODY) != 0) + fields |= MAIL_FETCH_STREAM_HEADER; + + str = t_str_new(64); + str_printfa(str, "UID FETCH %u (", _mail->uid); + if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) + str_append(str, "INTERNALDATE "); + if ((fields & MAIL_FETCH_SAVE_DATE) != 0) { + i_assert(HAS_ALL_BITS(mbox->capabilities, + IMAPC_CAPABILITY_SAVEDATE)); + str_append(str, "SAVEDATE "); + } + if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0) + str_append(str, "RFC822.SIZE "); + if ((fields & MAIL_FETCH_GUID) != 0) { + str_append(str, mbox->guid_fetch_field_name); + str_append_c(str, ' '); + } + if ((fields & MAIL_FETCH_IMAP_BODY) != 0) + str_append(str, "BODY "); + if ((fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0) + str_append(str, "BODYSTRUCTURE "); + + if ((fields & MAIL_FETCH_STREAM_BODY) != 0) { + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_ZIMBRA_WORKAROUNDS)) + str_append(str, "BODY.PEEK[] "); + else { + /* BODY.PEEK[] can return different headers than + BODY.PEEK[HEADER] (e.g. invalid 8bit chars replaced + with '?' in HEADER) - this violates IMAP protocol + and messes up dsync since it sometimes fetches the + full body and sometimes only the headers. */ + str_append(str, "BODY.PEEK[HEADER] BODY.PEEK[TEXT] "); + } + fields |= MAIL_FETCH_STREAM_HEADER; + } else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0) + str_append(str, "BODY.PEEK[HEADER] "); + else if (headers != NULL) { + mail->fetching_headers = + headers_merge(mail->imail.mail.data_pool, headers, + mail->fetching_headers); + str_append(str, "BODY.PEEK[HEADER.FIELDS ("); + for (i = 0; mail->fetching_headers[i] != NULL; i++) { + if (i > 0) + str_append_c(str, ' '); + imap_append_astring(str, mail->fetching_headers[i]); + } + str_append(str, ")] "); + mail->header_list_fetched = FALSE; + } + str_truncate(str, str_len(str)-1); + str_append_c(str, ')'); + + mail->fetching_fields |= fields; + mail->fetch_count++; + mail->fetch_sent = FALSE; + mail->fetch_failed = FALSE; + + imapc_mail_delayed_send_or_merge(mail, str); + return 1; +} + +static void imapc_mail_cache_get(struct imapc_mail *mail, + struct imapc_mail_cache *cache) +{ + if (mail->body_fetched) + return; + + if (cache->fd != -1) { + mail->fd = cache->fd; + mail->imail.data.stream = i_stream_create_fd(mail->fd, 0); + cache->fd = -1; + } else if (cache->buf != NULL) { + mail->body = cache->buf; + mail->imail.data.stream = + i_stream_create_from_data(mail->body->data, + mail->body->used); + cache->buf = NULL; + } else { + return; + } + mail->header_fetched = TRUE; + mail->body_fetched = TRUE; + /* The stream was already accessed and now it's cached. + It still needs to be set accessed to avoid assert-crash. */ + mail->imail.mail.mail.mail_stream_accessed = TRUE; + imapc_mail_init_stream(mail); +} + +static enum mail_fetch_field +imapc_mail_get_wanted_fetch_fields(struct imapc_mail *mail) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box); + struct index_mail_data *data = &mail->imail.data; + enum mail_fetch_field fields = 0; + + if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 && + data->received_date == (time_t)-1) + fields |= MAIL_FETCH_RECEIVED_DATE; + if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0 && + data->save_date == (time_t)-1) { + if (HAS_ALL_BITS(mbox->capabilities, IMAPC_CAPABILITY_SAVEDATE)) + fields |= MAIL_FETCH_SAVE_DATE; + else + fields |= MAIL_FETCH_RECEIVED_DATE; + } + if ((data->wanted_fields & (MAIL_FETCH_PHYSICAL_SIZE | + MAIL_FETCH_VIRTUAL_SIZE)) != 0 && + data->physical_size == UOFF_T_MAX && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) + fields |= MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE; + if ((data->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0 && + data->body == NULL && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) + fields |= MAIL_FETCH_IMAP_BODY; + if ((data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0 && + data->bodystructure == NULL && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) + fields |= MAIL_FETCH_IMAP_BODYSTRUCTURE; + if ((data->wanted_fields & MAIL_FETCH_GUID) != 0 && + data->guid == NULL && mbox->guid_fetch_field_name != NULL) + fields |= MAIL_FETCH_GUID; + + if (data->stream == NULL && data->access_part != 0) { + if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0) + fields |= MAIL_FETCH_STREAM_BODY; + fields |= MAIL_FETCH_STREAM_HEADER; + } + return fields; +} + +void imapc_mail_try_init_stream_from_cache(struct imapc_mail *mail) +{ + struct mail *_mail = &mail->imail.mail.mail; + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + + if (mbox->prev_mail_cache.uid == _mail->uid) + imapc_mail_cache_get(mail, &mbox->prev_mail_cache); +} + +bool imapc_mail_prefetch(struct mail *_mail) +{ + struct imapc_mail *mail = IMAPC_MAIL(_mail); + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct index_mail_data *data = &mail->imail.data; + enum mail_fetch_field fields; + const char *const *headers = NULL; + + /* try to get as much from cache as possible */ + imapc_mail_update_access_parts(&mail->imail); + /* If mail is already cached we can avoid re-FETCHing the mail. + However, don't initialize the stream if we don't actually want to + access the mail. */ + if (mail->imail.data.access_part != 0) + imapc_mail_try_init_stream_from_cache(mail); + + fields = imapc_mail_get_wanted_fetch_fields(mail); + if (data->wanted_headers != NULL && data->stream == NULL && + (fields & MAIL_FETCH_STREAM_HEADER) == 0 && + !imapc_mail_has_headers_in_cache(&mail->imail, data->wanted_headers)) { + /* fetch specific headers */ + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) + headers = data->wanted_headers->name; + else + fields |= MAIL_FETCH_STREAM_HEADER; + } + if (fields != 0 || headers != NULL) T_BEGIN { + if (imapc_mail_send_fetch(_mail, fields, headers) > 0) + mail->imail.data.prefetch_sent = TRUE; + } T_END; + return !mail->imail.data.prefetch_sent; +} + +static bool +imapc_mail_have_fields(struct imapc_mail *imail, enum mail_fetch_field fields) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(imail->imail.mail.mail.box); + + if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) { + if (imail->imail.data.received_date == (time_t)-1) + return FALSE; + fields &= ENUM_NEGATE(MAIL_FETCH_RECEIVED_DATE); + } + if ((fields & MAIL_FETCH_SAVE_DATE) != 0) { + i_assert(HAS_ALL_BITS(mbox->capabilities, + IMAPC_CAPABILITY_SAVEDATE)); + if (imail->imail.data.save_date == (time_t)-1) + return FALSE; + fields &= ENUM_NEGATE(MAIL_FETCH_SAVE_DATE); + } + if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0) { + if (imail->imail.data.physical_size == UOFF_T_MAX) + return FALSE; + fields &= ENUM_NEGATE(MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE); + } + if ((fields & MAIL_FETCH_GUID) != 0) { + if (imail->imail.data.guid == NULL) + return FALSE; + fields &= ENUM_NEGATE(MAIL_FETCH_GUID); + } + if ((fields & MAIL_FETCH_IMAP_BODY) != 0) { + if (imail->imail.data.body == NULL) + return FALSE; + fields &= ENUM_NEGATE(MAIL_FETCH_IMAP_BODY); + } + if ((fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0) { + if (imail->imail.data.bodystructure == NULL) + return FALSE; + fields &= ENUM_NEGATE(MAIL_FETCH_IMAP_BODYSTRUCTURE); + } + if ((fields & (MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY)) != 0) { + if (imail->imail.data.stream == NULL) + return FALSE; + fields &= ENUM_NEGATE(MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY); + } + i_assert(fields == 0); + return TRUE; +} + +int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields, + const char *const *headers) +{ + struct imapc_mail *imail = IMAPC_MAIL(_mail); + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + int ret; + + if ((fields & MAIL_FETCH_GUID) != 0 && + mbox->guid_fetch_field_name == NULL) { + mail_storage_set_error(_mail->box->storage, + MAIL_ERROR_NOTPOSSIBLE, + "Message GUID not available in this server"); + return -1; + } + if (_mail->saving) { + mail_storage_set_error(_mail->box->storage, + MAIL_ERROR_NOTPOSSIBLE, + "Attempting to issue FETCH for a mail not yet committed"); + return -1; + } + + fields |= imapc_mail_get_wanted_fetch_fields(imail); + T_BEGIN { + ret = imapc_mail_send_fetch(_mail, fields, headers); + } T_END; + if (ret < 0) + return -1; + + /* we'll continue waiting until we've got all the fields we wanted, + or until all FETCH replies have been received (i.e. some FETCHes + failed) */ + if (ret > 0) + imapc_mail_fetch_flush(mbox); + while (imail->fetch_count > 0 && + (!imapc_mail_have_fields(imail, fields) || + !imail->header_list_fetched)) { + imapc_mailbox_run_nofetch(mbox); + } + if (imail->fetch_failed) { + mail_storage_set_internal_error(&mbox->storage->storage); + return -1; + } + return 0; +} + +void imapc_mail_fetch_flush(struct imapc_mailbox *mbox) +{ + struct imapc_command *cmd; + struct imapc_mail *mail; + + if (mbox->pending_fetch_request == NULL) { + i_assert(mbox->to_pending_fetch_send == NULL); + return; + } + + array_foreach_elem(&mbox->pending_fetch_request->mails, mail) + mail->fetch_sent = TRUE; + + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_mail_fetch_callback, + mbox->pending_fetch_request); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + array_push_back(&mbox->fetch_requests, &mbox->pending_fetch_request); + + imapc_command_send(cmd, str_c(mbox->pending_fetch_cmd)); + + mbox->pending_fetch_request = NULL; + timeout_remove(&mbox->to_pending_fetch_send); + str_truncate(mbox->pending_fetch_cmd, 0); +} + +static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply, + const struct imap_arg *arg, int *fd_r) +{ + const struct imap_arg *list; + unsigned int i, count; + + for (i = 0; i < reply->file_args_count; i++) { + const struct imapc_arg_file *farg = &reply->file_args[i]; + + if (farg->parent_arg == arg->parent && + imap_arg_get_list_full(arg->parent, &list, &count) && + farg->list_idx < count && &list[farg->list_idx] == arg) { + *fd_r = farg->fd; + return TRUE; + } + } + return FALSE; +} + +static void imapc_stream_filter(struct istream **input) +{ + static const char *imapc_hide_headers[] = { + /* Added by MS Exchange 2010 when \Flagged flag is set. + This violates IMAP guarantee of messages being immutable. */ + "X-Message-Flag" + }; + struct istream *filter_input; + + filter_input = i_stream_create_header_filter(*input, + HEADER_FILTER_EXCLUDE, + imapc_hide_headers, N_ELEMENTS(imapc_hide_headers), + *null_header_filter_callback, NULL); + i_stream_unref(input); + *input = filter_input; +} + +void imapc_mail_init_stream(struct imapc_mail *mail) +{ + struct index_mail *imail = &mail->imail; + struct mail *_mail = &imail->mail.mail; + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct istream *input; + uoff_t size; + int ret; + + i_stream_set_name(imail->data.stream, + t_strdup_printf("imapc mail uid=%u", _mail->uid)); + index_mail_set_read_buffer_size(_mail, imail->data.stream); + + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) { + /* enable filtering only when we're not passing through + RFC822.SIZE. otherwise we'll get size mismatches. */ + imapc_stream_filter(&imail->data.stream); + } + if (imail->mail.v.istream_opened != NULL) { + if (imail->mail.v.istream_opened(_mail, + &imail->data.stream) < 0) { + index_mail_close_streams(imail); + return; + } + } + ret = i_stream_get_size(imail->data.stream, TRUE, &size); + if (ret < 0) { + index_mail_close_streams(imail); + return; + } + i_assert(ret != 0); + /* Once message body is fetched, we can be sure of what its size is. + If we had already received RFC822.SIZE, overwrite it here in case + it's wrong. Also in more special cases the RFC822.SIZE may be + smaller than the fetched message header. In this case change the + size as well, otherwise reading via istream-mail will fail. */ + if (mail->body_fetched || imail->data.physical_size < size) { + if (mail->body_fetched) { + imail->data.inexact_total_sizes = FALSE; + /* Don't trust any existing virtual_size. Also don't + set it to size, because there's no guarantees about + the content having proper CRLF newlines, especially + not if istream_opened() has changed the stream. */ + imail->data.virtual_size = UOFF_T_MAX; + } + imail->data.physical_size = size; + } + + imail->data.stream_has_only_header = !mail->body_fetched; + if (index_mail_init_stream(imail, NULL, NULL, &input) < 0) + index_mail_close_streams(imail); +} + +static void +imapc_fetch_stream(struct imapc_mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *arg, + bool have_header, bool have_body) +{ + struct index_mail *imail = &mail->imail; + struct imapc_mailbox *mbox = IMAPC_MAILBOX(imail->mail.mail.box); + struct istream *hdr_stream = NULL; + const char *value; + int fd; + + if (imail->data.stream != NULL) { + i_assert(mail->header_fetched); + if (mail->body_fetched || !have_body) + return; + if (have_header) { + /* replace the existing stream */ + } else if (mail->fd == -1) { + /* append this body stream to the existing + header stream */ + hdr_stream = imail->data.stream; + i_stream_ref(hdr_stream); + } else { + /* append this body stream to the existing + header stream. we'll need to recreate the stream + with autoclosed fd. */ + if (lseek(mail->fd, 0, SEEK_SET) < 0) + i_error("lseek(imapc) failed: %m"); + hdr_stream = i_stream_create_fd_autoclose(&mail->fd, 0); + } + index_mail_close_streams(imail); + i_close_fd(&mail->fd); + } else { + if (!have_header) { + /* BODY.PEEK[TEXT] received - we can't currently handle + this before receiving BODY.PEEK[HEADER] reply */ + return; + } + } + + if (arg->type == IMAP_ARG_LITERAL_SIZE) { + if (!imapc_find_lfile_arg(reply, arg, &fd)) { + i_stream_unref(&hdr_stream); + return; + } + if ((fd = dup(fd)) == -1) { + i_error("dup() failed: %m"); + i_stream_unref(&hdr_stream); + return; + } + mail->fd = fd; + imail->data.stream = i_stream_create_fd(fd, 0); + } else { + if (!imap_arg_get_nstring(arg, &value)) + value = NULL; + if (value == NULL || + (value[0] == '\0' && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_EMPTY_IS_EXPUNGED))) { + mail_set_expunged(&imail->mail.mail); + i_stream_unref(&hdr_stream); + return; + } + if (mail->body == NULL) { + mail->body = buffer_create_dynamic(default_pool, + arg->str_len + 1); + } else if (!have_header && hdr_stream != NULL) { + /* header is already in the buffer - add body now + without destroying the existing header data */ + i_stream_unref(&hdr_stream); + } else { + buffer_set_used_size(mail->body, 0); + } + buffer_append(mail->body, value, arg->str_len); + imail->data.stream = i_stream_create_from_data(mail->body->data, + mail->body->used); + } + if (have_header) + mail->header_fetched = TRUE; + mail->body_fetched = have_body; + + if (hdr_stream != NULL) { + struct istream *inputs[3]; + + inputs[0] = hdr_stream; + inputs[1] = imail->data.stream; + inputs[2] = NULL; + imail->data.stream = i_stream_create_concat(inputs); + i_stream_unref(&inputs[0]); + i_stream_unref(&inputs[1]); + } + + imapc_mail_init_stream(mail); +} + +static void +imapc_fetch_header_stream(struct imapc_mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *args) +{ + const enum message_header_parser_flags hdr_parser_flags = + MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP | + MESSAGE_HEADER_PARSER_FLAG_DROP_CR; + const struct imap_arg *hdr_list; + struct mailbox_header_lookup_ctx *headers_ctx; + struct message_header_parser_ctx *parser; + struct message_header_line *hdr; + struct istream *input; + ARRAY_TYPE(const_string) hdr_arr; + const char *value; + int ret, fd; + + if (!imap_arg_get_list(args, &hdr_list)) + return; + if (!imap_arg_atom_equals(args+1, "]")) + return; + args += 2; + + /* see if this is reply to the latest headers list request + (parse it even if it's not) */ + t_array_init(&hdr_arr, 16); + while (imap_arg_get_astring(hdr_list, &value)) { + array_push_back(&hdr_arr, &value); + hdr_list++; + } + if (hdr_list->type != IMAP_ARG_EOL) + return; + array_append_zero(&hdr_arr); + + if (headers_have_subset(array_front(&hdr_arr), mail->fetching_headers)) + mail->header_list_fetched = TRUE; + + if (args->type == IMAP_ARG_LITERAL_SIZE) { + if (!imapc_find_lfile_arg(reply, args, &fd)) + return; + input = i_stream_create_fd(fd, 0); + } else { + if (!imap_arg_get_nstring(args, &value)) + return; + if (value == NULL) { + mail_set_expunged(&mail->imail.mail.mail); + return; + } + input = i_stream_create_from_data(value, args->str_len); + } + + headers_ctx = mailbox_header_lookup_init(mail->imail.mail.mail.box, + array_front(&hdr_arr)); + index_mail_parse_header_init(&mail->imail, headers_ctx); + + parser = message_parse_header_init(input, NULL, hdr_parser_flags); + while ((ret = message_parse_header_next(parser, &hdr)) > 0) + index_mail_parse_header(NULL, hdr, &mail->imail); + i_assert(ret != 0); + index_mail_parse_header(NULL, NULL, &mail->imail); + message_parse_header_deinit(&parser); + + mailbox_header_lookup_unref(&headers_ctx); + i_stream_destroy(&input); +} + +static const char * +imapc_args_to_bodystructure(struct imapc_mail *mail, + const struct imap_arg *list_arg, bool extended) +{ + const struct imap_arg *args; + struct message_part *parts = NULL; + const char *ret, *error; + pool_t pool; + + if (!imap_arg_get_list(list_arg, &args)) { + mail_set_critical(&mail->imail.mail.mail, + "imapc: Server sent invalid BODYSTRUCTURE parameters"); + return NULL; + } + + pool = pool_alloconly_create("imap bodystructure", 1024); + if (imap_bodystructure_parse_args(args, pool, &parts, &error) < 0) { + mail_set_critical(&mail->imail.mail.mail, + "imapc: Server sent invalid BODYSTRUCTURE: %s", error); + ret = NULL; + } else { + string_t *str = t_str_new(128); + if (imap_bodystructure_write(parts, str, extended, &error) < 0) { + /* All the input to imap_bodystructure_write() came + from imap_bodystructure_parse_args(). We should never + get here. Instead, if something is wrong the + parsing should have returned an error already. */ + str_truncate(str, 0); + imap_write_args(str, args); + i_panic("Failed to write parsed BODYSTRUCTURE: %s " + "(original string: '%s')", error, str_c(str)); + } + ret = p_strdup(mail->imail.mail.data_pool, str_c(str)); + } + pool_unref(&pool); + return ret; +} + +void imapc_mail_fetch_update(struct imapc_mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *args) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box); + const char *key, *value; + unsigned int i; + uoff_t size; + time_t t; + int tz; + bool match = FALSE; + + for (i = 0; args[i].type != IMAP_ARG_EOL; i += 2) { + if (!imap_arg_get_atom(&args[i], &key) || + args[i+1].type == IMAP_ARG_EOL) + break; + + if (strcasecmp(key, "BODY[]") == 0) { + imapc_fetch_stream(mail, reply, &args[i+1], TRUE, TRUE); + match = TRUE; + } else if (strcasecmp(key, "BODY[HEADER]") == 0) { + imapc_fetch_stream(mail, reply, &args[i+1], TRUE, FALSE); + match = TRUE; + } else if (strcasecmp(key, "BODY[TEXT]") == 0) { + imapc_fetch_stream(mail, reply, &args[i+1], FALSE, TRUE); + match = TRUE; + } else if (strcasecmp(key, "BODY[HEADER.FIELDS") == 0) { + imapc_fetch_header_stream(mail, reply, &args[i+1]); + match = TRUE; + } else if (strcasecmp(key, "INTERNALDATE") == 0) { + if (imap_arg_get_astring(&args[i+1], &value) && + imap_parse_datetime(value, &t, &tz)) { + mail->imail.data.received_date = t; + if (HAS_NO_BITS(mbox->capabilities, + IMAPC_CAPABILITY_SAVEDATE)) + mail->imail.data.save_date = t; + } + match = TRUE; + } else if (strcasecmp(key, "SAVEDATE") == 0) { + if (imap_arg_get_astring(&args[i+1], &value)) { + if (strcasecmp(value, "NIL") == 0) + mail->imail.data.save_date = 0; + else if (imap_parse_datetime(value, &t, &tz)) + mail->imail.data.save_date = t; + } + match = TRUE; + } else if (strcasecmp(key, "BODY") == 0) { + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) { + mail->imail.data.body = + imapc_args_to_bodystructure(mail, &args[i+1], FALSE); + } + match = TRUE; + } else if (strcasecmp(key, "BODYSTRUCTURE") == 0) { + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) { + mail->imail.data.bodystructure = + imapc_args_to_bodystructure(mail, &args[i+1], TRUE); + } + match = TRUE; + } else if (strcasecmp(key, "RFC822.SIZE") == 0) { + if (imap_arg_get_atom(&args[i+1], &value) && + str_to_uoff(value, &size) == 0 && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) { + mail->imail.data.physical_size = size; + mail->imail.data.virtual_size = size; + mail->imail.data.inexact_total_sizes = TRUE; + } + match = TRUE; + } else if (strcasecmp(key, "X-GM-MSGID") == 0 || + strcasecmp(key, "X-GUID") == 0) { + if (imap_arg_get_astring(&args[i+1], &value)) { + mail->imail.data.guid = + p_strdup(mail->imail.mail.pool, value); + } + match = TRUE; + } + } + if (!match) { + /* this is only a FETCH FLAGS update for the wanted mail */ + } else { + imapc_client_stop(mbox->storage->client->client); + } +} diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c new file mode 100644 index 0000000..1ecf03e --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-mail.c @@ -0,0 +1,675 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "hex-binary.h" +#include "sha1.h" +#include "istream.h" +#include "message-part-data.h" +#include "imap-envelope.h" +#include "imapc-msgmap.h" +#include "imapc-mail.h" +#include "imapc-storage.h" + +static bool imapc_mail_get_cached_guid(struct mail *_mail); + +struct mail * +imapc_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct imapc_mail *mail; + pool_t pool; + + pool = pool_alloconly_create("mail", 2048); + mail = p_new(pool, struct imapc_mail, 1); + mail->fd = -1; + + index_mail_init(&mail->imail, t, wanted_fields, wanted_headers, pool, NULL); + return &mail->imail.mail.mail; +} + +static bool imapc_mail_is_expunged(struct mail *_mail) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct imapc_msgmap *msgmap; + uint32_t lseq, rseq; + + if (!mbox->initial_sync_done) { + /* unknown at this point */ + return FALSE; + } + + if (mbox->sync_view != NULL) { + /* check if another session has already expunged it */ + if (!mail_index_lookup_seq(mbox->sync_view, _mail->uid, &lseq)) + return TRUE; + } + + /* check if we've received EXPUNGE for it */ + msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box); + if (!imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) + return TRUE; + + /* we may be running against a server that hasn't bothered sending + us an EXPUNGE. see if NOOP sends it. */ + imapc_mailbox_noop(mbox); + if (!mbox->initial_sync_done) { + /* NOOP caused a reconnection and desync */ + return FALSE; + } + + return !imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq); +} + +static int imapc_mail_failed(struct mail *mail, const char *field) +{ + struct imapc_mail *imail = IMAPC_MAIL(mail); + struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->box); + bool fix_broken_mail = FALSE; + + if (mail->expunged || imapc_mail_is_expunged(mail)) { + mail_set_expunged(mail); + } else if (!imapc_client_mailbox_is_opened(mbox->client_box)) { + /* we've already logged a disconnection error */ + mail_storage_set_internal_error(mail->box->storage); + } else { + /* By default we'll assume that this is a critical failure, + because we don't want to lose any data. We can be here + either because it's a temporary failure on the server or + it's a permanent failure. Unfortunately we can't know + which case it is, so permanent failures need to be worked + around by setting imapc_features=fetch-fix-broken-mails. + + One reason for permanent failures was that earlier Exchange + versions failed to return any data for messages in Calendars + mailbox. This seems to be fixed in newer versions. + */ + fix_broken_mail = imail->fetch_ignore_if_missing; + mail_set_critical(mail, + "imapc: Remote server didn't send %s%s (FETCH replied: %s)", + field, fix_broken_mail ? " - treating it as empty" : "", + imail->last_fetch_reply); + } + return fix_broken_mail ? 0 : -1; +} + +static uint64_t imapc_mail_get_modseq(struct mail *_mail) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct imapc_msgmap *msgmap; + const uint64_t *modseqs; + unsigned int count; + uint32_t rseq; + + if (!imapc_mailbox_has_modseqs(mbox)) + return index_mail_get_modseq(_mail); + + msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box); + if (imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) { + modseqs = array_get(&mbox->rseq_modseqs, &count); + if (rseq <= count) + return modseqs[rseq-1]; + } + return 1; /* unknown modseq */ +} + +static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (data->received_date == (time_t)-1) { + if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE, NULL) < 0) + return -1; + if (data->received_date == (time_t)-1) { + if (imapc_mail_failed(_mail, "INTERNALDATE") < 0) + return -1; + /* assume that the server never returns INTERNALDATE + for this mail (see BODY[] failure handling) */ + data->received_date = 0; + } + } + *date_r = data->received_date; + return 0; +} + +static int imapc_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + if (data->save_date != 0 && index_mail_get_save_date(_mail, date_r) > 0) + return 1; + + if (HAS_NO_BITS(mbox->capabilities, IMAPC_CAPABILITY_SAVEDATE)) { + data->save_date = 0; + } else if (data->save_date == (time_t)-1) { + if (imapc_mail_fetch(_mail, MAIL_FETCH_SAVE_DATE, NULL) < 0) + return -1; + if (data->save_date == (time_t)-1 && + imapc_mail_failed(_mail, "SAVEDATE") < 0) + return -1; + } + if (data->save_date == (time_t)-1 || data->save_date == 0) { + if (imapc_mail_get_received_date(_mail, date_r) < 0) + return -1; + return 0; + } + *date_r = data->save_date; + return 1; +} + +static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct istream *input; + uoff_t old_offset; + int ret; + + if (data->physical_size == UOFF_T_MAX) + (void)index_mail_get_physical_size(_mail, size_r); + if (data->physical_size != UOFF_T_MAX) { + *size_r = data->physical_size; + return 0; + } + + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) && + data->stream == NULL) { + /* Trust RFC822.SIZE to be correct enough to present to the + IMAP client. However, it can be wrong in some implementation + so try not to trust it too much. */ + if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0) + return -1; + if (data->physical_size == UOFF_T_MAX) { + if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0) + return -1; + /* assume that the server never returns RFC822.SIZE + for this mail (see BODY[] failure handling) */ + data->physical_size = 0; + } + *size_r = data->physical_size; + return 0; + } + + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + return -1; + i_assert(data->stream != NULL); + i_stream_seek(data->stream, old_offset); + + ret = i_stream_get_size(data->stream, TRUE, + &data->physical_size); + if (ret <= 0) { + i_assert(ret != 0); + mail_set_critical(_mail, "imapc: stat(%s) failed: %m", + i_stream_get_name(data->stream)); + return -1; + } + *size_r = data->physical_size; + return 0; +} + +static int imapc_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + if (imapc_mail_get_physical_size(_mail, size_r) < 0) + return -1; + data->virtual_size = data->physical_size; + return 0; +} + +static int +imapc_mail_get_header_stream(struct mail *_mail, + struct mailbox_header_lookup_ctx *headers, + struct istream **stream_r) +{ + struct imapc_mail *mail = IMAPC_MAIL(_mail); + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + enum mail_lookup_abort old_abort = _mail->lookup_abort; + int ret; + + if (mail->imail.data.access_part != 0 || + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) { + /* we're going to be reading the header/body anyway */ + return index_mail_get_header_stream(_mail, headers, stream_r); + } + + /* see if the wanted headers are already in cache */ + _mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + ret = index_mail_get_header_stream(_mail, headers, stream_r); + _mail->lookup_abort = old_abort; + if (ret == 0) + return 0; + + /* fetch only the wanted headers */ + if (imapc_mail_fetch(_mail, 0, headers->name) < 0) + return -1; + /* the headers should cached now. */ + return index_mail_get_header_stream(_mail, headers, stream_r); +} + +static int +imapc_mail_get_headers(struct mail *_mail, const char *field, + bool decode_to_utf8, const char *const **value_r) +{ + struct mailbox_header_lookup_ctx *headers; + const char *header_names[2]; + const unsigned char *data; + size_t size; + struct istream *input; + int ret; + + header_names[0] = field; + header_names[1] = NULL; + headers = mailbox_header_lookup_init(_mail->box, header_names); + ret = mail_get_header_stream(_mail, headers, &input); + mailbox_header_lookup_unref(&headers); + if (ret < 0) + return -1; + + while (i_stream_read_more(input, &data, &size) > 0) + i_stream_skip(input, size); + /* the header should cached now. */ + return index_mail_get_headers(_mail, field, decode_to_utf8, value_r); +} + +static int +imapc_mail_get_first_header(struct mail *_mail, const char *field, + bool decode_to_utf8, const char **value_r) +{ + const char *const *values; + int ret; + + ret = imapc_mail_get_headers(_mail, field, decode_to_utf8, &values); + if (ret <= 0) + return ret; + *value_r = values[0]; + return 1; +} + +static int +imapc_mail_get_stream(struct mail *_mail, bool get_body, + struct message_size *hdr_size, + struct message_size *body_size, struct istream **stream_r) +{ + struct imapc_mail *mail = IMAPC_MAIL(_mail); + struct index_mail_data *data = &mail->imail.data; + enum mail_fetch_field fetch_field; + + if (get_body && !mail->body_fetched && + mail->imail.data.stream != NULL) { + /* we've fetched the header, but we need the body now too */ + index_mail_close_streams(&mail->imail); + /* don't re-use any cached header sizes. we may be + intentionally downloading the full body because the header + wasn't returned correctly (e.g. pop3-migration does this) */ + data->hdr_size_set = FALSE; + } + + /* See if we can get it from cache. If the wanted_fields/headers are + set properly, this is usually already done by prefetching. */ + imapc_mail_try_init_stream_from_cache(mail); + + if (data->stream == NULL) { + if (!data->initialized) { + /* coming here from mail_set_seq() */ + mail_set_aborted(_mail); + return -1; + } + if (_mail->expunged) { + /* We already detected that the mail is expunged. + Don't spend time trying to FETCH it again. */ + mail_set_expunged(_mail); + return -1; + } + fetch_field = get_body || + (data->access_part & READ_BODY) != 0 ? + MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER; + if (imapc_mail_fetch(_mail, fetch_field, NULL) < 0) + return -1; + + if (data->stream == NULL) { + if (imapc_mail_failed(_mail, "BODY[]") < 0) + return -1; + i_assert(data->stream == NULL); + + /* return the broken email as empty */ + mail->body_fetched = TRUE; + data->stream = i_stream_create_from_data(NULL, 0); + imapc_mail_init_stream(mail); + } + } + + return index_mail_init_stream(&mail->imail, hdr_size, body_size, + stream_r); +} + +bool imapc_mail_has_headers_in_cache(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers) +{ + struct mail *_mail = &mail->mail.mail; + unsigned int i; + + for (i = 0; i < headers->count; i++) { + if (mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, headers->idx[i]) <= 0) + return FALSE; + } + return TRUE; +} + +void imapc_mail_update_access_parts(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct index_mail_data *data = &mail->data; + struct mailbox_header_lookup_ctx *header_ctx; + const char *str; + time_t date; + uoff_t size; + + if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0) + (void)index_mail_get_received_date(_mail, &date); + if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0) { + if (index_mail_get_save_date(_mail, &date) < 0 && + HAS_NO_BITS(mbox->capabilities, + IMAPC_CAPABILITY_SAVEDATE)) { + (void)index_mail_get_received_date(_mail, &date); + data->save_date = data->received_date; + } + } + if ((data->wanted_fields & (MAIL_FETCH_PHYSICAL_SIZE | + MAIL_FETCH_VIRTUAL_SIZE)) != 0) { + if (index_mail_get_physical_size(_mail, &size) < 0 && + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) + data->access_part |= READ_HDR | READ_BODY; + } + if ((data->wanted_fields & MAIL_FETCH_GUID) != 0) + (void)imapc_mail_get_cached_guid(_mail); + if ((data->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0) + (void)index_mail_get_cached_body(mail, &str); + if ((data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0) + (void)index_mail_get_cached_bodystructure(mail, &str); + + if (data->access_part == 0 && data->wanted_headers != NULL && + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) { + /* see if all wanted headers exist in cache */ + if (!imapc_mail_has_headers_in_cache(mail, data->wanted_headers)) + data->access_part |= PARSE_HDR; + } + if (data->access_part == 0 && + (data->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0 && + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) { + /* the common code already checked this partially, + but we need a guaranteed correct answer */ + header_ctx = mailbox_header_lookup_init(_mail->box, + message_part_envelope_headers); + if (!imapc_mail_has_headers_in_cache(mail, header_ctx)) + data->access_part |= PARSE_HDR; + mailbox_header_lookup_unref(&header_ctx); + } +} + +static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving) +{ + struct imapc_mail *imail = IMAPC_MAIL(_mail); + struct index_mail *mail = &imail->imail; + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; + + index_mail_set_seq(_mail, seq, saving); + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) { + /* RFC822.SIZE may be read from vsize record or cache. It may + not be exactly correct. */ + mail->data.inexact_total_sizes = TRUE; + } + + /* searching code handles prefetching internally, + elsewhere we want to do it immediately */ + if (!mail->mail.search_mail && !_mail->saving) + (void)imapc_mail_prefetch(_mail); +} + +static void +imapc_mail_add_temp_wanted_fields(struct mail *_mail, + enum mail_fetch_field fields, + struct mailbox_header_lookup_ctx *headers) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + index_mail_add_temp_wanted_fields(_mail, fields, headers); + if (_mail->seq != 0) + imapc_mail_update_access_parts(mail); +} + +static void imapc_mail_close(struct mail *_mail) +{ + struct imapc_mail *mail = IMAPC_MAIL(_mail); + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct imapc_mail_cache *cache = &mbox->prev_mail_cache; + + if (mail->fetch_count > 0) { + imapc_mail_fetch_flush(mbox); + while (mail->fetch_count > 0) + imapc_mailbox_run_nofetch(mbox); + } + + index_mail_close(_mail); + + mail->fetching_headers = NULL; + if (mail->body_fetched) { + imapc_mail_cache_free(cache); + cache->uid = _mail->uid; + if (mail->fd != -1) { + cache->fd = mail->fd; + mail->fd = -1; + } else { + cache->buf = mail->body; + mail->body = NULL; + } + } + i_close_fd(&mail->fd); + buffer_free(&mail->body); + mail->header_fetched = FALSE; + mail->body_fetched = FALSE; + + i_assert(mail->fetch_count == 0); +} + +static int imapc_mail_get_hdr_hash(struct index_mail *imail) +{ + struct istream *input; + const unsigned char *data; + size_t size; + uoff_t old_offset; + struct sha1_ctxt sha1_ctx; + unsigned char sha1_output[SHA1_RESULTLEN]; + const char *sha1_str; + + sha1_init(&sha1_ctx); + old_offset = imail->data.stream == NULL ? 0 : + imail->data.stream->v_offset; + if (mail_get_hdr_stream(&imail->mail.mail, NULL, &input) < 0) + return -1; + i_assert(imail->data.stream != NULL); + while (i_stream_read_more(input, &data, &size) > 0) { + sha1_loop(&sha1_ctx, data, size); + i_stream_skip(input, size); + } + i_stream_seek(imail->data.stream, old_offset); + sha1_result(&sha1_ctx, sha1_output); + + sha1_str = binary_to_hex(sha1_output, sizeof(sha1_output)); + imail->data.guid = p_strdup(imail->mail.data_pool, sha1_str); + return 0; +} + +static bool imapc_mail_get_cached_guid(struct mail *_mail) +{ + struct index_mail *imail = INDEX_MAIL(_mail); + const enum index_cache_field cache_idx = + imail->ibox->cache_fields[MAIL_CACHE_GUID].idx; + string_t *str; + + if (imail->data.guid != NULL) { + if (mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, cache_idx)) { + /* GUID was prefetched - add to cache */ + index_mail_cache_add_idx(imail, cache_idx, + imail->data.guid, strlen(imail->data.guid)); + } + return TRUE; + } + + str = str_new(imail->mail.data_pool, 64); + if (mail_cache_lookup_field(_mail->transaction->cache_view, + str, imail->mail.mail.seq, cache_idx) > 0) { + imail->data.guid = str_c(str); + return TRUE; + } + return FALSE; +} + +static int imapc_mail_get_guid(struct mail *_mail, const char **value_r) +{ + struct index_mail *imail = INDEX_MAIL(_mail); + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + const enum index_cache_field cache_idx = + imail->ibox->cache_fields[MAIL_CACHE_GUID].idx; + + if (imapc_mail_get_cached_guid(_mail)) { + *value_r = imail->data.guid; + return 0; + } + + /* GUID not in cache, fetch it */ + if (mbox->guid_fetch_field_name != NULL) { + if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID, NULL) < 0) + return -1; + if (imail->data.guid == NULL) { + (void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name); + return -1; + } + } else { + /* use hash of message headers as the GUID */ + if (imapc_mail_get_hdr_hash(imail) < 0) + return -1; + } + + index_mail_cache_add_idx(imail, cache_idx, + imail->data.guid, strlen(imail->data.guid)); + *value_r = imail->data.guid; + return 0; +} + +static int +imapc_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box); + struct index_mail *imail = INDEX_MAIL(_mail); + uint64_t num; + + switch (field) { + case MAIL_FETCH_GUID: + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GUID_FORCED) && + mbox->guid_fetch_field_name == NULL) { + /* GUIDs not supported by server */ + break; + } + *value_r = ""; + return imapc_mail_get_guid(_mail, value_r); + case MAIL_FETCH_UIDL_BACKEND: + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) + break; + if (imapc_mail_get_guid(_mail, value_r) < 0) + return -1; + if (str_to_uint64(*value_r, &num) < 0) { + mail_set_critical(_mail, + "X-GM-MSGID not 64bit integer as expected for POP3 UIDL generation: %s", *value_r); + return -1; + } + + *value_r = p_strdup_printf(imail->mail.data_pool, + "GmailId%"PRIx64, num); + return 0; + case MAIL_FETCH_IMAP_BODY: + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) + break; + + if (index_mail_get_cached_body(imail, value_r)) + return 0; + if (imapc_mail_fetch(_mail, field, NULL) < 0) + return -1; + if (imail->data.body == NULL) { + (void)imapc_mail_failed(_mail, "BODY"); + return -1; + } + *value_r = imail->data.body; + return 0; + case MAIL_FETCH_IMAP_BODYSTRUCTURE: + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) + break; + + if (index_mail_get_cached_bodystructure(imail, value_r)) + return 0; + if (imapc_mail_fetch(_mail, field, NULL) < 0) + return -1; + if (imail->data.bodystructure == NULL) { + (void)imapc_mail_failed(_mail, "BODYSTRUCTURE"); + return -1; + } + *value_r = imail->data.bodystructure; + return 0; + default: + break; + } + + return index_mail_get_special(_mail, field, value_r); +} + +struct mail_vfuncs imapc_mail_vfuncs = { + imapc_mail_close, + index_mail_free, + imapc_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + imapc_mail_prefetch, + index_mail_precache, + imapc_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + imapc_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + imapc_mail_get_received_date, + imapc_mail_get_save_date, + imapc_mail_get_virtual_size, + imapc_mail_get_physical_size, + imapc_mail_get_first_header, + imapc_mail_get_headers, + imapc_mail_get_header_stream, + imapc_mail_get_stream, + index_mail_get_binary_stream, + imapc_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/imapc/imapc-mail.h b/src/lib-storage/index/imapc/imapc-mail.h new file mode 100644 index 0000000..52dfe5e --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-mail.h @@ -0,0 +1,51 @@ +#ifndef IMAPC_MAIL_H +#define IMAPC_MAIL_H + +#include "index-mail.h" + +struct imap_arg; +struct imapc_untagged_reply; +struct imapc_mailbox; + +struct imapc_mail { + struct index_mail imail; + + enum mail_fetch_field fetching_fields; + const char *const *fetching_headers; + unsigned int fetch_count; + bool fetch_sent; + const char *last_fetch_reply; + + int fd; + buffer_t *body; + bool header_fetched; + bool body_fetched; + bool header_list_fetched; + bool fetch_ignore_if_missing; + bool fetch_failed; +}; + +#define IMAPC_MAIL(s) container_of(s, struct imapc_mail, imail.mail.mail) + +extern struct mail_vfuncs imapc_mail_vfuncs; + +struct mail * +imapc_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); +int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields, + const char *const *headers); +void imapc_mail_try_init_stream_from_cache(struct imapc_mail *mail); +bool imapc_mail_prefetch(struct mail *mail); +void imapc_mail_fetch_flush(struct imapc_mailbox *mbox); +void imapc_mail_init_stream(struct imapc_mail *mail); +bool imapc_mail_has_headers_in_cache(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers); + +void imapc_mail_fetch_update(struct imapc_mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *args); +void imapc_mail_update_access_parts(struct index_mail *mail); +void imapc_mail_command_flush(struct imapc_mailbox *mbox); + +#endif diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c new file mode 100644 index 0000000..73a38e2 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-mailbox.c @@ -0,0 +1,1015 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "mail-index-modseq.h" +#include "imap-arg.h" +#include "imap-seqset.h" +#include "imap-util.h" +#include "imapc-mail.h" +#include "imapc-msgmap.h" +#include "imapc-list.h" +#include "imapc-search.h" +#include "imapc-sync.h" +#include "imapc-storage.h" + +#define NOTIFY_DELAY_MSECS 500 + +void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox, + const char *reason, ...) +{ + const char *errmsg; + va_list va; + + va_start(va, reason); + errmsg = t_strdup_printf("Mailbox '%s' state corrupted: %s", + mbox->box.name, t_strdup_vprintf(reason, va)); + va_end(va); + + mail_storage_set_internal_error(&mbox->storage->storage); + + if (!mbox->initial_sync_done) { + /* we failed during initial sync. need to rebuild indexes if + we want to get this fixed */ + mail_index_mark_corrupted(mbox->box.index); + } else { + /* maybe the remote server is buggy and has become confused. + try reconnecting. */ + } + imapc_client_mailbox_reconnect(mbox->client_box, errmsg); +} + +struct mail_index_view * +imapc_mailbox_get_sync_view(struct imapc_mailbox *mbox) +{ + if (mbox->sync_view == NULL) + mbox->sync_view = mail_index_view_open(mbox->box.index); + return mbox->sync_view; +} + +static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox) +{ + if (mbox->delayed_sync_trans != NULL) + return; + + i_assert(mbox->delayed_sync_cache_view == NULL); + i_assert(mbox->delayed_sync_cache_trans == NULL); + + mbox->delayed_sync_trans = + mail_index_transaction_begin(imapc_mailbox_get_sync_view(mbox), + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mbox->delayed_sync_view = + mail_index_transaction_open_updated_view(mbox->delayed_sync_trans); + + mbox->delayed_sync_cache_view = + mail_cache_view_open(mbox->box.cache, mbox->delayed_sync_view); + mbox->delayed_sync_cache_trans = + mail_cache_get_transaction(mbox->delayed_sync_cache_view, + mbox->delayed_sync_trans); +} + +static int imapc_mailbox_commit_delayed_expunges(struct imapc_mailbox *mbox) +{ + struct mail_index_view *view = imapc_mailbox_get_sync_view(mbox); + struct mail_index_transaction *trans; + struct seq_range_iter iter; + unsigned int n; + uint32_t lseq, uid; + int ret; + + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + + seq_range_array_iter_init(&iter, &mbox->delayed_expunged_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + if (mail_index_lookup_seq(view, uid, &lseq)) + mail_index_expunge(trans, lseq); + } + array_clear(&mbox->delayed_expunged_uids); + ret = mail_index_transaction_commit(&trans); + if (ret < 0) + mailbox_set_index_error(&mbox->box); + return ret; +} + +int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox, + bool force, bool *changes_r) +{ + int ret = 0; + + *changes_r = FALSE; + + if (mbox->delayed_sync_view != NULL) + mail_index_view_close(&mbox->delayed_sync_view); + if (mbox->delayed_sync_trans == NULL) + ; + else if (!mbox->selected && !force) { + /* ignore any changes done during SELECT */ + mail_index_transaction_rollback(&mbox->delayed_sync_trans); + } else { + if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } + *changes_r = TRUE; + } + mbox->delayed_sync_cache_trans = NULL; + if (mbox->delayed_sync_cache_view != NULL) + mail_cache_view_close(&mbox->delayed_sync_cache_view); + + if (array_count(&mbox->delayed_expunged_uids) > 0) { + /* delayed expunges - commit them now in a separate + transaction. Reopen mbox->sync_view to see changes + committed in delayed_sync_trans. */ + if (mbox->sync_view != NULL) + mail_index_view_close(&mbox->sync_view); + if (imapc_mailbox_commit_delayed_expunges(mbox) < 0) + ret = -1; + } + + if (mbox->sync_view != NULL) + mail_index_view_close(&mbox->sync_view); + i_assert(mbox->delayed_sync_trans == NULL); + i_assert(mbox->delayed_sync_view == NULL); + i_assert(mbox->delayed_sync_cache_trans == NULL); + return ret; +} + +static void imapc_mailbox_idle_timeout(struct imapc_mailbox *mbox) +{ + timeout_remove(&mbox->to_idle_delay); + if (mbox->box.notify_callback != NULL) + mbox->box.notify_callback(&mbox->box, mbox->box.notify_context); +} + +static void imapc_mailbox_idle_notify(struct imapc_mailbox *mbox) +{ + struct ioloop *old_ioloop = current_ioloop; + + if (mbox->box.notify_callback != NULL && + mbox->to_idle_delay == NULL) { + io_loop_set_current(mbox->storage->root_ioloop); + mbox->to_idle_delay = + timeout_add_short(NOTIFY_DELAY_MSECS, + imapc_mailbox_idle_timeout, mbox); + io_loop_set_current(old_ioloop); + } +} + +static void +imapc_mailbox_index_expunge(struct imapc_mailbox *mbox, uint32_t uid) +{ + uint32_t lseq; + + if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq)) + mail_index_expunge(mbox->delayed_sync_trans, lseq); + else if (mail_index_lookup_seq(mbox->delayed_sync_view, uid, &lseq)) { + /* this message exists only in this transaction. lib-index + can't currently handle expunging anything except the last + appended message in a transaction, and fixing it would be + quite a lot of trouble. so instead we'll just delay doing + this expunge until after the current transaction has been + committed. */ + seq_range_array_add(&mbox->delayed_expunged_uids, uid); + } else { + /* already expunged by another session */ + } +} + +static void +imapc_mailbox_fetch_state_finish(struct imapc_mailbox *mbox) +{ + uint32_t lseq, uid, msg_count; + + if (mbox->sync_next_lseq == 0) { + /* FETCH n:*, not 1:* */ + i_assert(mbox->state_fetched_success || + (mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0); + return; + } + + /* if we haven't seen FETCH reply for some messages at the end of + mailbox they've been externally expunged. */ + msg_count = mail_index_view_get_messages_count(mbox->delayed_sync_view); + for (lseq = mbox->sync_next_lseq; lseq <= msg_count; lseq++) { + mail_index_lookup_uid(mbox->delayed_sync_view, lseq, &uid); + if (uid >= mbox->sync_uid_next) { + /* another process already added new messages to index + that our IMAP connection hasn't seen yet */ + break; + } + imapc_mailbox_index_expunge(mbox, uid); + } + + mbox->sync_next_lseq = 0; + mbox->sync_next_rseq = 0; + mbox->state_fetched_success = TRUE; +} + +static void +imapc_mailbox_fetch_state_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_mailbox *mbox = context; + + mbox->state_fetching_uid1 = FALSE; + mbox->delayed_untagged_exists = FALSE; + imapc_client_stop(mbox->storage->client->client); + + switch (reply->state) { + case IMAPC_COMMAND_STATE_OK: + imapc_mailbox_fetch_state_finish(mbox); + break; + case IMAPC_COMMAND_STATE_NO: + imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS, reply); + break; + case IMAPC_COMMAND_STATE_DISCONNECTED: + mail_storage_set_internal_error(mbox->box.storage); + + break; + default: + mail_storage_set_critical(mbox->box.storage, + "imapc: state FETCH failed: %s", reply->text_full); + break; + } +} + +void imap_mailbox_select_finish(struct imapc_mailbox *mbox) +{ + if (mbox->exists_count == 0) { + /* no mails. expunge everything. */ + mbox->sync_next_lseq = 1; + imapc_mailbox_init_delayed_trans(mbox); + imapc_mailbox_fetch_state_finish(mbox); + } else { + /* We don't know the latest flags, refresh them. */ + (void)imapc_mailbox_fetch_state(mbox, 1); + } + mbox->selected = TRUE; +} + +bool +imapc_mailbox_fetch_state(struct imapc_mailbox *mbox, uint32_t first_uid) +{ + struct imapc_command *cmd; + + if (mbox->exists_count == 0) { + /* empty mailbox - no point in fetching anything. + just make sure everything is expunged in local index. + Delay calling imapc_mailbox_fetch_state_finish() until + SELECT finishes, so we see the updated UIDNEXT. */ + return FALSE; + } + if (mbox->state_fetching_uid1) { + /* retrying after reconnection - don't send duplicate */ + return FALSE; + } + + string_t *str = t_str_new(64); + str_printfa(str, "UID FETCH %u:* (FLAGS", first_uid); + if (imapc_mailbox_has_modseqs(mbox)) { + str_append(str, " MODSEQ"); + mail_index_modseq_enable(mbox->box.index); + } + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) { + enum mailbox_info_flags flags; + + if (!mail_index_is_in_memory(mbox->box.index)) { + /* these can be efficiently fetched among flags and + stored into cache */ + str_append(str, " X-GM-MSGID"); + } + /* do this only for the \All mailbox */ + if (imapc_list_get_mailbox_flags(mbox->box.list, + mbox->box.name, &flags) == 0 && + (flags & MAILBOX_SPECIALUSE_ALL) != 0) + str_append(str, " X-GM-LABELS"); + + } + str_append_c(str, ')'); + + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_mailbox_fetch_state_callback, mbox); + if (first_uid == 1) { + mbox->sync_next_lseq = 1; + mbox->sync_next_rseq = 1; + mbox->state_fetched_success = FALSE; + /* only the FETCH 1:* is retriable - others will be retried + by the 1:* after the reconnection */ + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + } + mbox->state_fetching_uid1 = first_uid == 1; + imapc_command_send(cmd, str_c(str)); + return TRUE; +} + +static void +imapc_untagged_exists(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + struct mail_index_view *view; + uint32_t exists_count = reply->num; + + if (mbox == NULL) + return; + if (IMAPC_MAILBOX_IS_FULLY_SELECTED(mbox) && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES)) { + /* ignore all except the first EXISTS reply (returned by + SELECT) */ + return; + } + + mbox->exists_count = exists_count; + mbox->exists_received = TRUE; + + view = mbox->delayed_sync_view; + if (view == NULL) + view = imapc_mailbox_get_sync_view(mbox); + + if (!mbox->selecting && mbox->sync_fetch_first_uid != 1) { + const struct mail_index_header *hdr; + hdr = mail_index_get_header(view); + mbox->sync_fetch_first_uid = hdr->next_uid; + mbox->delayed_untagged_exists = TRUE; + } + imapc_mailbox_idle_notify(mbox); +} + +static bool keywords_are_equal(struct mail_keywords *kw, + const ARRAY_TYPE(keyword_indexes) *kw_arr) +{ + const unsigned int *kw_idx; + unsigned int i, j, count; + + kw_idx = array_get(kw_arr, &count); + if (count != kw->count) + return FALSE; + + /* there are normally only a few keywords, so O(n^2) is fine */ + for (i = 0; i < count; i++) { + for (j = 0; j < count; j++) { + if (kw->idx[i] == kw_idx[j]) + break; + } + if (j == count) + return FALSE; + } + return TRUE; +} + +static int +imapc_mailbox_msgmap_update(struct imapc_mailbox *mbox, + uint32_t rseq, uint32_t fetch_uid, + uint32_t *lseq_r, uint32_t *uid_r, + bool *new_message_r) +{ + struct imapc_msgmap *msgmap; + uint32_t uid, msg_count, rseq2; + + *lseq_r = 0; + *uid_r = uid = fetch_uid; + *new_message_r = FALSE; + + if (rseq > mbox->exists_count) { + /* Receiving a FETCH for a message that EXISTS hasn't + announced yet. MS Exchange has a bug where our UID FETCH + request sometimes sends replies where sequences are above + EXISTS value, but their UIDs are for existing messages. + We'll just ignore these replies. */ + return 0; + } + if (rseq < mbox->prev_skipped_rseq && + fetch_uid > mbox->prev_skipped_uid) { + /* This was the initial attempt at catching the above + MS Exchange bug, but the above one appears to catch all + these cases. But keep it here just in case. */ + imapc_mailbox_set_corrupted(mbox, + "FETCH sequence/UID order is mixed " + "(seq=%u,%u vs uid=%u,%u)", + mbox->prev_skipped_rseq, rseq, + mbox->prev_skipped_uid, fetch_uid); + return -1; + } + + msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box); + msg_count = imapc_msgmap_count(msgmap); + if (fetch_uid != 0 && mbox->state_fetched_success && + (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS) || + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES))) { + /* if we know the UID, use own own generated rseq instead of + the potentially broken rseq that the server sent. + Skip this during the initial FETCH 1:* (UID ..) handling, + or we can't detect duplicate UIDs and will instead + assert-crash later on. */ + uint32_t fixed_rseq; + + if (imapc_msgmap_uid_to_rseq(msgmap, fetch_uid, &fixed_rseq)) + rseq = fixed_rseq; + else if (fetch_uid >= imapc_msgmap_uidnext(msgmap) && + rseq <= msg_count) { + /* The current rseq is wrong. Lets hope that the + correct rseq is the next new one. This happens + especially with no-msn-updates when mails have been + expunged and new mails arrive in the same session. */ + rseq = msg_count+1; + } + } + + if (rseq <= msg_count) { + uid = imapc_msgmap_rseq_to_uid(msgmap, rseq); + if (uid != fetch_uid && fetch_uid != 0) { + imapc_mailbox_set_corrupted(mbox, + "FETCH UID mismatch (%u != %u)", + fetch_uid, uid); + return -1; + } + *uid_r = uid; + } else if (fetch_uid == 0 || rseq != msg_count+1) { + /* probably a flag update for a message we haven't yet + received our initial UID FETCH for. we should get + another one. */ + if (fetch_uid == 0) + return 0; + + if (imapc_msgmap_uid_to_rseq(msgmap, fetch_uid, &rseq2)) { + imapc_mailbox_set_corrupted(mbox, + "FETCH returned wrong sequence for UID %u " + "(%u != %u)", fetch_uid, rseq, rseq2); + return -1; + } + mbox->prev_skipped_rseq = rseq; + mbox->prev_skipped_uid = fetch_uid; + /* Check if this uid must be added later when syncing. */ + *new_message_r = TRUE; + } else if (fetch_uid < imapc_msgmap_uidnext(msgmap)) { + imapc_mailbox_set_corrupted(mbox, + "Expunged message reappeared in session " + "(uid=%u < next_uid=%u)", + fetch_uid, imapc_msgmap_uidnext(msgmap)); + return -1; + } else { + /* newly seen message */ + imapc_msgmap_append(msgmap, rseq, uid); + if (uid < mbox->min_append_uid || + uid < mail_index_get_header(mbox->delayed_sync_view)->next_uid) { + /* message is already added to index */ + } else if (mbox->state_fetching_uid1) { + /* Initial fetching, allow messages to be appened to + index directly */ + mail_index_append(mbox->delayed_sync_trans, + uid, lseq_r); + mbox->min_append_uid = uid + 1; + } else { + /* message is not yet added to index, in order to + prevent log synchronization errors add this + message later, when the mailbox is synced. */ + *new_message_r = TRUE; + } + } + return 0; +} + +bool imapc_mailbox_name_equals(struct imapc_mailbox *mbox, + const char *remote_name) +{ + const char *imapc_remote_name = + imapc_mailbox_get_remote_name(mbox); + + if (strcmp(imapc_remote_name, remote_name) == 0) { + /* match */ + return TRUE; + } else if (strcasecmp(mbox->box.name, "INBOX") == 0 && + strcasecmp(remote_name, "INBOX") == 0) { + /* case-insensitive INBOX */ + return TRUE; + } + return FALSE; +} + +static struct imapc_untagged_fetch_ctx * +imapc_untagged_fetch_ctx_create(void) +{ + pool_t pool = pool_alloconly_create("imapc untagged fetch ctx", 128); + struct imapc_untagged_fetch_ctx *ctx = + p_new(pool, struct imapc_untagged_fetch_ctx, 1); + ctx->pool = pool; + return ctx; +} + +void imapc_untagged_fetch_ctx_free(struct imapc_untagged_fetch_ctx **_ctx) +{ + struct imapc_untagged_fetch_ctx *ctx = *_ctx; + + *_ctx = NULL; + i_assert(ctx != NULL); + + pool_unref(&ctx->pool); +} + +void imapc_untagged_fetch_update_flags(struct imapc_mailbox *mbox, + struct imapc_untagged_fetch_ctx *ctx, + struct mail_index_view *view, + uint32_t lseq) +{ + ARRAY_TYPE(keyword_indexes) old_kws; + struct mail_keywords *kw; + const struct mail_index_record *rec = NULL; + const char *atom; + + if (!ctx->have_flags) + return; + + rec = mail_index_lookup(view, lseq); + if (rec->flags != ctx->flags) { + mail_index_update_flags(mbox->delayed_sync_trans, lseq, + MODIFY_REPLACE, ctx->flags); + } + + t_array_init(&old_kws, 8); + mail_index_lookup_keywords(view, lseq, &old_kws); + + if (ctx->have_gmail_labels) { + /* add keyword for mails that have GMail labels. + this can be used for "All Mail" mailbox migrations + with dsync */ + atom = "$GMailHaveLabels"; + array_push_back(&ctx->keywords, &atom); + } + + array_append_zero(&ctx->keywords); + kw = mail_index_keywords_create(mbox->box.index, + array_front(&ctx->keywords)); + if (!keywords_are_equal(kw, &old_kws)) { + mail_index_update_keywords(mbox->delayed_sync_trans, + lseq, MODIFY_REPLACE, kw); + } + mail_index_keywords_unref(&kw); +} + +static bool imapc_untagged_fetch_handle(struct imapc_mailbox *mbox, + struct imapc_untagged_fetch_ctx *ctx, + uint32_t rseq) +{ + uint32_t lseq; + bool new_message; + + imapc_mailbox_init_delayed_trans(mbox); + if (imapc_mailbox_msgmap_update(mbox, rseq, ctx->fetch_uid, + &lseq, &ctx->uid, + &new_message) < 0 || ctx->uid == 0) + return FALSE; + + if ((ctx->flags & MAIL_RECENT) == 0 && mbox->highest_nonrecent_uid < ctx->uid) { + /* remember for STATUS_FIRST_RECENT_UID */ + mbox->highest_nonrecent_uid = ctx->uid; + } + /* FIXME: we should ideally also pass these through so they show up + to clients. */ + ctx->flags &= ENUM_NEGATE(MAIL_RECENT); + + if (lseq == 0) { + if (!mail_index_lookup_seq(mbox->delayed_sync_view, + ctx->uid, &lseq)) { + /* already expunged by another session */ + if (rseq == mbox->sync_next_rseq) + mbox->sync_next_rseq++; + return new_message; + } + } + + if (rseq == mbox->sync_next_rseq) { + /* we're doing the initial full sync of mails. expunge any + mails that no longer exist. */ + while (mbox->sync_next_lseq < lseq) { + mail_index_lookup_uid(mbox->delayed_sync_view, + mbox->sync_next_lseq, &ctx->uid); + imapc_mailbox_index_expunge(mbox, ctx->uid); + mbox->sync_next_lseq++; + } + i_assert(lseq == mbox->sync_next_lseq); + mbox->sync_next_rseq++; + mbox->sync_next_lseq++; + } + + if (!new_message) { + /* Only update flags immediately for existing messages */ + imapc_untagged_fetch_update_flags(mbox, ctx, + mbox->delayed_sync_view, lseq); + } + + if (ctx->modseq != 0) { + if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < ctx->modseq) + mail_index_update_modseq(mbox->delayed_sync_trans, lseq, ctx->modseq); + array_idx_set(&mbox->rseq_modseqs, rseq-1, &ctx->modseq); + } + if (ctx->guid != NULL) { + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box); + const enum index_cache_field guid_cache_idx = + ibox->cache_fields[MAIL_CACHE_GUID].idx; + + if (mail_cache_field_can_add(mbox->delayed_sync_cache_trans, + lseq, guid_cache_idx)) { + mail_cache_add(mbox->delayed_sync_cache_trans, lseq, + guid_cache_idx, ctx->guid, strlen(ctx->guid)); + } + } + return new_message; +} + +static bool imapc_untagged_fetch_parse(struct imapc_mailbox *mbox, + struct imapc_untagged_fetch_ctx *ctx, + const struct imap_arg *list) +{ + const struct imap_arg *flags_list, *modseq_list; + const char *atom, *patom; + unsigned int i, j; + + ctx->fetch_uid = 0; ctx->flags = 0; + for (i = 0; list[i].type != IMAP_ARG_EOL; i += 2) { + if (!imap_arg_get_atom(&list[i], &atom) || + list[i+1].type == IMAP_ARG_EOL) + return FALSE; + + if (strcasecmp(atom, "UID") == 0) { + if (!imap_arg_get_atom(&list[i+1], &atom) || + str_to_uint32(atom, &ctx->fetch_uid) < 0) + return FALSE; + } else if (strcasecmp(atom, "FLAGS") == 0) { + if (!imap_arg_get_list(&list[i+1], &flags_list)) + return FALSE; + + p_array_init(&ctx->keywords, ctx->pool, 8); + ctx->have_flags = TRUE; + for (j = 0; flags_list[j].type != IMAP_ARG_EOL; j++) { + if (!imap_arg_get_atom(&flags_list[j], &atom)) + return FALSE; + if (atom[0] == '\\') + ctx->flags |= imap_parse_system_flag(atom); + else { + patom = p_strdup(ctx->pool, atom); + /* keyword */ + array_push_back(&ctx->keywords, &patom); + } + } + } else if (strcasecmp(atom, "MODSEQ") == 0 && + imapc_mailbox_has_modseqs(mbox)) { + /* (modseq-number) */ + if (!imap_arg_get_list(&list[i+1], &modseq_list)) + return FALSE; + if (!imap_arg_get_atom(&modseq_list[0], &atom) || + str_to_uint64(atom, &ctx->modseq) < 0 || + modseq_list[1].type != IMAP_ARG_EOL) + return FALSE; + } else if (strcasecmp(atom, "X-GM-MSGID") == 0 && + !mbox->initial_sync_done) { + if (imap_arg_get_atom(&list[i+1], &atom)) + ctx->guid = atom; + } else if (strcasecmp(atom, "X-GM-LABELS") == 0 && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) { + if (!imap_arg_get_list(&list[i+1], &flags_list)) + return FALSE; + for (j = 0; flags_list[j].type != IMAP_ARG_EOL; j++) { + if (!imap_arg_get_astring(&flags_list[j], &atom)) + return FALSE; + if (strcasecmp(atom, "\\Muted") != 0) + ctx->have_gmail_labels = TRUE; + } + } + } + if (ctx->fetch_uid == 0 && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES)) { + /* UID missing and we're not tracking MSNs */ + return FALSE; + } + + return TRUE; +} + +static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + const struct imap_arg *list; + struct imapc_fetch_request *fetch_request; + struct imapc_mail *mail; + bool new_message = FALSE; + + if (mbox == NULL || reply->num == 0 || !imap_arg_get_list(reply->args, &list)) + return; + if (!IMAPC_MAILBOX_IS_FULLY_SELECTED(mbox)) { + /* SELECTing another mailbox - this FETCH is still for the + previous selected mailbox. */ + return; + } + + struct imapc_untagged_fetch_ctx *ctx = + imapc_untagged_fetch_ctx_create(); + if (!imapc_untagged_fetch_parse(mbox, ctx, list)) { + imapc_untagged_fetch_ctx_free(&ctx); + return; + } + + new_message = imapc_untagged_fetch_handle(mbox, ctx, reply->num); + + /* if this is a reply to some FETCH request, update the mail's fields */ + array_foreach_elem(&mbox->fetch_requests, fetch_request) { + array_foreach_elem(&fetch_request->mails, mail) { + if (mail->imail.mail.mail.uid == ctx->uid) + imapc_mail_fetch_update(mail, reply, list); + } + } + + if (!new_message) { + /* Handling this context is finished if the mail was not new + to the local index. It has not been added to + mbox->untagged_fetch_contexts so no need to delete it from + the array. The context itself can be freed here. */ + imapc_untagged_fetch_ctx_free(&ctx); + } else { + /* If this is a new message store this context to be handled + when syncing */ + array_push_back(&mbox->untagged_fetch_contexts, &ctx); + } + imapc_mailbox_idle_notify(mbox); +} + +static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + struct imapc_msgmap *msgmap; + uint32_t uid, rseq = reply->num; + + if (mbox == NULL || rseq == 0 || + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES)) + return; + if (!IMAPC_MAILBOX_IS_FULLY_SELECTED(mbox)) { + /* SELECTing another mailbox - this EXPUNGE is still for the + previous selected mailbox. */ + return; + } + + mbox->prev_skipped_rseq = 0; + mbox->prev_skipped_uid = 0; + + if (mbox->exists_count == 0) { + imapc_mailbox_set_corrupted(mbox, + "EXPUNGE received for empty mailbox"); + return; + } + mbox->exists_count--; + + msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box); + if (rseq > imapc_msgmap_count(msgmap)) { + /* we haven't even seen this message yet */ + return; + } + uid = imapc_msgmap_rseq_to_uid(msgmap, rseq); + imapc_msgmap_expunge(msgmap, rseq); + if (array_is_created(&mbox->rseq_modseqs)) + array_delete(&mbox->rseq_modseqs, rseq-1, 1); + + imapc_mailbox_init_delayed_trans(mbox); + imapc_mailbox_index_expunge(mbox, uid); + imapc_mailbox_idle_notify(mbox); +} + +static void +imapc_untagged_esearch_gmail_pop3(const struct imap_arg *args, + struct imapc_mailbox *mbox) +{ + struct imapc_msgmap *msgmap; + const char *atom; + struct seq_range_iter iter; + ARRAY_TYPE(seq_range) rseqs; + unsigned int n; + uint32_t rseq, lseq, uid; + ARRAY_TYPE(keyword_indexes) keywords; + struct mail_keywords *kw; + unsigned int pop3_deleted_kw_idx; + + i_free_and_null(mbox->sync_gmail_pop3_search_tag); + + /* It should contain ALL <seqset> or nonexistent if nothing matched */ + if (args[0].type == IMAP_ARG_EOL) + return; + t_array_init(&rseqs, 64); + if (!imap_arg_atom_equals(&args[0], "ALL") || + !imap_arg_get_atom(&args[1], &atom) || + imap_seq_set_nostar_parse(atom, &rseqs) < 0) { + i_error("Invalid gmail-pop3 ESEARCH reply"); + return; + } + + mail_index_keyword_lookup_or_create(mbox->box.index, + mbox->storage->set->pop3_deleted_flag, &pop3_deleted_kw_idx); + + t_array_init(&keywords, 1); + array_push_back(&keywords, &pop3_deleted_kw_idx); + kw = mail_index_keywords_create_from_indexes(mbox->box.index, &keywords); + + msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box); + seq_range_array_iter_init(&iter, &rseqs); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &rseq)) { + if (rseq > imapc_msgmap_count(msgmap)) { + /* we haven't even seen this message yet */ + break; + } + uid = imapc_msgmap_rseq_to_uid(msgmap, rseq); + if (!mail_index_lookup_seq(mbox->delayed_sync_view, + uid, &lseq)) + continue; + + /* add the pop3_deleted_flag */ + mail_index_update_keywords(mbox->delayed_sync_trans, + lseq, MODIFY_ADD, kw); + } + mail_index_keywords_unref(&kw); +} + +static void imapc_untagged_search(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + if (mbox == NULL) + return; + if (!IMAPC_MAILBOX_IS_FULLY_SELECTED(mbox)) { + /* SELECTing another mailbox - this SEARCH is still for the + previous selected mailbox. */ + return; + } + imapc_search_reply_search(reply->args, mbox); +} + +static void imapc_untagged_esearch(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + const struct imap_arg *tag_list; + const char *str; + + if (mbox == NULL || !imap_arg_get_list(reply->args, &tag_list)) + return; + if (!IMAPC_MAILBOX_IS_FULLY_SELECTED(mbox)) { + /* SELECTing another mailbox - this ESEARCH is still for the + previous selected mailbox. */ + return; + } + + /* ESEARCH begins with (TAG <tag>) */ + if (!imap_arg_atom_equals(&tag_list[0], "TAG") || + !imap_arg_get_string(&tag_list[1], &str) || + tag_list[2].type != IMAP_ARG_EOL) + return; + + /* for now the only ESEARCH reply that we have is for getting GMail's + list of hidden POP3 messages. */ + if (mbox->sync_gmail_pop3_search_tag != NULL && + strcmp(mbox->sync_gmail_pop3_search_tag, str) == 0) + imapc_untagged_esearch_gmail_pop3(reply->args+1, mbox); + else + imapc_search_reply_esearch(reply->args+1, mbox); +} + +static void imapc_sync_uid_validity(struct imapc_mailbox *mbox) +{ + const struct mail_index_header *hdr; + + imapc_mailbox_init_delayed_trans(mbox); + hdr = mail_index_get_header(mbox->delayed_sync_view); + if (hdr->uid_validity != mbox->sync_uid_validity && + mbox->sync_uid_validity != 0) { + if (hdr->uid_validity != 0) { + /* uidvalidity changed, reset the entire mailbox */ + mail_index_reset(mbox->delayed_sync_trans); + mbox->sync_fetch_first_uid = 1; + /* The reset needs to be committed before FETCH 1:* + results are received. */ + bool changes; + if (imapc_mailbox_commit_delayed_trans(mbox, TRUE, &changes) < 0) + mail_index_mark_corrupted(mbox->box.index); + imapc_mailbox_init_delayed_trans(mbox); + } + mail_index_update_header(mbox->delayed_sync_trans, + offsetof(struct mail_index_header, uid_validity), + &mbox->sync_uid_validity, + sizeof(mbox->sync_uid_validity), TRUE); + } +} + +static void +imapc_resp_text_uidvalidity(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + uint32_t uid_validity; + + if (mbox == NULL || + str_to_uint32(reply->resp_text_value, &uid_validity) < 0 || + uid_validity == 0) + return; + + if (mbox->sync_uid_validity != uid_validity) { + mbox->sync_uid_validity = uid_validity; + imapc_mail_cache_free(&mbox->prev_mail_cache); + imapc_sync_uid_validity(mbox); + } +} + +static void +imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + uint32_t uid_next; + + if (mbox == NULL || + str_to_uint32(reply->resp_text_value, &uid_next) < 0) + return; + + mbox->sync_uid_next = uid_next; +} + +static void +imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + uint64_t highestmodseq; + + if (mbox == NULL || + str_to_uint64(reply->resp_text_value, &highestmodseq) < 0) + return; + + mbox->sync_highestmodseq = highestmodseq; +} + +static void +imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + const struct imap_arg *flags_args, *arg; + const char *flag; + unsigned int idx; + + i_assert(reply->args[0].type == IMAP_ARG_ATOM); + + if (mbox == NULL || !imap_arg_get_list(&reply->args[1], &flags_args)) + return; + + mbox->permanent_flags = 0; + mbox->box.disallow_new_keywords = TRUE; + + for (arg = flags_args; arg->type != IMAP_ARG_EOL; arg++) { + if (!imap_arg_get_atom(arg, &flag)) + continue; + + if (strcmp(flag, "\\*") == 0) + mbox->box.disallow_new_keywords = FALSE; + else if (*flag == '\\') + mbox->permanent_flags |= imap_parse_system_flag(flag); + else { + /* we'll simply make sure that it exists in the index */ + mail_index_keyword_lookup_or_create(mbox->box.index, + flag, &idx); + } + } +} + +void imapc_mailbox_register_untagged(struct imapc_mailbox *mbox, + const char *key, + imapc_mailbox_callback_t *callback) +{ + struct imapc_mailbox_event_callback *cb; + + cb = array_append_space(&mbox->untagged_callbacks); + cb->name = p_strdup(mbox->box.pool, key); + cb->callback = callback; +} + +void imapc_mailbox_register_resp_text(struct imapc_mailbox *mbox, + const char *key, + imapc_mailbox_callback_t *callback) +{ + struct imapc_mailbox_event_callback *cb; + + cb = array_append_space(&mbox->resp_text_callbacks); + cb->name = p_strdup(mbox->box.pool, key); + cb->callback = callback; +} + +void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox) +{ + imapc_mailbox_register_untagged(mbox, "EXISTS", + imapc_untagged_exists); + imapc_mailbox_register_untagged(mbox, "FETCH", + imapc_untagged_fetch); + imapc_mailbox_register_untagged(mbox, "EXPUNGE", + imapc_untagged_expunge); + imapc_mailbox_register_untagged(mbox, "SEARCH", + imapc_untagged_search); + imapc_mailbox_register_untagged(mbox, "ESEARCH", + imapc_untagged_esearch); + imapc_mailbox_register_resp_text(mbox, "UIDVALIDITY", + imapc_resp_text_uidvalidity); + imapc_mailbox_register_resp_text(mbox, "UIDNEXT", + imapc_resp_text_uidnext); + imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ", + imapc_resp_text_highestmodseq); + imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS", + imapc_resp_text_permanentflags); +} diff --git a/src/lib-storage/index/imapc/imapc-save.c b/src/lib-storage/index/imapc/imapc-save.c new file mode 100644 index 0000000..c50f46b --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-save.c @@ -0,0 +1,829 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "imap-date.h" +#include "imap-util.h" +#include "imap-seqset.h" +#include "imap-quote.h" +#include "index-mail.h" +#include "mail-copy.h" +#include "mailbox-list-private.h" +#include "imapc-msgmap.h" +#include "imapc-storage.h" +#include "imapc-sync.h" +#include "imapc-mail.h" +#include "seq-set-builder.h" + +struct imapc_save_context { + struct mail_save_context ctx; + + struct imapc_mailbox *mbox; + struct imapc_mailbox *src_mbox; + struct mail_index_transaction *trans; + + int fd; + char *temp_path; + struct istream *input; + + uint32_t dest_uid_validity; + ARRAY_TYPE(seq_range) dest_saved_uids; + unsigned int save_count; + + bool failed:1; + bool finished:1; +}; + +struct imapc_save_cmd_context { + struct imapc_save_context *ctx; + int ret; +}; + +#define IMAPC_SAVECTX(s) container_of(s, struct imapc_save_context, ctx) +#define IMAPC_SERVER_CMDLINE_MAX_LEN 8000 + +void imapc_transaction_save_rollback(struct mail_save_context *_ctx); +static void imapc_mail_copy_bulk_flush(struct imapc_mailbox *mbox); + +struct mail_save_context * +imapc_save_alloc(struct mailbox_transaction_context *t) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(t->box); + struct imapc_save_context *ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx == NULL) { + ctx = i_new(struct imapc_save_context, 1); + ctx->ctx.transaction = t; + ctx->mbox = mbox; + ctx->src_mbox = NULL; + ctx->trans = t->itrans; + ctx->fd = -1; + t->save_ctx = &ctx->ctx; + } + return t->save_ctx; +} + +int imapc_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + const char *path; + + i_assert(ctx->fd == -1); + + if (imapc_storage_client_handle_auth_failure(ctx->mbox->storage->client)) + return -1; + + ctx->fd = imapc_client_create_temp_fd(ctx->mbox->storage->client->client, + &path); + if (ctx->fd == -1) { + mail_set_critical(_ctx->dest_mail, + "Couldn't create temp file %s", path); + ctx->failed = TRUE; + return -1; + } + /* we may not know the size of the input, or be sure that it contains + only CRLFs. so we'll always first write the mail to a temp file and + upload it from there to remote server. */ + ctx->finished = FALSE; + ctx->temp_path = i_strdup(path); + ctx->input = i_stream_create_crlf(input); + _ctx->data.output = o_stream_create_fd_file(ctx->fd, 0, FALSE); + o_stream_cork(_ctx->data.output); + return 0; +} + +int imapc_save_continue(struct mail_save_context *_ctx) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + + if (ctx->failed) + return -1; + + if (index_storage_save_continue(_ctx, ctx->input, NULL) < 0) { + ctx->failed = TRUE; + return -1; + } + return 0; +} + +static void imapc_save_appenduid(struct imapc_save_context *ctx, + const struct imapc_command_reply *reply, + uint32_t *uid_r) +{ + const char *const *args; + uint32_t uid_validity, dest_uid; + + *uid_r = 0; + + /* <uidvalidity> <dest uid-set> */ + args = t_strsplit(reply->resp_text_value, " "); + if (str_array_length(args) != 2) + return; + + if (str_to_uint32(args[0], &uid_validity) < 0) + return; + if (ctx->dest_uid_validity == 0) + ctx->dest_uid_validity = uid_validity; + else if (ctx->dest_uid_validity != uid_validity) + return; + + if (str_to_uint32(args[1], &dest_uid) == 0) { + seq_range_array_add_with_init(&ctx->dest_saved_uids, + 32, dest_uid); + *uid_r = dest_uid; + } +} + +static void +imapc_save_add_to_index(struct imapc_save_context *ctx, uint32_t uid) +{ + struct mail *_mail = ctx->ctx.dest_mail; + struct index_mail *imail = INDEX_MAIL(_mail); + uint32_t seq; + + /* we'll temporarily append messages and at commit time expunge + them all, since we can't guarantee that no one else has saved + messages to remote server during our transaction */ + mail_index_append(ctx->trans, uid, &seq); + mail_set_seq_saving(_mail, seq); + imail->data.no_caching = TRUE; + imail->data.forced_no_caching = TRUE; + + if (ctx->fd != -1) { + struct imapc_mail *imapc_mail = IMAPC_MAIL(_mail); + imail->data.stream = i_stream_create_fd_autoclose(&ctx->fd, 0); + imapc_mail->header_fetched = TRUE; + imapc_mail->body_fetched = TRUE; + /* The saved stream wasn't actually read, but it needs to be + set accessed to avoid assert-crash. */ + _mail->mail_stream_accessed = TRUE; + imapc_mail_init_stream(imapc_mail); + } + + ctx->save_count++; +} + +static void imapc_save_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_save_cmd_context *ctx = context; + uint32_t uid = 0; + + if (reply->state == IMAPC_COMMAND_STATE_OK) { + if (reply->resp_text_key != NULL && + strcasecmp(reply->resp_text_key, "APPENDUID") == 0) + imapc_save_appenduid(ctx->ctx, reply, &uid); + imapc_save_add_to_index(ctx->ctx, uid); + ctx->ret = 0; + } else if (imapc_storage_client_handle_auth_failure(ctx->ctx->mbox->storage->client)) { + ctx->ret = -1; + } else if (reply->state == IMAPC_COMMAND_STATE_NO) { + imapc_copy_error_from_reply(ctx->ctx->mbox->storage, + MAIL_ERROR_PARAMS, reply); + ctx->ret = -1; + } else { + mailbox_set_critical(&ctx->ctx->mbox->box, + "imapc: APPEND failed: %s", reply->text_full); + ctx->ret = -1; + } + imapc_client_stop(ctx->ctx->mbox->storage->client->client); +} + +static void +imapc_save_noop_callback(const struct imapc_command_reply *reply ATTR_UNUSED, + void *context) +{ + struct imapc_save_cmd_context *ctx = context; + + /* we don't really care about the reply */ + ctx->ret = 0; + imapc_client_stop(ctx->ctx->mbox->storage->client->client); +} + +static void +imapc_copy_rollback_store_callback(const struct imapc_command_reply *reply ATTR_UNUSED, + void *context) +{ + struct imapc_save_context *ctx = context; + /* Can't do much about a non successful STORE here */ + if (reply->state != IMAPC_COMMAND_STATE_OK) { + e_error(ctx->src_mbox->box.event, + "imapc: Failed to set \\Deleted flag for rolling back " + "failed copy: %s", reply->text_full); + ctx->src_mbox->rollback_pending = FALSE; + ctx->finished = TRUE; + ctx->failed = TRUE; + } else { + i_assert(ctx->src_mbox->rollback_pending); + } + /* No need stop the imapc client here there is always an additional + expunge callback after this. */ +} + +static void +imapc_copy_rollback_expunge_callback(const struct imapc_command_reply *reply ATTR_UNUSED, + void *context) +{ + struct imapc_save_context *ctx = context; + + /* Can't do much about a non successful EXPUNGE here */ + if (reply->state != IMAPC_COMMAND_STATE_OK) { + e_error(ctx->src_mbox->box.event, + "imapc: Failed to expunge messages for rolling back " + "failed copy: %s", reply->text_full); + ctx->src_mbox->rollback_pending = FALSE; + ctx->finished = TRUE; + ctx->failed = TRUE; + } else { + ctx->finished = TRUE; + ctx->src_mbox->rollback_pending = FALSE; + } + imapc_client_stop(ctx->src_mbox->storage->client->client); +} + +static void +imapc_append_keywords(string_t *str, struct mail_keywords *kw) +{ + const ARRAY_TYPE(keywords) *kw_arr; + const char *kw_str; + unsigned int i; + + kw_arr = mail_index_get_keywords(kw->index); + for (i = 0; i < kw->count; i++) { + kw_str = array_idx_elem(kw_arr, kw->idx[i]); + if (str_len(str) > 1) + str_append_c(str, ' '); + str_append(str, kw_str); + } +} + +static int imapc_save_append(struct imapc_save_context *ctx) +{ + struct mail_save_context *_ctx = &ctx->ctx; + struct mail_save_data *mdata = &_ctx->data; + struct imapc_command *cmd; + struct imapc_save_cmd_context sctx; + struct istream *input; + const char *flags = "", *internaldate = ""; + + if (mdata->flags != 0 || mdata->keywords != NULL) { + string_t *str = t_str_new(64); + + str_append(str, " ("); + imap_write_flags(str, mdata->flags & ENUM_NEGATE(MAIL_RECENT), + NULL); + if (mdata->keywords != NULL) + imapc_append_keywords(str, mdata->keywords); + str_append_c(str, ')'); + flags = str_c(str); + } + if (mdata->received_date != (time_t)-1) { + internaldate = t_strdup_printf(" \"%s\"", + imap_to_datetime(mdata->received_date)); + } + + ctx->mbox->exists_received = FALSE; + + input = i_stream_create_fd(ctx->fd, IO_BLOCK_SIZE); + sctx.ctx = ctx; + sctx.ret = -2; + cmd = imapc_client_cmd(ctx->mbox->storage->client->client, + imapc_save_callback, &sctx); + imapc_command_sendf(cmd, "APPEND %s%1s%1s %p", + imapc_mailbox_get_remote_name(ctx->mbox), + flags, internaldate, input); + i_stream_unref(&input); + while (sctx.ret == -2) + imapc_mailbox_run(ctx->mbox); + + if (sctx.ret == 0 && ctx->mbox->selected && + !ctx->mbox->exists_received) { + /* e.g. Courier doesn't send EXISTS reply before the tagged + APPEND reply. That isn't exactly required by the IMAP RFC, + but it makes the behavior better. See if NOOP finds + the mail. */ + sctx.ret = -2; + cmd = imapc_client_cmd(ctx->mbox->storage->client->client, + imapc_save_noop_callback, &sctx); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_send(cmd, "NOOP"); + while (sctx.ret == -2) + imapc_mailbox_run(ctx->mbox); + } + return sctx.ret; +} + +int imapc_save_finish(struct mail_save_context *_ctx) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + struct mail_storage *storage = _ctx->transaction->box->storage; + + ctx->finished = TRUE; + + if (!ctx->failed) { + if (o_stream_finish(_ctx->data.output) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(_ctx->dest_mail, + "write(%s) failed: %s", ctx->temp_path, + o_stream_get_error(_ctx->data.output)); + } + ctx->failed = TRUE; + } + } + + if (!ctx->failed) { + if (imapc_save_append(ctx) < 0) + ctx->failed = TRUE; + } + + o_stream_unref(&_ctx->data.output); + i_stream_unref(&ctx->input); + i_close_fd_path(&ctx->fd, ctx->temp_path); + i_free(ctx->temp_path); + index_save_context_free(_ctx); + return ctx->failed ? -1 : 0; +} + +void imapc_save_cancel(struct mail_save_context *_ctx) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)imapc_transaction_save_commit_pre(_ctx); + (void)imapc_save_finish(_ctx); +} + +static void imapc_copy_bulk_finish(struct imapc_save_context *ctx) +{ + while (ctx->src_mbox != NULL && ctx->src_mbox->pending_copy_request != NULL) + imapc_mailbox_run_nofetch(ctx->src_mbox); +} + +int imapc_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + struct mail_transaction_commit_changes *changes = + _ctx->transaction->changes; + uint32_t i, last_seq; + + i_assert(ctx->finished || ctx->failed); + + /* expunge all added messages from index before commit */ + last_seq = mail_index_view_get_messages_count(_ctx->transaction->view); + if (last_seq == 0) + return -1; + for (i = 0; i < ctx->save_count; i++) + mail_index_expunge(ctx->trans, last_seq - i); + + if (!ctx->failed && array_is_created(&ctx->dest_saved_uids)) { + changes->uid_validity = ctx->dest_uid_validity; + array_append_array(&changes->saved_uids, &ctx->dest_saved_uids); + } + return 0; +} + +int imapc_transaction_save_commit(struct mailbox_transaction_context *t) +{ + struct imapc_save_context *ctx = NULL; + struct imapc_mailbox *src_mbox = NULL; + + if (t->save_ctx != NULL) { + ctx = IMAPC_SAVECTX(t->save_ctx); + src_mbox = ctx->src_mbox; + } + + if (src_mbox != NULL && src_mbox->pending_copy_request != NULL) { + /* If there is still a copy command to send flush it now */ + imapc_mail_copy_bulk_flush(src_mbox); + imapc_copy_bulk_finish(ctx); + } + + if (ctx != NULL) + return ctx->failed ? -1 : 0; + return 0; +} + +void imapc_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result ATTR_UNUSED) +{ + imapc_transaction_save_rollback(_ctx); +} + +static void +imapc_expunge_construct_cmd_str(string_t *store_cmd, + string_t *expunge_cmd, + string_t *uids) +{ + str_append(store_cmd, "UID STORE "); + str_append_str(store_cmd, uids); + str_append(store_cmd, " +FLAGS (\\Deleted)"); + str_append(expunge_cmd, "UID EXPUNGE "); + str_append_str(expunge_cmd, uids); + /* Clear already appened uids */ + str_truncate(uids, 0); +} + +static void +imapc_expunge_send_cmd_str(struct imapc_save_context *ctx, + string_t *uids) +{ + struct imapc_command *store_cmd, *expunge_cmd; + + string_t *store_cmd_str, *expunge_cmd_str; + store_cmd_str = t_str_new(128); + expunge_cmd_str = t_str_new(128); + + imapc_expunge_construct_cmd_str(store_cmd_str, expunge_cmd_str, uids); + /* Make sure line length is less than 8k */ + i_assert(str_len(store_cmd_str) < IMAPC_SERVER_CMDLINE_MAX_LEN); + i_assert(str_len(expunge_cmd_str) < IMAPC_SERVER_CMDLINE_MAX_LEN); + + store_cmd = imapc_client_mailbox_cmd(ctx->src_mbox->client_box, + imapc_copy_rollback_store_callback, + ctx); + expunge_cmd = imapc_client_mailbox_cmd(ctx->src_mbox->client_box, + imapc_copy_rollback_expunge_callback, + ctx); + ctx->src_mbox->rollback_pending = TRUE; + imapc_command_send(store_cmd, str_c(store_cmd_str)); + imapc_command_send(expunge_cmd, str_c(expunge_cmd_str)); +} + +static void +imapc_rollback_send_expunge(struct imapc_save_context *ctx) +{ + string_t *uids_str; + struct seqset_builder *seqset_builder; + struct seq_range_iter iter; + unsigned int i = 0; + uint32_t uid; + + if (!array_not_empty(&ctx->src_mbox->copy_rollback_expunge_uids)) + return; + + uids_str = t_str_new(128); + seqset_builder = seqset_builder_init(uids_str); + seq_range_array_iter_init(&iter, &ctx->src_mbox->copy_rollback_expunge_uids); + + /* Iterate over all uids that must be rolled back */ + while (seq_range_array_iter_nth(&iter, i++, &uid)) { + /* Try to add the to the seqset builder while respecting + the maximum length of IMAPC_SERVER_CMDLINE_MAX_LEN. */ + if (!seqset_builder_try_add(seqset_builder, + IMAPC_SERVER_CMDLINE_MAX_LEN - + strlen("UID STORE +FLAGS (\\Deleted)"), + uid)) { + /* Maximum length is reached send the rollback + and wait for it to be finished. */ + imapc_expunge_send_cmd_str(ctx, uids_str); + while (ctx->src_mbox->rollback_pending) + imapc_mailbox_run_nofetch(ctx->src_mbox); + + /* Truncate the uids_str and create a new + seqset_builder for the next command */ + seqset_builder_deinit(&seqset_builder); + str_truncate(uids_str, 0); + seqset_builder = seqset_builder_init(uids_str); + /* Make sure the current uid which is part of + the next uid_str */ + seqset_builder_add(seqset_builder, uid); + } + } + if (str_len(uids_str) > 0) + imapc_expunge_send_cmd_str(ctx, uids_str); + while (ctx->src_mbox->rollback_pending) + imapc_mailbox_run_nofetch(ctx->src_mbox); +} + +static void imapc_copy_bulk_ctx_deinit(struct imapc_save_context *ctx) +{ + /* Clean up the pending copy and the context attached to it */ + str_truncate(ctx->src_mbox->pending_copy_cmd, 0); + i_free(ctx->src_mbox->copy_dest_box); +} + +void imapc_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + + if ((ctx->src_mbox != NULL && ctx->src_mbox->pending_copy_request != NULL) || + !ctx->finished) { + /* There is still a pending copy which should not be send + as rollback() is called or the transaction has not yet + finished and rollback is called */ + ctx->failed = TRUE; + (void)imapc_transaction_save_commit_pre(_ctx); + + i_assert(ctx->finished || ctx->src_mbox != NULL); + /* Clean up the pending copy and the context attached to it */ + if (ctx->src_mbox != NULL) { + if (ctx->src_mbox->pending_copy_request != NULL) { + seqset_builder_deinit(&ctx->src_mbox->pending_copy_request->uidset_builder); + i_free(ctx->src_mbox->pending_copy_request); + } + imapc_copy_bulk_ctx_deinit(ctx); + imapc_client_stop(ctx->src_mbox->storage->client->client); + } + } + + /* Expunge all added messages from index */ + if (ctx->failed && array_is_created(&ctx->dest_saved_uids)) { + i_assert(ctx->src_mbox != NULL); + seq_range_array_merge(&ctx->src_mbox->copy_rollback_expunge_uids, &ctx->dest_saved_uids); + /* Make sure context is not finished already */ + ctx->finished = FALSE; + imapc_rollback_send_expunge(ctx); + array_free(&ctx->dest_saved_uids); + } + + if (ctx->finished || ctx->failed) { + array_free(&ctx->dest_saved_uids); + i_free(ctx); + } +} + +static bool imapc_save_copyuid(struct imapc_save_context *ctx, + const struct imapc_command_reply *reply, + uint32_t *uid_r) +{ + ARRAY_TYPE(seq_range) dest_uidset, source_uidset; + struct seq_range_iter iter; + const char *const *args; + uint32_t uid_validity; + + *uid_r = 0; + + /* <uidvalidity> <source uid-set> <dest uid-set> */ + args = t_strsplit(reply->resp_text_value, " "); + if (str_array_length(args) != 3) + return FALSE; + + if (str_to_uint32(args[0], &uid_validity) < 0) + return FALSE; + if (ctx->dest_uid_validity == 0) + ctx->dest_uid_validity = uid_validity; + else if (ctx->dest_uid_validity != uid_validity) + return FALSE; + + t_array_init(&source_uidset, 8); + t_array_init(&dest_uidset, 8); + + if (imap_seq_set_nostar_parse(args[1], &source_uidset) < 0) + return FALSE; + if (imap_seq_set_nostar_parse(args[2], &dest_uidset) < 0) + return FALSE; + + if (!array_is_created(&ctx->dest_saved_uids)) + i_array_init(&ctx->dest_saved_uids, 8); + + seq_range_array_merge(&ctx->dest_saved_uids, &dest_uidset); + + seq_range_array_iter_init(&iter, &dest_uidset); + (void)seq_range_array_iter_nth(&iter, 0, uid_r); + return TRUE; +} + +static void imapc_copy_set_error(struct imapc_save_context *sctx, + const struct imapc_command_reply *reply) +{ + sctx->failed = TRUE; + + if (reply->state != IMAPC_COMMAND_STATE_BAD) + imapc_copy_error_from_reply(sctx->mbox->storage, + MAIL_ERROR_PARAMS, reply); + else + mailbox_set_critical(&sctx->mbox->box, + "imapc: COPY failed: %s", + reply->text_full); +} + +static void +imapc_copy_simple_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_save_cmd_context *ctx = context; + uint32_t uid = 0; + + if (reply->state == IMAPC_COMMAND_STATE_OK) { + if (reply->resp_text_key != NULL && + strcasecmp(reply->resp_text_key, "COPYUID") == 0) + imapc_save_copyuid(ctx->ctx, reply, &uid); + imapc_save_add_to_index(ctx->ctx, uid); + ctx->ret = 0; + } else if (reply->state == IMAPC_COMMAND_STATE_NO) { + imapc_copy_error_from_reply(ctx->ctx->mbox->storage, + MAIL_ERROR_PARAMS, reply); + ctx->ret = -1; + } else { + mailbox_set_critical(&ctx->ctx->mbox->box, + "imapc: COPY failed: %s", reply->text_full); + ctx->ret = -1; + } + imapc_client_stop(ctx->ctx->mbox->storage->client->client); +} + +static int +imapc_copy_simple(struct mail_save_context *_ctx, struct mail *mail) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + struct imapc_save_cmd_context sctx; + struct imapc_command *cmd; + + sctx.ret = -2; + sctx.ctx = ctx; + cmd = imapc_client_mailbox_cmd(ctx->src_mbox->client_box, + imapc_copy_simple_callback, + &sctx); + imapc_command_sendf(cmd, "UID COPY %u %s", mail->uid, _t->box->name); + while (sctx.ret == -2) + imapc_mailbox_run(ctx->src_mbox); + ctx->finished = TRUE; + return sctx.ret; +} + +static void imapc_copy_bulk_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_copy_request *request = context; + struct imapc_save_context *ctx = request->sctx; + struct imapc_mailbox *mbox = ctx->src_mbox; + unsigned int uid; + + i_assert(mbox != NULL); + i_assert(request == mbox->pending_copy_request); + + /* Check the reply state and add uid's to index and + dest_saved_uids. */ + if (ctx->failed) { + /* If the saving already failed try to find UIDs already + copied from the reply so that rollback can expunge + them */ + if (null_strcasecmp(reply->resp_text_key, "COPYUID") == 0) { + (void)imapc_save_copyuid(ctx, reply, &uid); + imapc_transaction_save_rollback(&ctx->ctx); + } + } else if (reply->state == IMAPC_COMMAND_STATE_OK) { + if (reply->resp_text_key != NULL && + strcasecmp(reply->resp_text_key, "COPYUID") == 0 && + imapc_save_copyuid(ctx, reply, &uid)) { + ctx->finished = TRUE; + } + } else { + imapc_copy_set_error(ctx, reply); + } + + ctx->src_mbox->pending_copy_request = NULL; + i_free(request); + imapc_client_stop(mbox->storage->client->client); +} + +static void imapc_mail_copy_bulk_flush(struct imapc_mailbox *mbox) +{ + struct imapc_command *cmd; + + i_assert(mbox != NULL); + i_assert(mbox->pending_copy_request != NULL); + i_assert(mbox->client_box != NULL); + + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_copy_bulk_callback, + mbox->pending_copy_request); + + seqset_builder_deinit(&mbox->pending_copy_request->uidset_builder); + + str_append(mbox->pending_copy_cmd, " "); + imap_append_astring(mbox->pending_copy_cmd, mbox->copy_dest_box); + + imapc_command_send(cmd, str_c(mbox->pending_copy_cmd)); + + imapc_copy_bulk_ctx_deinit(mbox->pending_copy_request->sctx); +} + +static bool +imapc_mail_copy_bulk_try_merge(struct imapc_mailbox *mbox, uint32_t uid, + const char *box) +{ + i_assert(str_begins(str_c(mbox->pending_copy_cmd), "UID COPY ")); + + if (strcmp(box, mbox->copy_dest_box) != 0) { + /* Not the same mailbox merging not possible */ + return FALSE; + } + return seqset_builder_try_add(mbox->pending_copy_request->uidset_builder, + IMAPC_SERVER_CMDLINE_MAX_LEN, uid); +} + +static void +imapc_mail_copy_bulk_delayed_send_or_merge(struct imapc_save_context *ctx, + uint32_t uid, + const char *box) +{ + struct imapc_mailbox *mbox = ctx->src_mbox; + + if (mbox->pending_copy_request != NULL && + !imapc_mail_copy_bulk_try_merge(mbox, uid, box)) { + /* send the previous COPY and create new one after + waiting for this one to be finished. */ + imapc_mail_copy_bulk_flush(mbox); + imapc_copy_bulk_finish(mbox->pending_copy_request->sctx); + } + if (mbox->pending_copy_request == NULL) { + mbox->pending_copy_request = + i_new(struct imapc_copy_request, 1); + str_printfa(mbox->pending_copy_cmd, "UID COPY "); + mbox->pending_copy_request->uidset_builder = + seqset_builder_init(mbox->pending_copy_cmd); + seqset_builder_add(mbox->pending_copy_request->uidset_builder, + uid); + mbox->copy_dest_box = i_strdup(box); + } else { + i_assert(mbox->pending_copy_request->sctx == ctx); + } + mbox->pending_copy_request->sctx = ctx; +} + +static int +imapc_copy_bulk(struct imapc_save_context *ctx, struct mail *mail) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(ctx->ctx.transaction->box); + + imapc_mail_copy_bulk_delayed_send_or_merge(ctx, mail->uid, + imapc_mailbox_get_remote_name(mbox)); + imapc_save_add_to_index(ctx, 0); + + return ctx->failed ? -1 : 0; +} + +static bool imapc_is_mail_expunged(struct imapc_mailbox *mbox, uint32_t uid) +{ + if (array_is_created(&mbox->delayed_expunged_uids) && + seq_range_exists(&mbox->delayed_expunged_uids, uid)) + return TRUE; + if (mbox->delayed_sync_trans == NULL) + return FALSE; + + struct mail_index_view *view = + mail_index_transaction_get_view(mbox->delayed_sync_trans); + uint32_t seq; + return mail_index_lookup_seq(view, uid, &seq) && + mail_index_transaction_is_expunged(mbox->delayed_sync_trans, seq); +} + +int imapc_copy(struct mail_save_context *_ctx, struct mail *mail) +{ + struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + struct imapc_msgmap *src_msgmap; + uint32_t rseq; + int ret; + + i_assert((_t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (_t->box->storage == mail->box->storage) { + /* Currently we don't support copying mails from multiple + different source mailboxes within the same transaction. */ + i_assert(ctx->src_mbox == NULL || &ctx->src_mbox->box == mail->box); + ctx->src_mbox = IMAPC_MAILBOX(mail->box); + if (!mail->expunged && imapc_is_mail_expunged(ctx->mbox, mail->uid)) + mail_set_expunged(mail); + /* same server, we can use COPY for the mail */ + src_msgmap = + imapc_client_mailbox_get_msgmap(ctx->src_mbox->client_box); + if (mail->expunged || + !imapc_msgmap_uid_to_rseq(src_msgmap, mail->uid, &rseq)) { + mail_storage_set_error(mail->box->storage, + MAIL_ERROR_EXPUNGED, + "Some of the requested messages no longer exist."); + ctx->finished = TRUE; + index_save_context_free(_ctx); + return -1; + } + /* Mail has not been expunged and can be copied. */ + if (ctx->mbox->capabilities == 0) { + /* The destination mailbox has not yet been selected + so the capabilities are unknown */ + if (imapc_client_get_capabilities(ctx->mbox->storage->client->client, + &ctx->mbox->capabilities) < 0) { + mail_storage_set_error(mail->box->storage, + MAIL_ERROR_UNAVAILABLE, + "Failed to determine capabilities for mailbox."); + ctx->finished = TRUE; + index_save_context_free(_ctx); + return -1; + } + } + if ((ctx->mbox->capabilities & IMAPC_CAPABILITY_UIDPLUS) != 0) + ret = imapc_copy_bulk(ctx, mail); + else + ret = imapc_copy_simple(_ctx, mail); + index_save_context_free(_ctx); + return ret; + } + return mail_storage_copy(_ctx, mail); +} diff --git a/src/lib-storage/index/imapc/imapc-search.c b/src/lib-storage/index/imapc/imapc-search.c new file mode 100644 index 0000000..98fa8b5 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-search.c @@ -0,0 +1,332 @@ +/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "imap-arg.h" +#include "imap-seqset.h" +#include "imap-util.h" +#include "mail-search.h" +#include "imapc-msgmap.h" +#include "imapc-storage.h" +#include "imapc-search.h" + +#define IMAPC_SEARCHCTX(obj) \ + MODULE_CONTEXT(obj, imapc_storage_module) + +struct imapc_search_context { + union mail_search_module_context module_ctx; + + ARRAY_TYPE(seq_range) rseqs; + struct seq_range_iter iter; + unsigned int n; + bool finished; + bool success; +}; + +static MODULE_CONTEXT_DEFINE_INIT(imapc_storage_module, + &mail_storage_module_register); + +static bool +imapc_build_search_query_args(struct imapc_mailbox *mbox, + const struct mail_search_arg *args, + bool parent_or, string_t *str); + +static bool imapc_search_is_fast_local(const struct mail_search_arg *args) +{ + const struct mail_search_arg *arg; + + for (arg = args; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_OR: + case SEARCH_SUB: + if (!imapc_search_is_fast_local(arg->value.subargs)) + return FALSE; + break; + case SEARCH_ALL: + case SEARCH_SEQSET: + case SEARCH_UIDSET: + case SEARCH_FLAGS: + case SEARCH_KEYWORDS: + case SEARCH_MODSEQ: + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GUID: + case SEARCH_MAILBOX_GLOB: + case SEARCH_REAL_UID: + break; + default: + return FALSE; + } + } + return TRUE; +} + +static bool +imapc_build_search_query_arg(struct imapc_mailbox *mbox, + const struct mail_search_arg *arg, + string_t *str) +{ + struct mail_search_arg arg2 = *arg; + const char *error; + + if (arg->match_not) + str_append(str, "NOT "); + arg2.match_not = FALSE; + arg = &arg2; + + switch (arg->type) { + case SEARCH_OR: + imapc_build_search_query_args(mbox, arg->value.subargs, TRUE, str); + return TRUE; + case SEARCH_SUB: + str_append_c(str, '('); + imapc_build_search_query_args(mbox, arg->value.subargs, FALSE, str); + str_append_c(str, ')'); + return TRUE; + case SEARCH_SEQSET: + /* translate to UIDs */ + T_BEGIN { + ARRAY_TYPE(seq_range) uids; + + t_array_init(&uids, 64); + mailbox_get_uid_range(&mbox->box, &arg->value.seqset, + &uids); + str_append(str, "UID "); + imap_write_seq_range(str, &uids); + } T_END; + return TRUE; + case SEARCH_BEFORE: + case SEARCH_SINCE: + case SEARCH_ON: + if (arg->type != SEARCH_ON && + (mbox->capabilities & IMAPC_CAPABILITY_WITHIN) == 0) { + /* a bit kludgy way to check this.. */ + size_t pos = str_len(str); + if (!mail_search_arg_to_imap(str, arg, &error)) + return FALSE; + if (strncasecmp(str_c(str) + pos, "OLDER", 5) == 0 || + strncasecmp(str_c(str) + pos, "YOUNGER", 7) == 0) + return FALSE; + return TRUE; + } + if (arg->value.date_type == MAIL_SEARCH_DATE_TYPE_SAVED && + (mbox->capabilities & IMAPC_CAPABILITY_SAVEDATE) == 0) { + /* Fall back to internal date if save date is not + supported. */ + arg2.value.date_type = MAIL_SEARCH_DATE_TYPE_RECEIVED; + } + /* fall through */ + case SEARCH_ALL: + case SEARCH_UIDSET: + case SEARCH_FLAGS: + case SEARCH_KEYWORDS: + case SEARCH_SMALLER: + case SEARCH_LARGER: + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + case SEARCH_BODY: + case SEARCH_TEXT: + return mail_search_arg_to_imap(str, arg, &error); + /* extensions */ + case SEARCH_MODSEQ: + if ((mbox->capabilities & IMAPC_CAPABILITY_CONDSTORE) == 0) + return FALSE; + return mail_search_arg_to_imap(str, arg, &error); + case SEARCH_SAVEDATESUPPORTED: + if ((mbox->capabilities & IMAPC_CAPABILITY_SAVEDATE) == 0) + return FALSE; + return mail_search_arg_to_imap(str, arg, &error); + case SEARCH_INTHREAD: + case SEARCH_GUID: + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GUID: + case SEARCH_MAILBOX_GLOB: + case SEARCH_REAL_UID: + case SEARCH_MIMEPART: + /* not supported for now */ + break; + case SEARCH_NIL: + i_unreached(); + } + return FALSE; +} + +static bool +imapc_build_search_query_args(struct imapc_mailbox *mbox, + const struct mail_search_arg *args, + bool parent_or, string_t *str) +{ + const struct mail_search_arg *arg; + + for (arg = args; arg != NULL; arg = arg->next) { + if (parent_or && arg->next != NULL) + str_append(str, "OR "); + if (!imapc_build_search_query_arg(mbox, arg, str)) + return FALSE; + str_append_c(str, ' '); + } + str_truncate(str, str_len(str)-1); + return TRUE; +} + +static bool imapc_build_search_query(struct imapc_mailbox *mbox, + const struct mail_search_args *args, + const char **query_r) +{ + string_t *str = t_str_new(128); + + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_SEARCH)) { + /* SEARCH command passthrough not enabled */ + return FALSE; + } + if (imapc_search_is_fast_local(args->args)) + return FALSE; + + if ((mbox->capabilities & IMAPC_CAPABILITY_ESEARCH) != 0) + str_append(str, "SEARCH RETURN (ALL) "); + else + str_append(str, "UID SEARCH "); + if (!imapc_build_search_query_args(mbox, args->args, FALSE, str)) + return FALSE; + *query_r = str_c(str); + return TRUE; +} + +static void imapc_search_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct mail_search_context *ctx = context; + struct imapc_mailbox *mbox = IMAPC_MAILBOX(ctx->transaction->box); + struct imapc_search_context *ictx = IMAPC_SEARCHCTX(ctx); + i_assert(ictx != NULL); + + ictx->finished = TRUE; + if (reply->state == IMAPC_COMMAND_STATE_OK) { + seq_range_array_iter_init(&ictx->iter, &ictx->rseqs); + ictx->success = TRUE; + } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + mail_storage_set_internal_error(mbox->box.storage); + } else { + mailbox_set_critical(&mbox->box, + "imapc: Command failed: %s", reply->text_full); + } + imapc_client_stop(mbox->storage->client->client); +} + +struct mail_search_context * +imapc_search_init(struct mailbox_transaction_context *t, + struct mail_search_args *args, + const enum mail_sort_type *sort_program, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(t->box); + struct mail_search_context *ctx; + struct imapc_search_context *ictx; + struct imapc_command *cmd; + const char *search_query; + + ctx = index_storage_search_init(t, args, sort_program, + wanted_fields, wanted_headers); + + if (!imapc_build_search_query(mbox, args, &search_query)) { + /* can't optimize this with SEARCH */ + return ctx; + } + + ictx = i_new(struct imapc_search_context, 1); + i_array_init(&ictx->rseqs, 64); + MODULE_CONTEXT_SET(ctx, imapc_storage_module, ictx); + + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_search_callback, ctx); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_send(cmd, search_query); + + i_assert(mbox->search_ctx == NULL); + mbox->search_ctx = ictx; + while (!ictx->finished) + imapc_client_run(mbox->storage->client->client); + mbox->search_ctx = NULL; + return ctx; +} + +static void imapc_search_set_matches(struct mail_search_arg *args) +{ + for (; args != NULL; args = args->next) { + if (args->type == SEARCH_OR || + args->type == SEARCH_SUB) + imapc_search_set_matches(args->value.subargs); + args->match_always = TRUE; + args->result = 1; + } +} + +bool imapc_search_next_update_seq(struct mail_search_context *ctx) +{ + struct imapc_search_context *ictx = IMAPC_SEARCHCTX(ctx); + + if (ictx == NULL || !ictx->success) + return index_storage_search_next_update_seq(ctx); + + if (!seq_range_array_iter_nth(&ictx->iter, ictx->n++, &ctx->seq)) + return FALSE; + ctx->progress_cur = ctx->seq; + + imapc_search_set_matches(ctx->args->args); + return TRUE; +} + +int imapc_search_deinit(struct mail_search_context *ctx) +{ + struct imapc_search_context *ictx = IMAPC_SEARCHCTX(ctx); + + if (ictx != NULL) { + array_free(&ictx->rseqs); + i_free(ictx); + } + return index_storage_search_deinit(ctx); +} + +void imapc_search_reply_search(const struct imap_arg *args, + struct imapc_mailbox *mbox) +{ + struct imapc_msgmap *msgmap = + imapc_client_mailbox_get_msgmap(mbox->client_box); + const char *atom; + uint32_t uid, rseq; + + if (mbox->search_ctx == NULL) { + i_error("Unexpected SEARCH reply"); + return; + } + + /* we're doing UID SEARCH, so need to convert UIDs to sequences */ + for (unsigned int i = 0; args[i].type != IMAP_ARG_EOL; i++) { + if (!imap_arg_get_atom(&args[i], &atom) || + str_to_uint32(atom, &uid) < 0 || uid == 0) { + i_error("Invalid SEARCH reply"); + break; + } + if (imapc_msgmap_uid_to_rseq(msgmap, uid, &rseq)) + seq_range_array_add(&mbox->search_ctx->rseqs, rseq); + } +} + +void imapc_search_reply_esearch(const struct imap_arg *args, + struct imapc_mailbox *mbox) +{ + const char *atom; + + if (mbox->search_ctx == NULL) { + i_error("Unexpected ESEARCH reply"); + return; + } + + /* It should contain ALL <seqset> or nonexistent if nothing matched */ + if (args[0].type != IMAP_ARG_EOL && + (!imap_arg_atom_equals(&args[0], "ALL") || + !imap_arg_get_atom(&args[1], &atom) || + imap_seq_set_nostar_parse(atom, &mbox->search_ctx->rseqs) < 0)) + i_error("Invalid ESEARCH reply"); +} diff --git a/src/lib-storage/index/imapc/imapc-search.h b/src/lib-storage/index/imapc/imapc-search.h new file mode 100644 index 0000000..0966569 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-search.h @@ -0,0 +1,18 @@ +#ifndef IMAPC_SEARCH_H +#define IMAPC_SEARCH_H + +struct mail_search_context * +imapc_search_init(struct mailbox_transaction_context *t, + struct mail_search_args *args, + const enum mail_sort_type *sort_program, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); +bool imapc_search_next_update_seq(struct mail_search_context *ctx); +int imapc_search_deinit(struct mail_search_context *ctx); + +void imapc_search_reply_search(const struct imap_arg *args, + struct imapc_mailbox *mbox); +void imapc_search_reply_esearch(const struct imap_arg *args, + struct imapc_mailbox *mbox); + +#endif diff --git a/src/lib-storage/index/imapc/imapc-settings.c b/src/lib-storage/index/imapc/imapc-settings.c new file mode 100644 index 0000000..6183f8f --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-settings.c @@ -0,0 +1,173 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "imapc-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct imapc_settings) + +static bool imapc_settings_check(void *_set, pool_t pool, const char **error_r); + +static const struct setting_define imapc_setting_defines[] = { + DEF(STR, imapc_host), + DEF(IN_PORT, imapc_port), + + DEF(STR_VARS, imapc_user), + DEF(STR_VARS, imapc_master_user), + DEF(STR, imapc_password), + DEF(STR, imapc_sasl_mechanisms), + + DEF(ENUM, imapc_ssl), + DEF(BOOL, imapc_ssl_verify), + + DEF(STR, imapc_features), + DEF(STR, imapc_rawlog_dir), + DEF(STR, imapc_list_prefix), + DEF(TIME, imapc_cmd_timeout), + DEF(TIME, imapc_max_idle_time), + DEF(UINT, imapc_connection_retry_count), + DEF(TIME_MSECS, imapc_connection_retry_interval), + DEF(SIZE, imapc_max_line_length), + + DEF(STR, pop3_deleted_flag), + + SETTING_DEFINE_LIST_END +}; + +static const struct imapc_settings imapc_default_settings = { + .imapc_host = "", + .imapc_port = 143, + + .imapc_user = "", + .imapc_master_user = "", + .imapc_password = "", + .imapc_sasl_mechanisms = "", + + .imapc_ssl = "no:imaps:starttls", + .imapc_ssl_verify = TRUE, + + .imapc_features = "", + .imapc_rawlog_dir = "", + .imapc_list_prefix = "", + .imapc_cmd_timeout = 5*60, + .imapc_max_idle_time = 60*29, + .imapc_connection_retry_count = 1, + .imapc_connection_retry_interval = 1000, + .imapc_max_line_length = 0, + + .pop3_deleted_flag = "" +}; + +static const struct setting_parser_info imapc_setting_parser_info = { + .module_name = "imapc", + .defines = imapc_setting_defines, + .defaults = &imapc_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct imapc_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info, + + .check_func = imapc_settings_check +}; + +const struct setting_parser_info *imapc_get_setting_parser_info(void) +{ + return &imapc_setting_parser_info; +} + +/* <settings checks> */ +struct imapc_feature_list { + const char *name; + enum imapc_features num; +}; + +static const struct imapc_feature_list imapc_feature_list[] = { + { "rfc822.size", IMAPC_FEATURE_RFC822_SIZE }, + { "guid-forced", IMAPC_FEATURE_GUID_FORCED }, + { "fetch-headers", IMAPC_FEATURE_FETCH_HEADERS }, + { "gmail-migration", IMAPC_FEATURE_GMAIL_MIGRATION }, + { "search", IMAPC_FEATURE_SEARCH }, + { "zimbra-workarounds", IMAPC_FEATURE_ZIMBRA_WORKAROUNDS }, + { "no-examine", IMAPC_FEATURE_NO_EXAMINE }, + { "proxyauth", IMAPC_FEATURE_PROXYAUTH }, + { "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS }, + { "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS }, + { "modseq", IMAPC_FEATURE_MODSEQ }, + { "delay-login", IMAPC_FEATURE_DELAY_LOGIN }, + { "fetch-bodystructure", IMAPC_FEATURE_FETCH_BODYSTRUCTURE }, + { "send-id", IMAPC_FEATURE_SEND_ID }, + { "fetch-empty-is-expunged", IMAPC_FEATURE_FETCH_EMPTY_IS_EXPUNGED }, + { "no-msn-updates", IMAPC_FEATURE_NO_MSN_UPDATES }, + { "acl", IMAPC_FEATURE_ACL }, + { NULL, 0 } +}; + +static int +imapc_settings_parse_throttle(struct imapc_settings *set, + const char *throttle_str, const char **error_r) +{ + const char *const *tmp; + + tmp = t_strsplit(throttle_str, ":"); + if (str_array_length(tmp) != 3 || + str_to_uint(tmp[0], &set->throttle_init_msecs) < 0 || + str_to_uint(tmp[1], &set->throttle_max_msecs) < 0 || + str_to_uint(tmp[2], &set->throttle_shrink_min_msecs) < 0) { + *error_r = "imapc_features: Invalid throttle settings"; + return -1; + } + return 0; +} + +static int +imapc_settings_parse_features(struct imapc_settings *set, + const char **error_r) +{ + enum imapc_features features = 0; + const struct imapc_feature_list *list; + const char *const *str; + + str = t_strsplit_spaces(set->imapc_features, " ,"); + for (; *str != NULL; str++) { + list = imapc_feature_list; + for (; list->name != NULL; list++) { + if (strcasecmp(*str, list->name) == 0) { + features |= list->num; + break; + } + } + if (strncasecmp(*str, "throttle:", 9) == 0) { + if (imapc_settings_parse_throttle(set, *str + 9, error_r) < 0) + return -1; + continue; + } + if (list->name == NULL) { + *error_r = t_strdup_printf("imapc_features: " + "Unknown feature: %s", *str); + return -1; + } + } + set->parsed_features = features; + return 0; +} + +static bool imapc_settings_check(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) +{ + struct imapc_settings *set = _set; + + if (set->imapc_max_idle_time == 0) { + *error_r = "imapc_max_idle_time must not be 0"; + return FALSE; + } + if (imapc_settings_parse_features(set, error_r) < 0) + return FALSE; + return TRUE; +} diff --git a/src/lib-storage/index/imapc/imapc-settings.h b/src/lib-storage/index/imapc/imapc-settings.h new file mode 100644 index 0000000..e4c1da1 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-settings.h @@ -0,0 +1,63 @@ +#ifndef IMAPC_SETTINGS_H +#define IMAPC_SETTINGS_H + +#include "net.h" + +/* <settings checks> */ +enum imapc_features { + IMAPC_FEATURE_RFC822_SIZE = 0x01, + IMAPC_FEATURE_GUID_FORCED = 0x02, + IMAPC_FEATURE_FETCH_HEADERS = 0x04, + IMAPC_FEATURE_GMAIL_MIGRATION = 0x08, + IMAPC_FEATURE_SEARCH = 0x10, + IMAPC_FEATURE_ZIMBRA_WORKAROUNDS = 0x20, + IMAPC_FEATURE_NO_EXAMINE = 0x40, + IMAPC_FEATURE_PROXYAUTH = 0x80, + IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS = 0x100, + IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS = 0x200, + IMAPC_FEATURE_MODSEQ = 0x400, + IMAPC_FEATURE_DELAY_LOGIN = 0x800, + IMAPC_FEATURE_FETCH_BODYSTRUCTURE = 0x1000, + IMAPC_FEATURE_SEND_ID = 0x2000, + IMAPC_FEATURE_FETCH_EMPTY_IS_EXPUNGED = 0x4000, + IMAPC_FEATURE_NO_MSN_UPDATES = 0x8000, + IMAPC_FEATURE_ACL = 0x10000, +}; +/* </settings checks> */ + +/* + * NOTE: Any additions here should be reflected in imapc_storage_create's + * serialization of settings. + */ +struct imapc_settings { + const char *imapc_host; + in_port_t imapc_port; + + const char *imapc_user; + const char *imapc_master_user; + const char *imapc_password; + const char *imapc_sasl_mechanisms; + + const char *imapc_ssl; + bool imapc_ssl_verify; + + const char *imapc_features; + const char *imapc_rawlog_dir; + const char *imapc_list_prefix; + unsigned int imapc_cmd_timeout; + unsigned int imapc_max_idle_time; + unsigned int imapc_connection_retry_count; + unsigned int imapc_connection_retry_interval; + uoff_t imapc_max_line_length; + + const char *pop3_deleted_flag; + + enum imapc_features parsed_features; + unsigned int throttle_init_msecs; + unsigned int throttle_max_msecs; + unsigned int throttle_shrink_min_msecs; +}; + +const struct setting_parser_info *imapc_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c new file mode 100644 index 0000000..2c9fcdf --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-storage.c @@ -0,0 +1,1353 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "imap-arg.h" +#include "imap-resp-code.h" +#include "mailbox-tree.h" +#include "imapc-connection.h" +#include "imapc-msgmap.h" +#include "imapc-mail.h" +#include "imapc-list.h" +#include "imapc-search.h" +#include "imapc-sync.h" +#include "imapc-settings.h" +#include "imapc-storage.h" + +#define DNS_CLIENT_SOCKET_NAME "dns-client" + +struct imapc_open_context { + struct imapc_mailbox *mbox; + int ret; +}; + +struct imapc_resp_code_map { + const char *code; + enum mail_error error; +}; + +extern struct mail_storage imapc_storage; +extern struct mailbox imapc_mailbox; + +static struct event_category event_category_imapc = { + .name = "imapc", + .parent = &event_category_storage, +}; + +static struct imapc_resp_code_map imapc_resp_code_map[] = { + { IMAP_RESP_CODE_UNAVAILABLE, MAIL_ERROR_TEMP }, + { IMAP_RESP_CODE_AUTHFAILED, MAIL_ERROR_PERM }, + { IMAP_RESP_CODE_AUTHZFAILED, MAIL_ERROR_PERM }, + { IMAP_RESP_CODE_EXPIRED, MAIL_ERROR_PERM }, + { IMAP_RESP_CODE_PRIVACYREQUIRED, MAIL_ERROR_PERM }, + { IMAP_RESP_CODE_CONTACTADMIN, MAIL_ERROR_PERM }, + { IMAP_RESP_CODE_NOPERM, MAIL_ERROR_PERM }, + { IMAP_RESP_CODE_INUSE, MAIL_ERROR_INUSE }, + { IMAP_RESP_CODE_EXPUNGEISSUED, MAIL_ERROR_EXPUNGED }, + { IMAP_RESP_CODE_CORRUPTION, MAIL_ERROR_TEMP }, + { IMAP_RESP_CODE_SERVERBUG, MAIL_ERROR_TEMP }, + /* { IMAP_RESP_CODE_CLIENTBUG, 0 }, */ + { IMAP_RESP_CODE_CANNOT, MAIL_ERROR_NOTPOSSIBLE }, + { IMAP_RESP_CODE_LIMIT, MAIL_ERROR_LIMIT }, + { IMAP_RESP_CODE_OVERQUOTA, MAIL_ERROR_NOQUOTA }, + { IMAP_RESP_CODE_ALREADYEXISTS, MAIL_ERROR_EXISTS }, + { IMAP_RESP_CODE_NONEXISTENT, MAIL_ERROR_NOTFOUND } +}; + +static void imapc_untagged_status(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client); +static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client); +static int imapc_mailbox_run_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r); + +bool imapc_resp_text_code_parse(const char *str, enum mail_error *error_r) +{ + unsigned int i; + + if (str == NULL) + return FALSE; + + for (i = 0; i < N_ELEMENTS(imapc_resp_code_map); i++) { + if (strcmp(imapc_resp_code_map[i].code, str) == 0) { + *error_r = imapc_resp_code_map[i].error; + return TRUE; + } + } + return FALSE; +} + +bool imapc_mail_error_to_resp_text_code(enum mail_error error, const char **str_r) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(imapc_resp_code_map); i++) { + if (imapc_resp_code_map[i].error == error) { + *str_r = imapc_resp_code_map[i].code; + return TRUE; + } + } + return FALSE; +} + +bool imapc_mailbox_has_modseqs(struct imapc_mailbox *mbox) +{ + return (mbox->capabilities & (IMAPC_CAPABILITY_CONDSTORE | + IMAPC_CAPABILITY_QRESYNC)) != 0 && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_MODSEQ); +} + +static struct mail_storage *imapc_storage_alloc(void) +{ + struct imapc_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("imapc storage", 2048); + storage = p_new(pool, struct imapc_storage, 1); + storage->storage = imapc_storage; + storage->storage.pool = pool; + storage->root_ioloop = current_ioloop; + return &storage->storage; +} + +void imapc_copy_error_from_reply(struct imapc_storage *storage, + enum mail_error default_error, + const struct imapc_command_reply *reply) +{ + enum mail_error error; + + if (imapc_resp_text_code_parse(reply->resp_text_key, &error)) { + mail_storage_set_error(&storage->storage, error, + reply->text_without_resp); + } else { + mail_storage_set_error(&storage->storage, default_error, + reply->text_without_resp); + } +} + +void imapc_simple_context_init(struct imapc_simple_context *sctx, + struct imapc_storage_client *client) +{ + i_zero(sctx); + sctx->client = client; + sctx->ret = -2; +} + +void imapc_simple_run(struct imapc_simple_context *sctx, + struct imapc_command **cmd) +{ + if (imapc_storage_client_handle_auth_failure(sctx->client)) { + imapc_command_abort(cmd); + imapc_client_logout(sctx->client->client); + sctx->ret = -1; + } + *cmd = NULL; + while (sctx->ret == -2) + imapc_client_run(sctx->client->client); +} + +void imapc_mailbox_run(struct imapc_mailbox *mbox) +{ + imapc_mail_fetch_flush(mbox); + imapc_mailbox_run_nofetch(mbox); +} + +void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox) +{ + do { + imapc_client_run(mbox->storage->client->client); + } while (mbox->storage->reopen_count > 0 || + mbox->state_fetching_uid1); +} + +void imapc_simple_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_simple_context *ctx = context; + + if (reply->state == IMAPC_COMMAND_STATE_OK) + ctx->ret = 0; + else if (reply->state == IMAPC_COMMAND_STATE_NO) { + imapc_copy_error_from_reply(ctx->client->_storage, + MAIL_ERROR_PARAMS, reply); + ctx->ret = -1; + } else if (imapc_storage_client_handle_auth_failure(ctx->client)) { + ctx->ret = -1; + } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + mail_storage_set_internal_error(&ctx->client->_storage->storage); + ctx->ret = -1; + } else { + mail_storage_set_critical(&ctx->client->_storage->storage, + "imapc: Command failed: %s", reply->text_full); + ctx->ret = -1; + } + imapc_client_stop(ctx->client->client); +} + +void imapc_mailbox_noop(struct imapc_mailbox *mbox) +{ + struct imapc_command *cmd; + struct imapc_simple_context sctx; + + if (mbox->client_box == NULL) { + /* mailbox opening hasn't finished yet */ + return; + } + + imapc_simple_context_init(&sctx, mbox->storage->client); + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_simple_callback, &sctx); + imapc_command_send(cmd, "NOOP"); + imapc_simple_run(&sctx, &cmd); +} + +static void +imapc_storage_client_untagged_cb(const struct imapc_untagged_reply *reply, + void *context) +{ + struct imapc_storage_client *client = context; + struct imapc_mailbox *mbox = reply->untagged_box_context; + const struct imapc_storage_event_callback *cb; + const struct imapc_mailbox_event_callback *mcb; + + array_foreach(&client->untagged_callbacks, cb) { + if (strcasecmp(reply->name, cb->name) == 0) + cb->callback(reply, client); + } + + if (mbox == NULL) + return; + + array_foreach(&mbox->untagged_callbacks, mcb) { + if (strcasecmp(reply->name, mcb->name) == 0) + mcb->callback(reply, mbox); + } + + if (reply->resp_text_key != NULL) { + array_foreach(&mbox->resp_text_callbacks, mcb) { + if (strcasecmp(reply->resp_text_key, mcb->name) == 0) + mcb->callback(reply, mbox); + } + } +} + +static void +imapc_storage_client_login_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_storage_client *client = context; + + client->auth_returned = TRUE; + imapc_client_stop(client->client); + + if (reply->state == IMAPC_COMMAND_STATE_OK) + return; + if (client->destroying && + reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + /* user's work was finished before imapc login finished - + it's not an error */ + return; + } + + client->auth_failed_state = reply->state; + client->auth_failed_reason = i_strdup(reply->text_full); + if (!imapc_storage_client_handle_auth_failure(client)) + i_unreached(); +} + +bool imapc_storage_client_handle_auth_failure(struct imapc_storage_client *client) +{ + if (client->auth_failed_state == IMAPC_COMMAND_STATE_OK) + return FALSE; + + /* We need to set the error to either storage or to list, depending on + whether the caller is from mail-storage.h API or mailbox-list.h API. + We don't know here what the caller is though, so just set the error + to both of them. */ + if (client->_storage != NULL) { + if (client->auth_failed_state == IMAPC_COMMAND_STATE_DISCONNECTED) + mail_storage_set_internal_error(&client->_storage->storage); + else { + mail_storage_set_error(&client->_storage->storage, + MAIL_ERROR_PERM, client->auth_failed_reason); + } + } + if (client->_list != NULL) { + if (client->auth_failed_state == IMAPC_COMMAND_STATE_DISCONNECTED) + mailbox_list_set_internal_error(&client->_list->list); + else { + mailbox_list_set_error(&client->_list->list, + MAIL_ERROR_PERM, client->auth_failed_reason); + } + } + return TRUE; +} + +static void imapc_storage_client_login(struct imapc_storage_client *client, + struct mail_user *user, const char *host) +{ + imapc_client_login(client->client); + if (!user->namespaces_created) { + /* we're still initializing the user. wait for the + login to finish, so we can fail the user creation + if it fails. */ + while (!client->auth_returned) + imapc_client_run(client->client); + if (imapc_storage_client_handle_auth_failure(client)) { + user->error = p_strdup_printf(user->pool, + "imapc: Login to %s failed: %s", + host, client->auth_failed_reason); + } + } +} + +int imapc_storage_client_create(struct mail_namespace *ns, + const struct imapc_settings *imapc_set, + const struct mail_storage_settings *mail_set, + struct imapc_storage_client **client_r, + const char **error_r) +{ + struct imapc_storage_client *client; + struct imapc_client_settings set; + string_t *str; + + i_zero(&set); + set.host = imapc_set->imapc_host; + if (*set.host == '\0') { + *error_r = "missing imapc_host"; + return -1; + } + set.port = imapc_set->imapc_port; + if (imapc_set->imapc_user[0] != '\0') + set.username = imapc_set->imapc_user; + else if (ns->owner != NULL) + set.username = ns->owner->username; + else + set.username = ns->user->username; + set.master_user = imapc_set->imapc_master_user; + set.password = imapc_set->imapc_password; + if (*set.password == '\0') { + *error_r = "missing imapc_password"; + return -1; + } + set.sasl_mechanisms = imapc_set->imapc_sasl_mechanisms; + set.use_proxyauth = (imapc_set->parsed_features & IMAPC_FEATURE_PROXYAUTH) != 0; + set.cmd_timeout_msecs = imapc_set->imapc_cmd_timeout * 1000; + set.connect_retry_count = imapc_set->imapc_connection_retry_count; + set.connect_retry_interval_msecs = imapc_set->imapc_connection_retry_interval; + set.max_idle_time = imapc_set->imapc_max_idle_time; + set.max_line_length = imapc_set->imapc_max_line_length; + set.dns_client_socket_path = *ns->user->set->base_dir == '\0' ? "" : + t_strconcat(ns->user->set->base_dir, "/", + DNS_CLIENT_SOCKET_NAME, NULL); + set.debug = mail_set->mail_debug; + set.rawlog_dir = mail_user_home_expand(ns->user, + imapc_set->imapc_rawlog_dir); + if ((imapc_set->parsed_features & IMAPC_FEATURE_SEND_ID) != 0) + set.session_id_prefix = ns->user->session_id; + + str = t_str_new(128); + mail_user_set_get_temp_prefix(str, ns->user->set); + set.temp_path_prefix = str_c(str); + + mail_user_init_ssl_client_settings(ns->user, &set.ssl_set); + if (!imapc_set->imapc_ssl_verify) + set.ssl_set.allow_invalid_cert = TRUE; + + if (strcmp(imapc_set->imapc_ssl, "imaps") == 0) + set.ssl_mode = IMAPC_CLIENT_SSL_MODE_IMMEDIATE; + else if (strcmp(imapc_set->imapc_ssl, "starttls") == 0) + set.ssl_mode = IMAPC_CLIENT_SSL_MODE_STARTTLS; + else + set.ssl_mode = IMAPC_CLIENT_SSL_MODE_NONE; + + set.throttle_set.init_msecs = imapc_set->throttle_init_msecs; + set.throttle_set.max_msecs = imapc_set->throttle_max_msecs; + set.throttle_set.shrink_min_msecs = imapc_set->throttle_shrink_min_msecs; + + client = i_new(struct imapc_storage_client, 1); + client->refcount = 1; + i_array_init(&client->untagged_callbacks, 16); + /* FIXME: storage->event would be better, but we first get here when + creating mailbox_list, and storage doesn't even exist yet. */ + client->client = imapc_client_init(&set, ns->user->event); + imapc_client_register_untagged(client->client, + imapc_storage_client_untagged_cb, client); + + imapc_client_set_login_callback(client->client, imapc_storage_client_login_callback, client); + + if ((ns->flags & NAMESPACE_FLAG_LIST_PREFIX) != 0 && + (imapc_set->parsed_features & IMAPC_FEATURE_DELAY_LOGIN) == 0) { + /* start logging in immediately */ + imapc_storage_client_login(client, ns->user, set.host); + } + + *client_r = client; + return 0; +} + +void imapc_storage_client_unref(struct imapc_storage_client **_client) +{ + struct imapc_storage_client *client = *_client; + struct imapc_storage_event_callback *cb; + + *_client = NULL; + + i_assert(client->refcount > 0); + if (--client->refcount > 0) + return; + imapc_client_deinit(&client->client); + array_foreach_modifiable(&client->untagged_callbacks, cb) + i_free(cb->name); + array_free(&client->untagged_callbacks); + i_free(client->auth_failed_reason); + i_free(client); +} + +static int +imapc_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, + const char **error_r) +{ + struct imapc_storage *storage = IMAPC_STORAGE(_storage); + struct imapc_mailbox_list *imapc_list = NULL; + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + + /* serialize all the settings */ + _storage->unique_root_dir = p_strdup_printf(_storage->pool, + "%s%s://(%s|%s):%s@%s:%u/%s mechs:%s features:%s " + "rawlog:%s cmd_timeout:%u maxidle:%u maxline:%zuu " + "pop3delflg:%s root_dir:%s", + storage->set->imapc_ssl, + storage->set->imapc_ssl_verify ? "(verify)" : "", + storage->set->imapc_user, + storage->set->imapc_master_user, + storage->set->imapc_password, + storage->set->imapc_host, + storage->set->imapc_port, + storage->set->imapc_list_prefix, + storage->set->imapc_sasl_mechanisms, + storage->set->imapc_features, + storage->set->imapc_rawlog_dir, + storage->set->imapc_cmd_timeout, + storage->set->imapc_max_idle_time, + (size_t) storage->set->imapc_max_line_length, + storage->set->pop3_deleted_flag, + ns->list->set.root_dir); + + if (strcmp(ns->list->name, MAILBOX_LIST_NAME_IMAPC) == 0) { + imapc_list = (struct imapc_mailbox_list *)ns->list; + storage->client = imapc_list->client; + storage->client->refcount++; + } else { + if (imapc_storage_client_create(ns, storage->set, _storage->set, + &storage->client, error_r) < 0) + return -1; + } + storage->client->_storage = storage; + p_array_init(&storage->remote_namespaces, _storage->pool, 4); + if (IMAPC_HAS_FEATURE(storage, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) { + _storage->nonbody_access_fields |= + MAIL_FETCH_IMAP_BODY | MAIL_FETCH_IMAP_BODYSTRUCTURE; + } + + imapc_storage_client_register_untagged(storage->client, "STATUS", + imapc_untagged_status); + imapc_storage_client_register_untagged(storage->client, "NAMESPACE", + imapc_untagged_namespace); + + return 0; +} + +static void imapc_storage_destroy(struct mail_storage *_storage) +{ + struct imapc_storage *storage = IMAPC_STORAGE(_storage); + + storage->client->destroying = TRUE; + + /* make sure all pending commands are aborted before anything is + deinitialized */ + imapc_client_logout(storage->client->client); + + imapc_storage_client_unref(&storage->client); + index_storage_destroy(_storage); +} + +void imapc_storage_client_register_untagged(struct imapc_storage_client *client, + const char *name, + imapc_storage_callback_t *callback) +{ + struct imapc_storage_event_callback *cb; + + cb = array_append_space(&client->untagged_callbacks); + cb->name = i_strdup(name); + cb->callback = callback; +} + +void imapc_storage_client_unregister_untagged(struct imapc_storage_client *client, + const char *name) +{ + struct imapc_storage_event_callback *cb; + unsigned int idx; + array_foreach_modifiable(&client->untagged_callbacks, cb) { + if (strcmp(cb->name, name) == 0) { + idx = array_foreach_idx(&client->untagged_callbacks, cb); + i_free(cb->name); + array_delete(&client->untagged_callbacks, idx, 1); + return; + } + } + i_unreached(); +} + +static void +imapc_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_IMAPC; + set->storage_name_escape_char = IMAPC_LIST_STORAGE_NAME_ESCAPE_CHAR; + /* We want to have all imapc mailboxes accessible, so escape them if + necessary. */ + if (set->vname_escape_char == '\0') + set->vname_escape_char = IMAPC_LIST_VNAME_ESCAPE_CHAR; +} + +static struct mailbox * +imapc_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct imapc_mailbox *mbox; + pool_t pool; + + pool = pool_alloconly_create("imapc mailbox", 1024*4); + mbox = p_new(pool, struct imapc_mailbox, 1); + mbox->box = imapc_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &imapc_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = IMAPC_STORAGE(storage); + + p_array_init(&mbox->untagged_callbacks, pool, 16); + p_array_init(&mbox->resp_text_callbacks, pool, 16); + p_array_init(&mbox->fetch_requests, pool, 16); + p_array_init(&mbox->untagged_fetch_contexts, pool, 16); + p_array_init(&mbox->delayed_expunged_uids, pool, 16); + p_array_init(&mbox->copy_rollback_expunge_uids, pool, 16); + mbox->pending_fetch_cmd = str_new(pool, 128); + mbox->pending_copy_cmd = str_new(pool, 128); + mbox->prev_mail_cache.fd = -1; + imapc_mailbox_register_callbacks(mbox); + return &mbox->box; +} + +const char *imapc_mailbox_get_remote_name(struct imapc_mailbox *mbox) +{ + struct imapc_mailbox_list *list = + container_of(mbox->box.list, struct imapc_mailbox_list, list); + + if (strcmp(mbox->box.list->name, MAILBOX_LIST_NAME_IMAPC) != 0) + return mbox->box.name; + return imapc_list_storage_to_remote_name(list, mbox->box.name); +} + +static int +imapc_mailbox_exists(struct mailbox *box, bool auto_boxes, + enum mailbox_existence *existence_r) +{ + if (auto_boxes && mailbox_is_autocreated(box)) { + *existence_r = MAILBOX_EXISTENCE_SELECT; + return 0; + } + + if (strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPC) != 0) { + if (box->inbox_any) + *existence_r = MAILBOX_EXISTENCE_SELECT; + else + *existence_r = MAILBOX_EXISTENCE_NONE; + return 0; + } + + enum mailbox_info_flags flags; + + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)box->list; + + if (imapc_storage_client_handle_auth_failure(list->client)) { + mail_storage_copy_list_error(box->storage, box->list); + return -1; + } + if (imapc_list_get_mailbox_flags(box->list, box->name, &flags) < 0) { + mail_storage_copy_list_error(box->storage, box->list); + return -1; + } + if ((flags & MAILBOX_NONEXISTENT) != 0) + *existence_r = MAILBOX_EXISTENCE_NONE; + else if ((flags & MAILBOX_NOSELECT) != 0) + *existence_r = MAILBOX_EXISTENCE_NOSELECT; + else + *existence_r = MAILBOX_EXISTENCE_SELECT; + return 0; +} + +static bool imapc_mailbox_want_examine(struct imapc_mailbox *mbox) +{ + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_EXAMINE)) { + /* mainly a Courier-workaround: With POP3-only Maildir that + doesn't have UIDVALIDITY set, EXAMINE won't generate a + permanent UIDVALIDITY while SELECT will. */ + return FALSE; + } + return (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0 && + ((mbox->box.flags & MAILBOX_FLAG_READONLY) != 0 || + (mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0); +} + +static bool +imapc_mailbox_verify_select(struct imapc_mailbox *mbox, const char **error_r) +{ + if (!mbox->exists_received) + *error_r = "EXISTS not received"; + else if (mbox->sync_uid_validity == 0) + *error_r = "UIDVALIDITY not received"; + else + return TRUE; + return FALSE; +} + +static void +imapc_mailbox_reopen_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_mailbox *mbox = context; + const char *errmsg; + + i_assert(mbox->storage->reopen_count > 0); + mbox->storage->reopen_count--; + mbox->selecting = FALSE; + if (reply->state != IMAPC_COMMAND_STATE_OK) + errmsg = reply->text_full; + else if (imapc_mailbox_verify_select(mbox, &errmsg)) { + imap_mailbox_select_finish(mbox); + errmsg = NULL; + } + + if (errmsg != NULL) { + imapc_client_mailbox_reconnect(mbox->client_box, + t_strdup_printf("Reopening mailbox '%s' failed: %s", + mbox->box.name, errmsg)); + } + + imapc_client_stop(mbox->storage->client->client); +} + +static void imapc_mailbox_reopen(void *context) +{ + struct imapc_mailbox *mbox = context; + struct imapc_command *cmd; + + /* we're reconnecting and need to reopen the mailbox */ + mbox->prev_skipped_rseq = 0; + mbox->prev_skipped_uid = 0; + imapc_msgmap_reset(imapc_client_mailbox_get_msgmap(mbox->client_box)); + + if (mbox->selecting) { + /* We reconnected during the initial SELECT/EXAMINE. It'll be + automatically resent by lib-imap-client, so we don't need to + send it again here. */ + i_assert(!mbox->initial_sync_done); + return; + } + if (!mbox->initial_sync_done) { + /* Initial FETCH 1:* didn't fully succeed. We're reconnecting + and lib-imap-client is automatically resending it. But we + need to reset the sync_next_* state so that if any of the + mails are now expunged we won't get confused and crash. */ + mbox->sync_next_lseq = 1; + mbox->sync_next_rseq = 1; + } + + mbox->state_fetched_success = FALSE; + mbox->initial_sync_done = FALSE; + mbox->selecting = TRUE; + mbox->selected = FALSE; + mbox->exists_received = FALSE; + + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_mailbox_reopen_callback, mbox); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT); + if (imapc_mailbox_want_examine(mbox)) { + imapc_command_sendf(cmd, "EXAMINE %s", + imapc_mailbox_get_remote_name(mbox)); + } else { + imapc_command_sendf(cmd, "SELECT %s", + imapc_mailbox_get_remote_name(mbox)); + } + mbox->storage->reopen_count++; +} + +static void +imapc_mailbox_open_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_open_context *ctx = context; + const char *error; + + ctx->mbox->selecting = FALSE; + if (reply->state == IMAPC_COMMAND_STATE_OK) { + if (!imapc_mailbox_verify_select(ctx->mbox, &error)) { + mailbox_set_critical(&ctx->mbox->box, + "imapc: Opening mailbox failed: %s", error); + ctx->ret = -1; + } else { + imap_mailbox_select_finish(ctx->mbox); + ctx->ret = 0; + } + } else if (reply->state == IMAPC_COMMAND_STATE_NO) { + /* Unless the remote IMAP server supports sending + resp-text-code, we don't know if the NO reply is because + the mailbox doesn't exist or because of some internal error. + We'll default to assuming it doesn't exist, so e.g. + mailbox { auto=create } will auto-create missing mailboxes. + However, INBOX is a special mailbox, which is always + autocreated if it doesn't exist. This is true in both the + local Dovecot and the remote IMAP server. This means that + there's no point in trying to send CREATE INBOX to the + remote server. We'll avoid that by defaulting to temporary + failure with INBOX. */ + enum mail_error default_error = + ctx->mbox->box.inbox_any ? + MAIL_ERROR_TEMP : MAIL_ERROR_NOTFOUND; + imapc_copy_error_from_reply(ctx->mbox->storage, + default_error, reply); + ctx->ret = -1; + } else if (imapc_storage_client_handle_auth_failure(ctx->mbox->storage->client)) { + ctx->ret = -1; + } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + ctx->ret = -1; + mail_storage_set_internal_error(ctx->mbox->box.storage); + } else { + mailbox_set_critical(&ctx->mbox->box, + "imapc: Opening mailbox failed: %s", reply->text_full); + ctx->ret = -1; + } + imapc_client_stop(ctx->mbox->storage->client->client); +} + +static int imapc_mailbox_get_capabilities(struct imapc_mailbox *mbox) +{ + /* If authentication failed, don't check again. */ + if (imapc_storage_client_handle_auth_failure(mbox->storage->client)) + return -1; + + return imapc_client_get_capabilities(mbox->storage->client->client, + &mbox->capabilities); + +} + +static void imapc_mailbox_get_extensions(struct imapc_mailbox *mbox) +{ + if (mbox->guid_fetch_field_name == NULL) { + /* see if we can get message GUIDs somehow */ + if ((mbox->capabilities & IMAPC_CAPABILITY_X_GM_EXT_1) != 0) { + /* GMail */ + mbox->guid_fetch_field_name = "X-GM-MSGID"; + } + } +} + +int imapc_mailbox_select(struct imapc_mailbox *mbox) +{ + struct imapc_command *cmd; + struct imapc_open_context ctx; + + i_assert(mbox->client_box == NULL); + + if (imapc_mailbox_get_capabilities(mbox) < 0) + return -1; + + if (imapc_mailbox_has_modseqs(mbox)) { + if (!array_is_created(&mbox->rseq_modseqs)) + i_array_init(&mbox->rseq_modseqs, 32); + else + array_clear(&mbox->rseq_modseqs); + } + + mbox->client_box = + imapc_client_mailbox_open(mbox->storage->client->client, mbox); + imapc_client_mailbox_set_reopen_cb(mbox->client_box, + imapc_mailbox_reopen, mbox); + + imapc_mailbox_get_extensions(mbox); + + mbox->selecting = TRUE; + mbox->exists_received = FALSE; + ctx.mbox = mbox; + ctx.ret = -2; + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_mailbox_open_callback, &ctx); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT | + IMAPC_COMMAND_FLAG_RETRIABLE); + if (imapc_mailbox_want_examine(mbox)) { + imapc_command_sendf(cmd, "EXAMINE %s", + imapc_mailbox_get_remote_name(mbox)); + } else { + imapc_command_sendf(cmd, "SELECT %s", + imapc_mailbox_get_remote_name(mbox)); + } + + while (ctx.ret == -2 || mbox->state_fetching_uid1) + imapc_mailbox_run(mbox); + if (!mbox->state_fetched_success) + ctx.ret = -1; + return ctx.ret; +} + +static int imapc_mailbox_open(struct mailbox *box) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + + if (index_storage_mailbox_open(box, FALSE) < 0) + return -1; + + if (box->deleting || (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) { + /* We don't actually want to SELECT the mailbox. */ + return 0; + } + + if (*box->name == '\0' && + (box->list->ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) { + /* trying to open INBOX as the namespace prefix. + Don't allow this. */ + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + "Mailbox isn't selectable"); + mailbox_close(box); + return -1; + } + + if (imapc_mailbox_select(mbox) < 0) { + mailbox_close(box); + return -1; + } + return 0; +} + +void imapc_mail_cache_free(struct imapc_mail_cache *cache) +{ + i_close_fd(&cache->fd); + buffer_free(&cache->buf); + cache->uid = 0; +} + +static void imapc_mailbox_close(struct mailbox *box) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + bool changes; + + (void)imapc_mailbox_commit_delayed_trans(mbox, FALSE, &changes); + imapc_mail_fetch_flush(mbox); + + /* Arriving here we may have fetch contexts still unprocessed, + if there have been no mailbox_sync() after receiving the untagged replies. + Losing these changes isn't a problem, since the same changes will be found + out after connecting to the server the next time. */ + struct imapc_untagged_fetch_ctx *untagged_fetch_context; + array_foreach_elem(&mbox->untagged_fetch_contexts, untagged_fetch_context) + imapc_untagged_fetch_ctx_free(&untagged_fetch_context); + array_clear(&mbox->untagged_fetch_contexts); + + if (mbox->client_box != NULL) + imapc_client_mailbox_close(&mbox->client_box); + if (array_is_created(&mbox->rseq_modseqs)) + array_free(&mbox->rseq_modseqs); + if (mbox->sync_view != NULL) + mail_index_view_close(&mbox->sync_view); + timeout_remove(&mbox->to_idle_delay); + timeout_remove(&mbox->to_idle_check); + imapc_mail_cache_free(&mbox->prev_mail_cache); + index_storage_mailbox_close(box); +} + +static int +imapc_mailbox_create(struct mailbox *box, + const struct mailbox_update *update ATTR_UNUSED, + bool directory) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + struct imapc_command *cmd; + struct imapc_simple_context sctx; + const char *remote_name = imapc_mailbox_get_remote_name(mbox); + + if (!directory) + ; + else if (strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPC) == 0) { + struct imapc_mailbox_list *imapc_list = + (struct imapc_mailbox_list *)box->list; + remote_name = t_strdup_printf("%s%c", remote_name, + imapc_list->root_sep); + } else { + remote_name = t_strdup_printf("%s%c", remote_name, + mailbox_list_get_hierarchy_sep(box->list)); + } + imapc_simple_context_init(&sctx, mbox->storage->client); + cmd = imapc_client_cmd(mbox->storage->client->client, + imapc_simple_callback, &sctx); + imapc_command_sendf(cmd, "CREATE %s", remote_name); + imapc_simple_run(&sctx, &cmd); + return sctx.ret; +} + +static int imapc_mailbox_update(struct mailbox *box, + const struct mailbox_update *update) +{ + if (!guid_128_is_empty(update->mailbox_guid) || + update->uid_validity != 0 || update->min_next_uid != 0 || + update->min_first_recent_uid != 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "Not supported"); + } + return index_storage_mailbox_update(box, update); +} + +static void imapc_untagged_status(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client) +{ + struct imapc_storage *storage = client->_storage; + struct mailbox_status *status; + const struct imap_arg *list; + const char *remote_name, *key, *value; + uint32_t num; + unsigned int i; + + if (!imap_arg_get_astring(&reply->args[0], &remote_name) || + !imap_arg_get_list(&reply->args[1], &list)) + return; + + if (storage->cur_status_box == NULL) + return; + + if (!imapc_mailbox_name_equals(storage->cur_status_box, + remote_name)) + return; + + status = storage->cur_status; + for (i = 0; list[i].type != IMAP_ARG_EOL; i += 2) { + if (!imap_arg_get_atom(&list[i], &key) || + !imap_arg_get_atom(&list[i+1], &value) || + str_to_uint32(value, &num) < 0) + return; + + if (strcasecmp(key, "MESSAGES") == 0) + status->messages = num; + else if (strcasecmp(key, "RECENT") == 0) + status->recent = num; + else if (strcasecmp(key, "UIDNEXT") == 0) + status->uidnext = num; + else if (strcasecmp(key, "UIDVALIDITY") == 0) + status->uidvalidity = num; + else if (strcasecmp(key, "UNSEEN") == 0) + status->unseen = num; + else if (strcasecmp(key, "HIGHESTMODSEQ") == 0 && + imapc_mailbox_has_modseqs(storage->cur_status_box)) + status->highest_modseq = num; + } +} + +static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client) +{ + struct imapc_storage *storage = client->_storage; + static enum mail_namespace_type ns_types[] = { + MAIL_NAMESPACE_TYPE_PRIVATE, + MAIL_NAMESPACE_TYPE_SHARED, + MAIL_NAMESPACE_TYPE_PUBLIC + }; + struct imapc_namespace *ns; + const struct imap_arg *list, *list2; + const char *prefix, *sep; + unsigned int i; + + array_clear(&storage->remote_namespaces); + for (i = 0; i < N_ELEMENTS(ns_types); i++) { + if (reply->args[i].type == IMAP_ARG_NIL) + continue; + if (!imap_arg_get_list(&reply->args[i], &list)) + break; + + for (; list->type != IMAP_ARG_EOL; list++) { + if (!imap_arg_get_list(list, &list2) || + !imap_arg_get_astring(&list2[0], &prefix) || + !imap_arg_get_nstring(&list2[1], &sep)) + break; + + ns = array_append_space(&storage->remote_namespaces); + ns->prefix = p_strdup(storage->storage.pool, prefix); + ns->separator = sep == NULL ? '\0' : sep[0]; + ns->type = ns_types[i]; + } + } +} + +static void imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox, + enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + index_storage_get_open_status(&mbox->box, items, status_r); + if ((items & STATUS_PERMANENT_FLAGS) != 0) + status_r->permanent_flags = mbox->permanent_flags; + if ((items & STATUS_FIRST_RECENT_UID) != 0) + status_r->first_recent_uid = mbox->highest_nonrecent_uid + 1; + if ((items & STATUS_HIGHESTMODSEQ) != 0) { + /* FIXME: this doesn't work perfectly. we're now just returning + the HIGHESTMODSEQ from the current index, which may or may + not be correct. with QRESYNC enabled we could be returning + sync_highestmodseq, but that would require implementing + VANISHED replies. and without QRESYNC we'd have to issue + STATUS (HIGHESTMODSEQ), which isn't efficient since we get + here constantly (after every IMAP command). */ + } + if (imapc_mailbox_has_modseqs(mbox)) { + /* even if local indexes are only in memory, we still + have modseqs on the IMAP server itself. */ + status_r->nonpermanent_modseqs = FALSE; + } +} + +static int imapc_mailbox_delete(struct mailbox *box) +{ + box->delete_skip_empty_check = TRUE; + return index_storage_mailbox_delete(box); +} + +static int imapc_mailbox_run_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + struct imapc_command *cmd; + struct imapc_simple_context sctx; + string_t *str; + + if (imapc_mailbox_get_capabilities(mbox) < 0) + return -1; + + str = t_str_new(256); + if ((items & STATUS_MESSAGES) != 0) + str_append(str, " MESSAGES"); + if ((items & STATUS_RECENT) != 0) + str_append(str, " RECENT"); + if ((items & STATUS_UIDNEXT) != 0) + str_append(str, " UIDNEXT"); + if ((items & STATUS_UIDVALIDITY) != 0) + str_append(str, " UIDVALIDITY"); + if ((items & STATUS_UNSEEN) != 0) + str_append(str, " UNSEEN"); + if ((items & STATUS_HIGHESTMODSEQ) != 0 && + imapc_mailbox_has_modseqs(mbox)) + str_append(str, " HIGHESTMODSEQ"); + + if (str_len(str) == 0) { + /* nothing requested */ + return 0; + } + + imapc_simple_context_init(&sctx, mbox->storage->client); + mbox->storage->cur_status_box = mbox; + mbox->storage->cur_status = status_r; + cmd = imapc_client_cmd(mbox->storage->client->client, + imapc_simple_callback, &sctx); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_sendf(cmd, "STATUS %s (%1s)", + imapc_mailbox_get_remote_name(mbox), str_c(str)+1); + imapc_simple_run(&sctx, &cmd); + mbox->storage->cur_status_box = NULL; + mbox->storage->cur_status = NULL; + return sctx.ret; +} + +static int imapc_mailbox_get_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + + if (mbox->guid_fetch_field_name != NULL || + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GUID_FORCED)) + status_r->have_guids = TRUE; + + if (box->opened) { + imapc_mailbox_get_selected_status(mbox, items, status_r); + } else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS | + STATUS_PERMANENT_FLAGS | + STATUS_FIRST_RECENT_UID)) != 0) { + /* getting these requires opening the mailbox */ + if (mailbox_open(box) < 0) + return -1; + imapc_mailbox_get_selected_status(mbox, items, status_r); + } else { + if (imapc_mailbox_run_status(box, items, status_r) < 0) + return -1; + /* If this mailbox has private indexes make sure to check + STATUS_UNSEEN from there. */ + if (box->list->set.index_pvt_dir != NULL && + (items & (STATUS_UNSEEN)) != 0) { + struct mailbox_status pvt_idx_status; + index_storage_get_status(box, STATUS_UNSEEN, + &pvt_idx_status); + status_r->unseen = pvt_idx_status.unseen; + } + } + + if (box->opened && !box->deleting && (items & STATUS_UIDNEXT) != 0 && + mbox->sync_uid_next == 0) { + /* Courier-workaround, it doesn't send UIDNEXT on SELECT */ + if (imapc_mailbox_run_status(box, STATUS_UIDNEXT, status_r) < 0) + return -1; + } + return 0; +} + +static int imapc_mailbox_get_namespaces(struct imapc_mailbox *mbox) +{ + struct imapc_storage *storage = mbox->storage; + struct imapc_command *cmd; + struct imapc_simple_context sctx; + + if (storage->namespaces_requested) + return 0; + + if (imapc_mailbox_get_capabilities(mbox) < 0) + return -1; + if ((mbox->capabilities & IMAPC_CAPABILITY_NAMESPACE) == 0) { + /* NAMESPACE capability not supported */ + return 0; + } + + imapc_simple_context_init(&sctx, storage->client); + cmd = imapc_client_cmd(storage->client->client, + imapc_simple_callback, &sctx); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_send(cmd, "NAMESPACE"); + imapc_simple_run(&sctx, &cmd); + + if (sctx.ret < 0) + return -1; + storage->namespaces_requested = TRUE; + return 0; +} + +static const struct imapc_namespace * +imapc_namespace_find_mailbox(struct imapc_storage *storage, + const char *remote_name) +{ + const struct imapc_namespace *ns, *best_ns = NULL; + size_t best_len = UINT_MAX, len; + + array_foreach(&storage->remote_namespaces, ns) { + len = strlen(ns->prefix); + if (str_begins(remote_name, ns->prefix)) { + if (best_len > len) { + best_ns = ns; + best_len = len; + } + } + } + return best_ns; +} + +static int imapc_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + const struct imapc_namespace *ns; + + if ((items & MAILBOX_METADATA_GUID) != 0) { + /* a bit ugly way to do this, but better than nothing for now. + FIXME: if indexes are enabled, keep this there. */ + mail_generate_guid_128_hash(box->name, metadata_r->guid); + items &= ENUM_NEGATE(MAILBOX_METADATA_GUID); + } + if ((items & MAILBOX_METADATA_BACKEND_NAMESPACE) != 0) { + if (imapc_mailbox_get_namespaces(mbox) < 0) + return -1; + + const char *remote_name = imapc_mailbox_get_remote_name(mbox); + ns = imapc_namespace_find_mailbox(mbox->storage, remote_name); + if (ns != NULL) { + metadata_r->backend_ns_prefix = ns->prefix; + metadata_r->backend_ns_type = ns->type; + } + items &= ENUM_NEGATE(MAILBOX_METADATA_BACKEND_NAMESPACE); + } + if (items != 0) { + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + } + return 0; +} + +static void imapc_noop_callback(const struct imapc_command_reply *reply, + void *context) + +{ + struct imapc_storage *storage = context; + + if (reply->state == IMAPC_COMMAND_STATE_OK) + ; + else if (reply->state == IMAPC_COMMAND_STATE_NO) + imapc_copy_error_from_reply(storage, MAIL_ERROR_PARAMS, reply); + else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) + mail_storage_set_internal_error(&storage->storage); + else { + mail_storage_set_critical(&storage->storage, + "imapc: NOOP failed: %s", reply->text_full); + } +} + +static void imapc_idle_timeout(struct imapc_mailbox *mbox) +{ + struct imapc_command *cmd; + + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_noop_callback, mbox->storage); + imapc_command_send(cmd, "NOOP"); +} + +static void imapc_idle_noop_callback(const struct imapc_command_reply *reply, + void *context) + +{ + struct imapc_mailbox *mbox = context; + + imapc_noop_callback(reply, mbox->box.storage); + if (mbox->client_box != NULL) + imapc_client_mailbox_idle(mbox->client_box); +} + +static void imapc_notify_changes(struct mailbox *box) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + const struct mail_storage_settings *set = box->storage->set; + struct imapc_command *cmd; + + if (box->notify_callback == NULL) { + timeout_remove(&mbox->to_idle_check); + return; + } + + if ((mbox->capabilities & IMAPC_CAPABILITY_IDLE) != 0) { + /* remote server is already in IDLE. but since some servers + don't notice changes immediately, we'll force them to check + here by sending a NOOP. this helps with clients that break + IDLE when clicking "get mail". */ + cmd = imapc_client_mailbox_cmd(mbox->client_box, + imapc_idle_noop_callback, mbox); + imapc_command_send(cmd, "NOOP"); + } else { + /* remote server doesn't support IDLE. + check for changes with NOOP every once in a while. */ + i_assert(!imapc_client_is_running(mbox->storage->client->client)); + mbox->to_idle_check = + timeout_add(set->mailbox_idle_check_interval * 1000, + imapc_idle_timeout, mbox); + } +} + +static bool imapc_is_inconsistent(struct mailbox *box) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + + if (box->view != NULL && + mail_index_view_is_inconsistent(box->view)) + return TRUE; + + return mbox->client_box == NULL ? FALSE : + !imapc_client_mailbox_is_opened(mbox->client_box); +} + +struct mail_storage imapc_storage = { + .name = IMAPC_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT | + MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT | + MAIL_STORAGE_CLASS_FLAG_SECONDARY_INDEX, + .event_category = &event_category_imapc, + + .v = { + imapc_get_setting_parser_info, + imapc_storage_alloc, + imapc_storage_create, + imapc_storage_destroy, + NULL, + imapc_storage_get_list_settings, + NULL, + imapc_mailbox_alloc, + NULL, + NULL, + } +}; + +static int +imapc_mailbox_transaction_commit(struct mailbox_transaction_context *t, + struct mail_transaction_commit_changes *changes_r) +{ + int ret = imapc_transaction_save_commit(t); + int ret2 = index_transaction_commit(t, changes_r); + return ret >= 0 && ret2 >= 0 ? 0 : -1; +} + +struct mailbox imapc_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + imapc_mailbox_exists, + imapc_mailbox_open, + imapc_mailbox_close, + index_storage_mailbox_free, + imapc_mailbox_create, + imapc_mailbox_update, + imapc_mailbox_delete, + index_storage_mailbox_rename, + imapc_mailbox_get_status, + imapc_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + NULL, + NULL, + imapc_mailbox_sync_init, + index_mailbox_sync_next, + imapc_mailbox_sync_deinit, + NULL, + imapc_notify_changes, + index_transaction_begin, + imapc_mailbox_transaction_commit, + index_transaction_rollback, + NULL, + imapc_mail_alloc, + imapc_search_init, + imapc_search_deinit, + index_storage_search_next_nonblock, + imapc_search_next_update_seq, + index_storage_search_next_match_mail, + imapc_save_alloc, + imapc_save_begin, + imapc_save_continue, + imapc_save_finish, + imapc_save_cancel, + imapc_copy, + imapc_transaction_save_commit_pre, + imapc_transaction_save_commit_post, + imapc_transaction_save_rollback, + imapc_is_inconsistent + } +}; diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h new file mode 100644 index 0000000..6624a0d --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-storage.h @@ -0,0 +1,274 @@ +#ifndef IMAPC_STORAGE_H +#define IMAPC_STORAGE_H + +#include "index-storage.h" +#include "imapc-settings.h" +#include "imapc-client.h" + +#define IMAPC_STORAGE_NAME "imapc" +/* storage_name separator */ +#define IMAPC_LIST_STORAGE_NAME_ESCAPE_CHAR '%' +/* fs_name separator */ +#define IMAPC_LIST_FS_NAME_ESCAPE_CHAR '%' +/* vname separator */ +#define IMAPC_LIST_VNAME_ESCAPE_CHAR '~' + +struct imap_arg; +struct imapc_untagged_reply; +struct imapc_command_reply; +struct imapc_mailbox; +struct imapc_storage_client; + +typedef void imapc_storage_callback_t(const struct imapc_untagged_reply *reply, + struct imapc_storage_client *client); +typedef void imapc_mailbox_callback_t(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox); + +struct imapc_storage_event_callback { + char *name; + imapc_storage_callback_t *callback; +}; + +struct imapc_mailbox_event_callback { + const char *name; + imapc_mailbox_callback_t *callback; +}; + +#define IMAPC_HAS_FEATURE(mstorage, feature) \ + (((mstorage)->set->parsed_features & feature) != 0) +#define IMAPC_BOX_HAS_FEATURE(mbox, feature) \ + (((mbox)->storage->set->parsed_features & feature) != 0) + +/* Returns TRUE if we can assume from now on that untagged EXPUNGE, FETCH, etc. + replies belong to this mailbox instead of to the previously selected + mailbox. */ +#define IMAPC_MAILBOX_IS_FULLY_SELECTED(mbox) \ + ((mbox)->sync_uid_validity != 0) + +struct imapc_namespace { + const char *prefix; + char separator; + enum mail_namespace_type type; +}; + +struct imapc_storage_client { + int refcount; + + /* either one of these may not be available: */ + struct imapc_storage *_storage; + struct imapc_mailbox_list *_list; + + struct imapc_client *client; + + ARRAY(struct imapc_storage_event_callback) untagged_callbacks; + + /* IMAPC_COMMAND_STATE_OK if no auth failure (yet), otherwise result to + the LOGIN/AUTHENTICATE command. */ + enum imapc_command_state auth_failed_state; + char *auth_failed_reason; + + /* Authentication reply was received (success or failure) */ + bool auth_returned:1; + bool destroying:1; +}; + +struct imapc_storage { + struct mail_storage storage; + const struct imapc_settings *set; + + struct ioloop *root_ioloop; + struct imapc_storage_client *client; + + struct imapc_mailbox *cur_status_box; + struct mailbox_status *cur_status; + unsigned int reopen_count; + + ARRAY(struct imapc_namespace) remote_namespaces; + + bool namespaces_requested:1; +}; + +struct imapc_mail_cache { + uint32_t uid; + + /* either fd != -1 or buf != NULL */ + int fd; + buffer_t *buf; +}; + +struct imapc_fetch_request { + ARRAY(struct imapc_mail *) mails; +}; + +struct imapc_untagged_fetch_ctx { + pool_t pool; + + /* keywords, flags, guid, modseq and fetch_uid may or may not be + received with an untagged fetch response */ + ARRAY_TYPE(const_string) keywords; + /* Is set if have_flags is TRUE */ + enum mail_flags flags; + const char *guid; + uint64_t modseq; + uint32_t fetch_uid; + + /* uid is generated locally based on the remote MSN or fetch_uid */ + uint32_t uid; + + bool have_gmail_labels:1; + bool have_flags:1; +}; + +struct imapc_copy_request { + struct imapc_save_context *sctx; + struct seqset_builder *uidset_builder; +}; + +struct imapc_mailbox { + struct mailbox box; + struct imapc_storage *storage; + struct imapc_client_mailbox *client_box; + enum imapc_capability capabilities; + + struct mail_index_transaction *delayed_sync_trans; + struct mail_index_view *sync_view, *delayed_sync_view; + struct mail_cache_view *delayed_sync_cache_view; + struct mail_cache_transaction_ctx *delayed_sync_cache_trans; + struct timeout *to_idle_check, *to_idle_delay; + + ARRAY(struct imapc_fetch_request *) fetch_requests; + ARRAY(struct imapc_untagged_fetch_ctx *) untagged_fetch_contexts; + /* if non-empty, contains the latest FETCH command we're going to be + sending soon (but still waiting to see if we can increase its + UID range) */ + string_t *pending_fetch_cmd; + /* if non-empty, contains the latest COPY command we're going to be + sending soon. */ + string_t *pending_copy_cmd; + char *copy_dest_box; + struct imapc_fetch_request *pending_fetch_request; + struct imapc_copy_request *pending_copy_request; + struct timeout *to_pending_fetch_send; + + ARRAY(struct imapc_mailbox_event_callback) untagged_callbacks; + ARRAY(struct imapc_mailbox_event_callback) resp_text_callbacks; + + enum mail_flags permanent_flags; + uint32_t highest_nonrecent_uid; + + ARRAY(uint64_t) rseq_modseqs; + ARRAY_TYPE(seq_range) delayed_expunged_uids; + ARRAY_TYPE(seq_range) copy_rollback_expunge_uids; + uint32_t sync_uid_validity; + uint32_t sync_uid_next; + uint64_t sync_highestmodseq; + uint32_t sync_fetch_first_uid; + uint32_t sync_next_lseq; + uint32_t sync_next_rseq; + uint32_t exists_count; + uint32_t min_append_uid; + char *sync_gmail_pop3_search_tag; + + /* keep the previous fetched message body cached, + mainly for partial IMAP fetches */ + struct imapc_mail_cache prev_mail_cache; + + uint32_t prev_skipped_rseq, prev_skipped_uid; + struct imapc_sync_context *sync_ctx; + + const char *guid_fetch_field_name; + struct imapc_search_context *search_ctx; + + bool selecting:1; + bool syncing:1; + bool initial_sync_done:1; + bool selected:1; + bool exists_received:1; + bool state_fetching_uid1:1; + bool state_fetched_success:1; + bool rollback_pending:1; + bool delayed_untagged_exists:1; +}; + +struct imapc_simple_context { + struct imapc_storage_client *client; + int ret; +}; + +#define IMAPC_STORAGE(s) container_of(s, struct imapc_storage, storage) +#define IMAPC_MAILBOX(s) container_of(s, struct imapc_mailbox, box) + +int imapc_storage_client_create(struct mail_namespace *ns, + const struct imapc_settings *imapc_set, + const struct mail_storage_settings *mail_set, + struct imapc_storage_client **client_r, + const char **error_r); +void imapc_storage_client_unref(struct imapc_storage_client **client); +bool imapc_storage_client_handle_auth_failure(struct imapc_storage_client *client); + +struct mail_save_context * +imapc_save_alloc(struct mailbox_transaction_context *_t); +int imapc_save_begin(struct mail_save_context *ctx, struct istream *input); +int imapc_save_continue(struct mail_save_context *ctx); +int imapc_save_finish(struct mail_save_context *ctx); +void imapc_save_cancel(struct mail_save_context *ctx); +int imapc_copy(struct mail_save_context *ctx, struct mail *mail); + +int imapc_transaction_save_commit(struct mailbox_transaction_context *t); +int imapc_transaction_save_commit_pre(struct mail_save_context *ctx); +void imapc_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void imapc_transaction_save_rollback(struct mail_save_context *ctx); + +void imapc_mailbox_run(struct imapc_mailbox *mbox); +void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox); +void imapc_mail_cache_free(struct imapc_mail_cache *cache); +int imapc_mailbox_select(struct imapc_mailbox *mbox); +void imap_mailbox_select_finish(struct imapc_mailbox *mbox); + +bool imapc_mailbox_has_modseqs(struct imapc_mailbox *mbox); +bool imapc_resp_text_code_parse(const char *str, enum mail_error *error_r); +bool imapc_mail_error_to_resp_text_code(enum mail_error error, const char **str_r); +void imapc_copy_error_from_reply(struct imapc_storage *storage, + enum mail_error default_error, + const struct imapc_command_reply *reply); +void imapc_simple_context_init(struct imapc_simple_context *sctx, + struct imapc_storage_client *client); +void imapc_simple_run(struct imapc_simple_context *sctx, + struct imapc_command **cmd); +void imapc_simple_callback(const struct imapc_command_reply *reply, + void *context); +int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox, + bool force, bool *changes_r); +bool imapc_mailbox_name_equals(struct imapc_mailbox *mbox, + const char *remote_name); +void imapc_mailbox_noop(struct imapc_mailbox *mbox); +void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox, + const char *reason, ...) ATTR_FORMAT(2, 3); +const char *imapc_mailbox_get_remote_name(struct imapc_mailbox *mbox); + +void imapc_storage_client_register_untagged(struct imapc_storage_client *client, + const char *name, + imapc_storage_callback_t *callback); +void imapc_storage_client_unregister_untagged(struct imapc_storage_client *client, + const char *name); +void imapc_mailbox_register_untagged(struct imapc_mailbox *mbox, + const char *name, + imapc_mailbox_callback_t *callback); +void imapc_mailbox_register_resp_text(struct imapc_mailbox *mbox, + const char *key, + imapc_mailbox_callback_t *callback); + +void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox); + +struct mail_index_view * +imapc_mailbox_get_sync_view(struct imapc_mailbox *mbox); + +void imapc_untagged_fetch_ctx_free(struct imapc_untagged_fetch_ctx **_ctx); +void imapc_untagged_fetch_update_flags(struct imapc_mailbox *mbox, + struct imapc_untagged_fetch_ctx *ctx, + struct mail_index_view *view, + uint32_t lseq); +bool imapc_mailbox_fetch_state(struct imapc_mailbox *mbox, uint32_t first_uid); + +#endif diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c new file mode 100644 index 0000000..3130121 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-sync.c @@ -0,0 +1,702 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "hash.h" +#include "str.h" +#include "sort.h" +#include "imap-util.h" +#include "mail-cache.h" +#include "mail-index-modseq.h" +#include "index-sync-private.h" +#include "imapc-msgmap.h" +#include "imapc-list.h" +#include "imapc-storage.h" +#include "imapc-sync.h" + +struct imapc_sync_command { + struct imapc_sync_context *ctx; + char *cmd_str; + bool ignore_no; +}; + +static void imapc_sync_callback(const struct imapc_command_reply *reply, + void *context) +{ + struct imapc_sync_command *cmd = context; + struct imapc_sync_context *ctx = cmd->ctx; + + i_assert(ctx->sync_command_count > 0); + + if (reply->state == IMAPC_COMMAND_STATE_OK) + ; + else if (reply->state == IMAPC_COMMAND_STATE_NO && cmd->ignore_no) { + /* maybe the message was expunged already. + some servers fail STOREs with NO in such situation. */ + } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + /* the disconnection is already logged, don't flood + the logs unnecessarily */ + mail_storage_set_internal_error(&ctx->mbox->storage->storage); + ctx->failed = TRUE; + } else { + mailbox_set_critical(&ctx->mbox->box, + "imapc: Sync command '%s' failed: %s", + cmd->cmd_str, reply->text_full); + ctx->failed = TRUE; + } + + if (--ctx->sync_command_count == 0) + imapc_client_stop(ctx->mbox->storage->client->client); + i_free(cmd->cmd_str); + i_free(cmd); +} + +static struct imapc_command * +imapc_sync_cmd_full(struct imapc_sync_context *ctx, const char *cmd_str, + bool ignore_no) +{ + struct imapc_sync_command *sync_cmd; + struct imapc_command *cmd; + + sync_cmd = i_new(struct imapc_sync_command, 1); + sync_cmd->ctx = ctx; + sync_cmd->cmd_str = i_strdup(cmd_str); + sync_cmd->ignore_no = ignore_no; + + ctx->sync_command_count++; + cmd = imapc_client_mailbox_cmd(ctx->mbox->client_box, + imapc_sync_callback, sync_cmd); + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + imapc_command_send(cmd, cmd_str); + return cmd; +} + +static struct imapc_command * +imapc_sync_cmd(struct imapc_sync_context *ctx, const char *cmd_str) +{ + return imapc_sync_cmd_full(ctx, cmd_str, FALSE); +} + +static unsigned int imapc_sync_store_hash(const struct imapc_sync_store *store) +{ + return str_hash(store->flags) ^ store->modify_type; +} + +static int imapc_sync_store_cmp(const struct imapc_sync_store *store1, + const struct imapc_sync_store *store2) +{ + if (store1->modify_type != store2->modify_type) + return 1; + return strcmp(store1->flags, store2->flags); +} + +static const char *imapc_sync_flags_sort(const char *flags) +{ + if (strchr(flags, ' ') == NULL) + return flags; + + const char **str = t_strsplit(flags, " "); + i_qsort(str, str_array_length(str), sizeof(const char *), + i_strcasecmp_p); + return t_strarray_join(str, " "); +} + +static void +imapc_sync_store_flush(struct imapc_sync_context *ctx) +{ + struct imapc_sync_store *store; + const char *sorted_flags; + + if (ctx->prev_uid1 == 0) + return; + + sorted_flags = imapc_sync_flags_sort(str_c(ctx->prev_flags)); + struct imapc_sync_store store_lookup = { + .modify_type = ctx->prev_modify_type, + .flags = sorted_flags, + }; + store = hash_table_lookup(ctx->stores, &store_lookup); + if (store == NULL) { + store = p_new(ctx->pool, struct imapc_sync_store, 1); + store->modify_type = ctx->prev_modify_type; + store->flags = p_strdup(ctx->pool, sorted_flags); + p_array_init(&store->uids, ctx->pool, 4); + hash_table_insert(ctx->stores, store, store); + } + seq_range_array_add_range(&store->uids, ctx->prev_uid1, ctx->prev_uid2); +} + +static void +imapc_sync_store(struct imapc_sync_context *ctx, + enum modify_type modify_type, uint32_t uid1, uint32_t uid2, + const char *flags) +{ + if (ctx->prev_flags == NULL) { + ctx->prev_flags = str_new(ctx->pool, 128); + hash_table_create(&ctx->stores, ctx->pool, 0, + imapc_sync_store_hash, imapc_sync_store_cmp); + } + + if (ctx->prev_uid1 != uid1 || ctx->prev_uid2 != uid2 || + ctx->prev_modify_type != modify_type) { + imapc_sync_store_flush(ctx); + ctx->prev_uid1 = uid1; + ctx->prev_uid2 = uid2; + ctx->prev_modify_type = modify_type; + str_truncate(ctx->prev_flags, 0); + } + if (str_len(ctx->prev_flags) > 0) + str_append_c(ctx->prev_flags, ' '); + str_append(ctx->prev_flags, flags); +} + +static void +imapc_sync_finish_store(struct imapc_sync_context *ctx) +{ + struct hash_iterate_context *iter; + struct imapc_sync_store *store; + string_t *cmd = t_str_new(128); + + imapc_sync_store_flush(ctx); + + if (!hash_table_is_created(ctx->stores)) + return; + + iter = hash_table_iterate_init(ctx->stores); + while (hash_table_iterate(iter, ctx->stores, &store, &store)) { + str_truncate(cmd, 0); + str_append(cmd, "UID STORE "); + imap_write_seq_range(cmd, &store->uids); + str_printfa(cmd, " %cFLAGS (%s)", + store->modify_type == MODIFY_ADD ? '+' : '-', + store->flags); + imapc_sync_cmd_full(ctx, str_c(cmd), TRUE); + } + hash_table_iterate_deinit(&iter); + hash_table_destroy(&ctx->stores); +} + +static void +imapc_sync_add_missing_deleted_flags(struct imapc_sync_context *ctx, + uint32_t seq1, uint32_t seq2) +{ + const struct mail_index_record *rec; + uint32_t seq, uid1, uid2; + + /* if any of them has a missing \Deleted flag, + just add it to all of them. */ + for (seq = seq1; seq <= seq2; seq++) { + rec = mail_index_lookup(ctx->sync_view, seq); + if ((rec->flags & MAIL_DELETED) == 0) + break; + } + + if (seq <= seq2) { + mail_index_lookup_uid(ctx->sync_view, seq1, &uid1); + mail_index_lookup_uid(ctx->sync_view, seq2, &uid2); + + imapc_sync_store(ctx, MODIFY_ADD, uid1, uid2, "\\Deleted"); + } +} + +static void imapc_sync_index_flags(struct imapc_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec) +{ + string_t *str = t_str_new(128); + + i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); + + if (sync_rec->add_flags != 0) { + i_assert((sync_rec->add_flags & MAIL_RECENT) == 0); + + imap_write_flags(str, sync_rec->add_flags, NULL); + imapc_sync_store(ctx, MODIFY_ADD, sync_rec->uid1, + sync_rec->uid2, str_c(str)); + } + + if (sync_rec->remove_flags != 0) { + i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0); + str_truncate(str, 0); + imap_write_flags(str, sync_rec->remove_flags, NULL); + imapc_sync_store(ctx, MODIFY_REMOVE, sync_rec->uid1, + sync_rec->uid2, str_c(str)); + } +} + +static void +imapc_sync_index_keyword(struct imapc_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec) +{ + const char *kw_str; + enum modify_type modify_type; + + switch (sync_rec->type) { + case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + modify_type = MODIFY_ADD; + break; + case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + modify_type = MODIFY_REMOVE; + break; + default: + i_unreached(); + } + + kw_str = array_idx_elem(ctx->keywords, sync_rec->keyword_idx); + imapc_sync_store(ctx, modify_type, sync_rec->uid1, + sync_rec->uid2, kw_str); +} + +static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx) +{ + string_t *str; + + if (array_count(&ctx->expunged_uids) == 0) + return; + + if ((ctx->mbox->capabilities & IMAPC_CAPABILITY_UIDPLUS) == 0) { + /* just expunge everything */ + imapc_sync_cmd(ctx, "EXPUNGE"); + return; + } + + /* build a list of UIDs to expunge */ + str = t_str_new(128); + str_append(str, "UID EXPUNGE "); + imap_write_seq_range(str, &ctx->expunged_uids); + imapc_sync_cmd(ctx, str_c(str)); +} + +static void imapc_sync_uid_next(struct imapc_sync_context *ctx) +{ + struct imapc_mailbox *mbox = ctx->mbox; + const struct mail_index_header *hdr; + uint32_t uid_next = mbox->sync_uid_next; + + if (uid_next < mbox->min_append_uid) + uid_next = mbox->min_append_uid; + + hdr = mail_index_get_header(ctx->sync_view); + if (hdr->next_uid < uid_next) { + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), FALSE); + } +} + +static void imapc_sync_highestmodseq(struct imapc_sync_context *ctx) +{ + if (imapc_mailbox_has_modseqs(ctx->mbox) && + mail_index_modseq_get_highest(ctx->sync_view) < ctx->mbox->sync_highestmodseq) + mail_index_update_highest_modseq(ctx->trans, ctx->mbox->sync_highestmodseq); +} + +static void +imapc_initial_sync_check(struct imapc_sync_context *ctx, bool nooped) +{ + struct imapc_msgmap *msgmap = + imapc_client_mailbox_get_msgmap(ctx->mbox->client_box); + struct mail_index_view *view = ctx->mbox->delayed_sync_view; + const struct mail_index_header *hdr = mail_index_get_header(view); + uint32_t rseq, lseq, ruid, luid, rcount, lcount; + + rseq = lseq = 1; + rcount = imapc_msgmap_count(msgmap); + lcount = mail_index_view_get_messages_count(view); + while (rseq <= rcount || lseq <= lcount) { + if (rseq <= rcount) + ruid = imapc_msgmap_rseq_to_uid(msgmap, rseq); + else + ruid = (uint32_t)-1; + if (lseq <= lcount) + mail_index_lookup_uid(view, lseq, &luid); + else + luid = (uint32_t)-1; + + if (ruid == luid) { + /* message exists in index and in remote server */ + lseq++; rseq++; + } else if (luid < ruid) { + /* message exists in index but not in remote server */ + if (luid >= ctx->mbox->sync_uid_next) { + /* the message was added to index by another + imapc session, and it's not visible yet + in this session */ + break; + } + /* it's already expunged and we should have marked it */ + i_assert(mail_index_is_expunged(view, lseq) || + seq_range_exists(&ctx->mbox->delayed_expunged_uids, luid)); + lseq++; + } else { + /* message doesn't exist in index, but exists in + remote server */ + if (lseq > lcount && ruid >= hdr->next_uid) { + /* the message hasn't been yet added to index */ + break; + } + + /* another imapc session expunged it => + NOOP should send us an EXPUNGE event */ + if (!nooped) { + imapc_mailbox_noop(ctx->mbox); + imapc_initial_sync_check(ctx, TRUE); + return; + } + /* already nooped => index is corrupted */ + imapc_mailbox_set_corrupted(ctx->mbox, + "Expunged message uid=%u reappeared", ruid); + ctx->failed = TRUE; + return; + } + } +} + +static void +imapc_sync_send_commands(struct imapc_sync_context *ctx) +{ + if (ctx->mbox->exists_count == 0) { + /* empty mailbox - no point in fetching anything */ + return; + } + + if (IMAPC_BOX_HAS_FEATURE(ctx->mbox, IMAPC_FEATURE_GMAIL_MIGRATION) && + ctx->mbox->storage->set->pop3_deleted_flag[0] != '\0') { + struct imapc_command *cmd; + + cmd = imapc_sync_cmd(ctx, "SEARCH RETURN (ALL) X-GM-RAW \"in:^pop\""); + i_free(ctx->mbox->sync_gmail_pop3_search_tag); + ctx->mbox->sync_gmail_pop3_search_tag = + i_strdup(imapc_command_get_tag(cmd)); + } +} + +static void imapc_sync_index(struct imapc_sync_context *ctx) +{ + struct imapc_mailbox *mbox = ctx->mbox; + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + + i_array_init(&ctx->expunged_uids, 64); + ctx->keywords = mail_index_get_keywords(mbox->box.index); + ctx->pool = pool_alloconly_create("imapc sync pool", 1024); + + while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN { + if (!mail_index_lookup_seq_range(ctx->sync_view, + sync_rec.uid1, sync_rec.uid2, + &seq1, &seq2)) { + /* already expunged, nothing to do. */ + } else switch (sync_rec.type) { + case MAIL_INDEX_SYNC_TYPE_EXPUNGE: + imapc_sync_add_missing_deleted_flags(ctx, seq1, seq2); + seq_range_array_add_range(&ctx->expunged_uids, + sync_rec.uid1, sync_rec.uid2); + break; + case MAIL_INDEX_SYNC_TYPE_FLAGS: + imapc_sync_index_flags(ctx, &sync_rec); + break; + case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + imapc_sync_index_keyword(ctx, &sync_rec); + break; + } + } T_END; + imapc_sync_finish_store(ctx); + pool_unref(&ctx->pool); + + if (!mbox->initial_sync_done) + imapc_sync_send_commands(ctx); + + imapc_sync_expunge_finish(ctx); + while (ctx->sync_command_count > 0) + imapc_mailbox_run(mbox); + array_free(&ctx->expunged_uids); + + if (!mbox->state_fetched_success) { + /* All the sync commands succeeded, but we got disconnected. + imapc_initial_sync_check() will crash if we go there. */ + ctx->failed = TRUE; + } + + /* add uidnext & highestmodseq after all appends */ + imapc_sync_uid_next(ctx); + imapc_sync_highestmodseq(ctx); + + mailbox_sync_notify(&mbox->box, 0, 0); + + if (!ctx->failed) { + /* reset only after a successful sync */ + mbox->sync_fetch_first_uid = 0; + } + if (!mbox->initial_sync_done && !ctx->failed) { + imapc_initial_sync_check(ctx, FALSE); + mbox->initial_sync_done = TRUE; + } +} + +static int +imapc_sync_begin(struct imapc_mailbox *mbox, + struct imapc_sync_context **ctx_r, bool force) +{ + struct imapc_sync_context *ctx; + enum mail_index_sync_flags sync_flags; + int ret; + + i_assert(!mbox->syncing); + + ctx = i_new(struct imapc_sync_context, 1); + ctx->mbox = mbox; + + sync_flags = index_storage_get_sync_flags(&mbox->box) | + MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY; + if (!force) + sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + + ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, + sync_flags); + if (ret <= 0) { + if (ret < 0) + mailbox_set_index_error(&mbox->box); + i_free(ctx); + *ctx_r = NULL; + return ret; + } + + i_assert(mbox->sync_view == NULL); + i_assert(mbox->delayed_sync_trans == NULL); + mbox->sync_view = ctx->sync_view; + mbox->delayed_sync_view = + mail_index_transaction_open_updated_view(ctx->trans); + mbox->delayed_sync_trans = ctx->trans; + mbox->delayed_sync_cache_view = + mail_cache_view_open(mbox->box.cache, mbox->delayed_sync_view); + mbox->delayed_sync_cache_trans = + mail_cache_get_transaction(mbox->delayed_sync_cache_view, + mbox->delayed_sync_trans); + mbox->min_append_uid = mail_index_get_header(ctx->sync_view)->next_uid; + + mbox->syncing = TRUE; + mbox->sync_ctx = ctx; + + if (mbox->delayed_untagged_exists) { + bool fetch_send = imapc_mailbox_fetch_state(mbox, + mbox->min_append_uid); + while (fetch_send && mbox->delayed_untagged_exists) + imapc_mailbox_run(mbox); + } + + if (!mbox->box.deleting) + imapc_sync_index(ctx); + + mail_index_view_close(&mbox->delayed_sync_view); + mbox->delayed_sync_trans = NULL; + mbox->sync_view = NULL; + + *ctx_r = ctx; + return 0; +} + +static int imapc_sync_finish(struct imapc_sync_context **_ctx) +{ + struct imapc_sync_context *ctx = *_ctx; + bool changes; + int ret = ctx->failed ? -1 : 0; + + *_ctx = NULL; + /* Commit the transaction even if we failed. This is important, because + during the sync delayed_sync_trans points to the sync transaction. + Even if the syncing doesn't fully succeed, we don't want to lose + changes in delayed_sync_trans. */ + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { + mailbox_set_index_error(&ctx->mbox->box); + ret = -1; + } + if (ctx->mbox->sync_gmail_pop3_search_tag != NULL) { + mailbox_set_critical(&ctx->mbox->box, + "gmail-pop3 search not successful"); + i_free_and_null(ctx->mbox->sync_gmail_pop3_search_tag); + ret = -1; + } + mail_cache_view_close(&ctx->mbox->delayed_sync_cache_view); + ctx->mbox->delayed_sync_cache_trans = NULL; + + ctx->mbox->syncing = FALSE; + ctx->mbox->sync_ctx = NULL; + + /* this is done simply to commit delayed expunges if there are any + (has to be done after sync is committed) */ + if (imapc_mailbox_commit_delayed_trans(ctx->mbox, FALSE, &changes) < 0) + ctx->failed = TRUE; + + i_free(ctx); + return ret; +} + +static int imapc_untagged_fetch_uid_cmp(struct imapc_untagged_fetch_ctx *const *ctx1, + struct imapc_untagged_fetch_ctx *const *ctx2) +{ + return (*ctx1)->uid < (*ctx2)->uid ? -1 : + (*ctx1)->uid > (*ctx2)->uid ? 1 : 0; +} + +static void imapc_sync_handle_untagged_fetches(struct imapc_mailbox *mbox) +{ + struct imapc_untagged_fetch_ctx *untagged_fetch_context; + struct mail_index_view *updated_view; + uint32_t lseq; + + i_assert(array_count(&mbox->untagged_fetch_contexts) > 0); + i_assert(mbox->delayed_sync_trans == NULL); + + array_sort(&mbox->untagged_fetch_contexts, imapc_untagged_fetch_uid_cmp); + + mbox->delayed_sync_trans = + mail_index_transaction_begin(imapc_mailbox_get_sync_view(mbox), + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + + array_foreach_elem(&mbox->untagged_fetch_contexts, untagged_fetch_context) { + if (untagged_fetch_context->uid < mbox->min_append_uid || + untagged_fetch_context->uid < mail_index_get_header(mbox->sync_view)->next_uid) { + /* The message was already added */ + continue; + } + + mail_index_append(mbox->delayed_sync_trans, + untagged_fetch_context->uid, + &lseq); + mbox->min_append_uid = untagged_fetch_context->uid + 1; + } + + updated_view = mail_index_transaction_open_updated_view(mbox->delayed_sync_trans); + + array_foreach_elem(&mbox->untagged_fetch_contexts, untagged_fetch_context) { + if (!untagged_fetch_context->have_flags) { + imapc_untagged_fetch_ctx_free(&untagged_fetch_context); + continue; + } + + /* Lookup the mail belonging to this context using the + context->uid */ + if (!mail_index_lookup_seq(updated_view, + untagged_fetch_context->uid, + &lseq)) { + /* mail is expunged already */ + imapc_untagged_fetch_ctx_free(&untagged_fetch_context); + continue; + } + + imapc_untagged_fetch_update_flags(mbox, untagged_fetch_context, + updated_view, lseq); + imapc_untagged_fetch_ctx_free(&untagged_fetch_context); + } + + mail_index_view_close(&updated_view); + if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) + mailbox_set_index_error(&mbox->box); + array_clear(&mbox->untagged_fetch_contexts); +} + +static int imapc_sync(struct imapc_mailbox *mbox) +{ + struct imapc_sync_context *sync_ctx; + bool force = mbox->sync_fetch_first_uid != 0; + + if ((mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0) { + /* we're only saving mails here - no syncing actually wanted */ + return 0; + } + + if (imapc_sync_begin(mbox, &sync_ctx, force) < 0) + return -1; + + if (!array_is_empty(&mbox->untagged_fetch_contexts)) + imapc_sync_handle_untagged_fetches(mbox); + + if (sync_ctx == NULL) + return 0; + if (imapc_sync_finish(&sync_ctx) < 0) + return -1; + return 0; +} + +static void +imapc_noop_if_needed(struct imapc_mailbox *mbox, enum mailbox_sync_flags flags) +{ + if (!mbox->initial_sync_done) { + /* we just SELECTed/EXAMINEd the mailbox, don't do another + NOOP. */ + } else if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 && + ((mbox->capabilities & IMAPC_CAPABILITY_IDLE) == 0 || + (flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0)) { + /* do NOOP to make sure we have the latest changes before + starting sync. this is necessary either because se don't + support IDLE at all, or because we want to be sure that we + have the latest changes (IDLE is started with a small delay, + so we might not actually even be in IDLE right not) */ + imapc_mailbox_noop(mbox); + } +} + +static bool imapc_mailbox_need_initial_fetch(struct imapc_mailbox *mbox) +{ + if (mbox->box.deleting) { + /* If the mailbox is about to be deleted there is no need to + expect initial fetch to be done */ + return FALSE; + } + if ((mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0) { + /* The mailbox is opened only for saving there is no need to + expect initial fetchting do be done. */ + return FALSE; + } + return TRUE; +} + +struct mailbox_sync_context * +imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(box); + struct imapc_mailbox_list *list = mbox->storage->client->_list; + bool changes; + int ret = 0; + + if (list != NULL) { + if (!list->refreshed_mailboxes && + list->last_refreshed_mailboxes < ioloop_time) + list->refreshed_mailboxes_recently = FALSE; + } + + imapc_noop_if_needed(mbox, flags); + + if (imapc_storage_client_handle_auth_failure(mbox->storage->client)) + ret = -1; + else if (!mbox->state_fetched_success && !mbox->state_fetching_uid1 && + imapc_mailbox_need_initial_fetch(mbox)) { + /* initial FETCH failed already */ + ret = -1; + } + if (imapc_mailbox_commit_delayed_trans(mbox, FALSE, &changes) < 0) + ret = -1; + if ((changes || mbox->sync_fetch_first_uid != 0 || + index_mailbox_want_full_sync(&mbox->box, flags)) && + ret == 0) + ret = imapc_sync(mbox); + + return index_mailbox_sync_init(box, flags, ret < 0); +} + +int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx, + struct mailbox_sync_status *status_r) +{ + struct imapc_mailbox *mbox = IMAPC_MAILBOX(ctx->box); + int ret; + + ret = index_mailbox_sync_deinit(ctx, status_r); + ctx = NULL; + + if (mbox->client_box == NULL) + return ret; + + imapc_client_mailbox_idle(mbox->client_box); + return ret; +} diff --git a/src/lib-storage/index/imapc/imapc-sync.h b/src/lib-storage/index/imapc/imapc-sync.h new file mode 100644 index 0000000..54a0368 --- /dev/null +++ b/src/lib-storage/index/imapc/imapc-sync.h @@ -0,0 +1,39 @@ +#ifndef IMAPC_SYNC_H +#define IMAPC_SYNC_H + +struct mailbox; +struct mailbox_sync_status; + +struct imapc_sync_store { + enum modify_type modify_type; + const char *flags; + + ARRAY_TYPE(seq_range) uids; +}; + +struct imapc_sync_context { + struct imapc_mailbox *mbox; + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + + const ARRAY_TYPE(keywords) *keywords; + ARRAY_TYPE(seq_range) expunged_uids; + unsigned int sync_command_count; + + pool_t pool; + HASH_TABLE(struct imapc_sync_store *, struct imapc_sync_store *) stores; + + uint32_t prev_uid1, prev_uid2; + enum modify_type prev_modify_type; + string_t *prev_flags; + + bool failed:1; +}; + +struct mailbox_sync_context * +imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); +int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx, + struct mailbox_sync_status *status_r); + +#endif diff --git a/src/lib-storage/index/index-attachment.c b/src/lib-storage/index/index-attachment.c new file mode 100644 index 0000000..6e51fab --- /dev/null +++ b/src/lib-storage/index/index-attachment.c @@ -0,0 +1,446 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "safe-mkstemp.h" +#include "fs-api.h" +#include "istream.h" +#include "ostream.h" +#include "base64.h" +#include "hash-format.h" +#include "str.h" +#include "message-parser.h" +#include "rfc822-parser.h" +#include "fs-api.h" +#include "istream-fs-file.h" +#include "istream-attachment-connector.h" +#include "istream-attachment-extractor.h" +#include "mail-user.h" +#include "index-mail.h" +#include "index-attachment.h" + +enum mail_attachment_decode_option { + MAIL_ATTACHMENT_DECODE_OPTION_NONE = '-', + MAIL_ATTACHMENT_DECODE_OPTION_BASE64 = 'B', + MAIL_ATTACHMENT_DECODE_OPTION_CRLF = 'C' +}; + +struct mail_save_attachment { + pool_t pool; + struct fs *fs; + struct istream *input; + + struct fs_file *cur_file; + ARRAY_TYPE(mail_attachment_extref) extrefs; +}; + +static const char *index_attachment_dir_get(struct mail_storage *storage) +{ + return mail_user_home_expand(storage->user, + storage->set->mail_attachment_dir); +} + +static bool index_attachment_want(const struct istream_attachment_header *hdr, + void *context) +{ + struct mail_save_context *ctx = context; + struct mail_attachment_part apart; + + i_zero(&apart); + apart.part = hdr->part; + apart.content_type = hdr->content_type; + apart.content_disposition = hdr->content_disposition; + + if (ctx->part_is_attachment != NULL) + return ctx->part_is_attachment(ctx, &apart); + + /* don't treat text/ parts as attachments */ + return hdr->content_type != NULL && + strncasecmp(hdr->content_type, "text/", 5) != 0; +} + +static int index_attachment_open_temp_fd(void *context) +{ + struct mail_save_context *ctx = context; + struct mail_storage *storage = ctx->transaction->box->storage; + string_t *temp_path; + int fd; + + temp_path = t_str_new(256); + mail_user_set_get_temp_prefix(temp_path, storage->user->set); + fd = safe_mkstemp_hostpid(temp_path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + mailbox_set_critical(ctx->transaction->box, + "safe_mkstemp(%s) failed: %m", str_c(temp_path)); + return -1; + } + if (unlink(str_c(temp_path)) < 0) { + mailbox_set_critical(ctx->transaction->box, + "unlink(%s) failed: %m", str_c(temp_path)); + i_close_fd(&fd); + return -1; + } + return fd; +} + +static int +index_attachment_open_ostream(struct istream_attachment_info *info, + struct ostream **output_r, + const char **error_r ATTR_UNUSED, void *context) +{ + struct mail_save_context *ctx = context; + struct mail_save_attachment *attach = ctx->data.attach; + struct mail_storage *storage = ctx->transaction->box->storage; + struct mail_attachment_extref *extref; + enum fs_open_flags flags = 0; + const char *attachment_dir, *path, *digest = info->hash; + guid_128_t guid_128; + + i_assert(attach->cur_file == NULL); + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) + flags |= FS_OPEN_FLAG_FSYNC; + + if (strlen(digest) < 4) { + /* make sure we can access first 4 bytes without accessing + out of bounds memory */ + digest = t_strconcat(digest, "\0\0\0\0", NULL); + } + + guid_128_generate(guid_128); + attachment_dir = index_attachment_dir_get(storage); + path = t_strdup_printf("%s/%c%c/%c%c/%s-%s", attachment_dir, + digest[0], digest[1], + digest[2], digest[3], digest, + guid_128_to_string(guid_128)); + attach->cur_file = fs_file_init(attach->fs, path, + FS_OPEN_MODE_REPLACE | flags); + + extref = array_append_space(&attach->extrefs); + extref->start_offset = info->start_offset; + extref->size = info->encoded_size; + extref->path = p_strdup(attach->pool, + path + strlen(attachment_dir) + 1); + extref->base64_blocks_per_line = info->base64_blocks_per_line; + extref->base64_have_crlf = info->base64_have_crlf; + + *output_r = fs_write_stream(attach->cur_file); + return 0; +} + +static int +index_attachment_close_ostream(struct ostream *output, bool success, + const char **error, void *context) +{ + struct mail_save_context *ctx = context; + struct mail_save_attachment *attach = ctx->data.attach; + int ret = success ? 0 : -1; + + i_assert(attach->cur_file != NULL); + + if (ret < 0) + fs_write_stream_abort_error(attach->cur_file, &output, "%s", *error); + else if (fs_write_stream_finish(attach->cur_file, &output) < 0) { + *error = t_strdup_printf("Couldn't create attachment %s: %s", + fs_file_path(attach->cur_file), + fs_file_last_error(attach->cur_file)); + ret = -1; + } + fs_file_deinit(&attach->cur_file); + + if (ret < 0) { + array_pop_back(&attach->extrefs); + } + return ret; +} + +void index_attachment_save_begin(struct mail_save_context *ctx, + struct fs *fs, struct istream *input) +{ + struct mail_storage *storage = ctx->transaction->box->storage; + struct mail_save_attachment *attach; + struct istream_attachment_settings set; + const char *error; + pool_t pool; + + i_assert(ctx->data.attach == NULL); + + if (*storage->set->mail_attachment_dir == '\0') + return; + + i_zero(&set); + set.min_size = storage->set->mail_attachment_min_size; + if (hash_format_init(storage->set->mail_attachment_hash, + &set.hash_format, &error) < 0) { + /* we already checked this when verifying settings */ + i_panic("mail_attachment_hash=%s unexpectedly failed: %s", + storage->set->mail_attachment_hash, error); + } + set.want_attachment = index_attachment_want; + set.open_temp_fd = index_attachment_open_temp_fd; + set.open_attachment_ostream = index_attachment_open_ostream; + set.close_attachment_ostream = index_attachment_close_ostream; + + pool = pool_alloconly_create("save attachment", 1024); + attach = p_new(pool, struct mail_save_attachment, 1); + attach->pool = pool; + attach->fs = fs; + attach->input = i_stream_create_attachment_extractor(input, &set, ctx); + p_array_init(&attach->extrefs, attach->pool, 8); + ctx->data.attach = attach; +} + +static int save_check_write_error(struct mail_save_context *ctx, + struct ostream *output) +{ + struct mail_storage *storage = ctx->transaction->box->storage; + + if (output->stream_errno == 0) + return 0; + + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(ctx->dest_mail, "write(%s) failed: %s", + o_stream_get_name(output), o_stream_get_error(output)); + } + return -1; +} + +int index_attachment_save_continue(struct mail_save_context *ctx) +{ + struct mail_save_attachment *attach = ctx->data.attach; + const unsigned char *data; + size_t size; + ssize_t ret; + + if (attach->input->stream_errno != 0) + return -1; + + do { + ret = i_stream_read(attach->input); + if (ret > 0 || ret == -2) { + data = i_stream_get_data(attach->input, &size); + o_stream_nsend(ctx->data.output, data, size); + i_stream_skip(attach->input, size); + } + index_mail_cache_parse_continue(ctx->dest_mail); + if (ret == 0 && !i_stream_attachment_extractor_can_retry(attach->input)) { + /* need more input */ + return 0; + } + } while (ret != -1); + + if (attach->input->stream_errno != 0) { + mail_set_critical(ctx->dest_mail, "read(%s) failed: %s", + i_stream_get_name(attach->input), + i_stream_get_error(attach->input)); + return -1; + } + if (ctx->data.output != NULL) { + if (save_check_write_error(ctx, ctx->data.output) < 0) + return -1; + } + return 0; +} + +int index_attachment_save_finish(struct mail_save_context *ctx) +{ + struct mail_save_attachment *attach = ctx->data.attach; + + (void)i_stream_read(attach->input); + i_assert(attach->input->eof); + return attach->input->stream_errno == 0 ? 0 : -1; +} + +void index_attachment_save_free(struct mail_save_context *ctx) +{ + struct mail_save_attachment *attach = ctx->data.attach; + + if (attach != NULL) { + i_stream_unref(&attach->input); + pool_unref(&attach->pool); + ctx->data.attach = NULL; + } +} + +const ARRAY_TYPE(mail_attachment_extref) * +index_attachment_save_get_extrefs(struct mail_save_context *ctx) +{ + return ctx->data.attach == NULL ? NULL : + &ctx->data.attach->extrefs; +} + +static int +index_attachment_delete_real(struct mail_storage *storage, + struct fs *fs, const char *name) +{ + struct fs_file *file; + const char *path; + int ret; + + path = t_strdup_printf("%s/%s", index_attachment_dir_get(storage), name); + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + if ((ret = fs_delete(file)) < 0) + mail_storage_set_critical(storage, "%s", fs_file_last_error(file)); + fs_file_deinit(&file); + return ret; +} + +int index_attachment_delete(struct mail_storage *storage, + struct fs *fs, const char *name) +{ + int ret; + + T_BEGIN { + ret = index_attachment_delete_real(storage, fs, name); + } T_END; + return ret; +} + +void index_attachment_append_extrefs(string_t *str, + const ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + const struct mail_attachment_extref *extref; + bool add_space = FALSE; + unsigned int startpos; + + array_foreach(extrefs, extref) { + if (!add_space) + add_space = TRUE; + else + str_append_c(str, ' '); + str_printfa(str, "%"PRIuUOFF_T" %"PRIuUOFF_T" ", + extref->start_offset, extref->size); + + startpos = str_len(str); + if (extref->base64_have_crlf) + str_append_c(str, MAIL_ATTACHMENT_DECODE_OPTION_CRLF); + if (extref->base64_blocks_per_line > 0) { + str_printfa(str, "%c%u", + MAIL_ATTACHMENT_DECODE_OPTION_BASE64, + extref->base64_blocks_per_line * 4); + } + if (startpos == str_len(str)) { + /* make it clear there are no options */ + str_append_c(str, MAIL_ATTACHMENT_DECODE_OPTION_NONE); + } + str_append_c(str, ' '); + str_append(str, extref->path); + } +} + +static bool +parse_extref_decode_options(const char *str, + struct mail_attachment_extref *extref) +{ + unsigned int num; + + if (*str == MAIL_ATTACHMENT_DECODE_OPTION_NONE) + return str[1] == '\0'; + + while (*str != '\0') { + switch (*str) { + case MAIL_ATTACHMENT_DECODE_OPTION_BASE64: + str++; num = 0; + while (*str >= '0' && *str <= '9') { + num = num*10 + (*str-'0'); + str++; + } + if (num == 0 || num % 4 != 0) + return FALSE; + + extref->base64_blocks_per_line = num/4; + break; + case MAIL_ATTACHMENT_DECODE_OPTION_CRLF: + extref->base64_have_crlf = TRUE; + str++; + break; + default: + return FALSE; + } + } + return TRUE; +} + +bool index_attachment_parse_extrefs(const char *line, pool_t pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + struct mail_attachment_extref extref; + const char *const *args; + unsigned int i, len; + uoff_t last_voffset; + + args = t_strsplit(line, " "); + len = str_array_length(args); + if ((len % 4) != 0) + return FALSE; + + last_voffset = 0; + for (i = 0; args[i] != NULL; i += 4) { + const char *start_offset_str = args[i+0]; + const char *size_str = args[i+1]; + const char *decode_options = args[i+2]; + const char *path = args[i+3]; + + i_zero(&extref); + if (str_to_uoff(start_offset_str, &extref.start_offset) < 0 || + str_to_uoff(size_str, &extref.size) < 0 || + extref.start_offset < last_voffset || + !parse_extref_decode_options(decode_options, &extref)) + return FALSE; + + last_voffset += extref.size + + (extref.start_offset - last_voffset); + + extref.path = p_strdup(pool, path); + array_push_back(extrefs, &extref); + } + return TRUE; +} + +int index_attachment_stream_get(struct fs *fs, const char *attachment_dir, + const char *path_suffix, + struct istream **stream, uoff_t full_size, + const char *ext_refs, const char **error_r) +{ + ARRAY_TYPE(mail_attachment_extref) extrefs_arr; + const struct mail_attachment_extref *extref; + struct istream_attachment_connector *conn; + struct istream *input; + struct fs_file *file; + const char *path; + int ret; + + *error_r = NULL; + + t_array_init(&extrefs_arr, 16); + if (!index_attachment_parse_extrefs(ext_refs, pool_datastack_create(), + &extrefs_arr)) { + *error_r = "Broken ext-refs string"; + return -1; + } + conn = istream_attachment_connector_begin(*stream, full_size); + + array_foreach(&extrefs_arr, extref) { + path = t_strdup_printf("%s/%s%s", attachment_dir, + extref->path, path_suffix); + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY | + FS_OPEN_FLAG_SEEKABLE); + input = i_stream_create_fs_file(&file, IO_BLOCK_SIZE); + + ret = istream_attachment_connector_add(conn, input, + extref->start_offset, extref->size, + extref->base64_blocks_per_line, + extref->base64_have_crlf, error_r); + i_stream_unref(&input); + if (ret < 0) { + istream_attachment_connector_abort(&conn); + return -1; + } + } + + input = istream_attachment_connector_finish(&conn); + i_stream_set_name(input, t_strdup_printf( + "attachments-connector(%s)", i_stream_get_name(*stream))); + i_stream_unref(stream); + *stream = input; + return 0; +} diff --git a/src/lib-storage/index/index-attachment.h b/src/lib-storage/index/index-attachment.h new file mode 100644 index 0000000..a52f158 --- /dev/null +++ b/src/lib-storage/index/index-attachment.h @@ -0,0 +1,52 @@ +#ifndef INDEX_ATTACHMENT_H +#define INDEX_ATTACHMENT_H + +#include "sha1.h" + +struct fs; +struct mail_save_context; +struct mail_storage; + +struct mail_attachment_extref { + /* path without attachment_dir/ prefix */ + const char *path; + /* offset in input stream where part begins */ + uoff_t start_offset; + uoff_t size; + + /* If non-zero, this attachment was saved as base64-decoded and it + need to be encoded back before presenting it to client. Each line + (except last one) consists of this many base64 blocks (4 chars of + base64 encoded data). */ + unsigned int base64_blocks_per_line; + /* Line feeds are CRLF instead of LF */ + bool base64_have_crlf; +}; +ARRAY_DEFINE_TYPE(mail_attachment_extref, struct mail_attachment_extref); + +void index_attachment_save_begin(struct mail_save_context *ctx, + struct fs *fs, struct istream *input); +int index_attachment_save_continue(struct mail_save_context *ctx); +int index_attachment_save_finish(struct mail_save_context *ctx); +void index_attachment_save_free(struct mail_save_context *ctx); +const ARRAY_TYPE(mail_attachment_extref) * +index_attachment_save_get_extrefs(struct mail_save_context *ctx); + +/* Delete a given attachment name from storage + (name is same as mail_attachment_extref.name). */ +int index_attachment_delete(struct mail_storage *storage, + struct fs *fs, const char *name); + +void index_attachment_append_extrefs(string_t *str, + const ARRAY_TYPE(mail_attachment_extref) *extrefs); +/* Parse extrefs value to given array. Names are allocated from the + given pool. */ +bool index_attachment_parse_extrefs(const char *line, pool_t pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs); + +int index_attachment_stream_get(struct fs *fs, const char *attachment_dir, + const char *path_suffix, + struct istream **stream, uoff_t full_size, + const char *ext_refs, const char **error_r); + +#endif diff --git a/src/lib-storage/index/index-attribute.c b/src/lib-storage/index/index-attribute.c new file mode 100644 index 0000000..3d4c415 --- /dev/null +++ b/src/lib-storage/index/index-attribute.c @@ -0,0 +1,333 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "dict.h" +#include "index-storage.h" + +struct index_storage_attribute_iter { + struct mailbox_attribute_iter iter; + struct dict_iterate_context *diter; + char *prefix; + size_t prefix_len; + bool dict_disabled; +}; + +static struct mail_namespace * +mail_user_find_attribute_namespace(struct mail_user *user) +{ + struct mail_namespace *ns; + + ns = mail_namespace_find_inbox(user->namespaces); + if (ns != NULL) + return ns; + + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + if (ns->type == MAIL_NAMESPACE_TYPE_PRIVATE) + return ns; + } + return NULL; +} + +static int +index_storage_get_user_dict(struct mail_storage *err_storage, + struct mail_user *user, struct dict **dict_r) +{ + struct dict_settings dict_set; + struct mail_namespace *ns; + struct mail_storage *attr_storage; + const char *error; + + if (user->_attr_dict != NULL) { + *dict_r = user->_attr_dict; + return 0; + } + if (user->attr_dict_failed) { + mail_storage_set_internal_error(err_storage); + return -1; + } + + ns = mail_user_find_attribute_namespace(user); + if (ns == NULL) { + /* probably never happens? */ + mail_storage_set_error(err_storage, MAIL_ERROR_NOTPOSSIBLE, + "Mailbox attributes not available for this mailbox"); + return -1; + } + attr_storage = mail_namespace_get_default_storage(ns); + + if (*attr_storage->set->mail_attribute_dict == '\0') { + mail_storage_set_error(err_storage, MAIL_ERROR_NOTPOSSIBLE, + "Mailbox attributes not enabled"); + return -1; + } + + i_zero(&dict_set); + dict_set.base_dir = user->set->base_dir; + dict_set.event_parent = user->event; + if (dict_init(attr_storage->set->mail_attribute_dict, &dict_set, + &user->_attr_dict, &error) < 0) { + mail_storage_set_critical(err_storage, + "mail_attribute_dict: dict_init(%s) failed: %s", + attr_storage->set->mail_attribute_dict, error); + user->attr_dict_failed = TRUE; + return -1; + } + *dict_r = user->_attr_dict; + return 0; +} + +static int +index_storage_get_dict(struct mailbox *box, enum mail_attribute_type type_flags, + struct dict **dict_r, const char **mailbox_prefix_r) +{ + enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK; + struct mail_storage *storage = box->storage; + struct mail_namespace *ns; + struct mailbox_metadata metadata; + struct dict_settings set; + const char *error; + + if ((type_flags & MAIL_ATTRIBUTE_TYPE_FLAG_VALIDATED) != 0) { + /* IMAP METADATA support isn't enabled, so don't allow using + mail_attribute_dict. */ + mail_storage_set_error(storage, MAIL_ERROR_NOTPOSSIBLE, + "Generic mailbox attributes not enabled"); + return -1; + } + + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) + return -1; + *mailbox_prefix_r = guid_128_to_string(metadata.guid); + + ns = mailbox_get_namespace(box); + if (type == MAIL_ATTRIBUTE_TYPE_PRIVATE) { + /* private attributes are stored in user's own dict */ + return index_storage_get_user_dict(storage, storage->user, dict_r); + } else if (ns->user == ns->owner) { + /* user owns the mailbox. shared attributes are stored in + the same dict. */ + return index_storage_get_user_dict(storage, storage->user, dict_r); + } else if (ns->owner != NULL) { + /* accessing shared attribute of a shared mailbox. + use the owner's dict. */ + return index_storage_get_user_dict(storage, ns->owner, dict_r); + } + + /* accessing shared attributes of a public mailbox. no user owns it, + so use the storage's dict. */ + if (storage->_shared_attr_dict != NULL) { + *dict_r = storage->_shared_attr_dict; + return 0; + } + if (*storage->set->mail_attribute_dict == '\0') { + mail_storage_set_error(storage, MAIL_ERROR_NOTPOSSIBLE, + "Mailbox attributes not enabled"); + return -1; + } + if (storage->shared_attr_dict_failed) { + mail_storage_set_internal_error(storage); + return -1; + } + + i_zero(&set); + set.base_dir = storage->user->set->base_dir; + set.event_parent = storage->user->event; + if (dict_init(storage->set->mail_attribute_dict, &set, + &storage->_shared_attr_dict, &error) < 0) { + mail_storage_set_critical(storage, + "mail_attribute_dict: dict_init(%s) failed: %s", + storage->set->mail_attribute_dict, error); + storage->shared_attr_dict_failed = TRUE; + return -1; + } + *dict_r = storage->_shared_attr_dict; + return 0; +} + +static const char * +key_get_prefixed(enum mail_attribute_type type_flags, const char *mailbox_prefix, + const char *key) +{ + enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK; + + switch (type) { + case MAIL_ATTRIBUTE_TYPE_PRIVATE: + return t_strconcat(DICT_PATH_PRIVATE, mailbox_prefix, "/", + key, NULL); + case MAIL_ATTRIBUTE_TYPE_SHARED: + return t_strconcat(DICT_PATH_SHARED, mailbox_prefix, "/", + key, NULL); + } + i_unreached(); +} + +static int +index_storage_attribute_get_dict_trans(struct mailbox_transaction_context *t, + enum mail_attribute_type type_flags, + struct dict_transaction_context **dtrans_r, + const char **mailbox_prefix_r) +{ + enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK; + struct dict_transaction_context **dtransp = NULL; + struct dict *dict; + struct mailbox_metadata metadata; + + switch (type) { + case MAIL_ATTRIBUTE_TYPE_PRIVATE: + dtransp = &t->attr_pvt_trans; + break; + case MAIL_ATTRIBUTE_TYPE_SHARED: + dtransp = &t->attr_shared_trans; + break; + } + i_assert(dtransp != NULL); + + if (*dtransp != NULL && + (type_flags & MAIL_ATTRIBUTE_TYPE_FLAG_VALIDATED) == 0) { + /* Transaction already created. Even if it was, don't use it + if _FLAG_VALIDATED is being used. It'll be handled below by + returning failure. */ + if (mailbox_get_metadata(t->box, MAILBOX_METADATA_GUID, + &metadata) < 0) + return -1; + *mailbox_prefix_r = guid_128_to_string(metadata.guid); + *dtrans_r = *dtransp; + return 0; + } + + if (index_storage_get_dict(t->box, type_flags, &dict, mailbox_prefix_r) < 0) + return -1; + i_assert(*dtransp == NULL); + + struct mail_user *user = mailbox_list_get_user(t->box->list); + const struct dict_op_settings *set = mail_user_get_dict_op_settings(user); + *dtransp = *dtrans_r = dict_transaction_begin(dict, set); + return 0; +} + +int index_storage_attribute_set(struct mailbox_transaction_context *t, + enum mail_attribute_type type_flags, + const char *key, + const struct mail_attribute_value *value) +{ + enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK; + struct dict_transaction_context *dtrans; + const char *mailbox_prefix; + bool pvt = type == MAIL_ATTRIBUTE_TYPE_PRIVATE; + time_t ts = value->last_change != 0 ? value->last_change : ioloop_time; + int ret = 0; + + if (index_storage_attribute_get_dict_trans(t, type_flags, &dtrans, + &mailbox_prefix) < 0) + return -1; + + T_BEGIN { + const char *prefixed_key = + key_get_prefixed(type_flags, mailbox_prefix, key); + const char *value_str; + + if (mailbox_attribute_value_to_string(t->box->storage, value, + &value_str) < 0) { + ret = -1; + } else if (value_str != NULL) { + dict_set(dtrans, prefixed_key, value_str); + mail_index_attribute_set(t->itrans, pvt, key, + ts, strlen(value_str)); + } else { + dict_unset(dtrans, prefixed_key); + mail_index_attribute_unset(t->itrans, pvt, key, ts); + } + } T_END; + return ret; +} + +int index_storage_attribute_get(struct mailbox *box, + enum mail_attribute_type type_flags, + const char *key, + struct mail_attribute_value *value_r) +{ + struct dict *dict; + const char *mailbox_prefix, *error; + int ret; + + i_zero(value_r); + + if (index_storage_get_dict(box, type_flags, &dict, &mailbox_prefix) < 0) + return -1; + + struct mail_user *user = mailbox_list_get_user(box->list); + const struct dict_op_settings *set = mail_user_get_dict_op_settings(user); + ret = dict_lookup(dict, set, pool_datastack_create(), + key_get_prefixed(type_flags, mailbox_prefix, key), + &value_r->value, &error); + if (ret < 0) { + mailbox_set_critical(box, + "Failed to get attribute %s: %s", key, error); + return -1; + } + return ret; +} + +struct mailbox_attribute_iter * +index_storage_attribute_iter_init(struct mailbox *box, + enum mail_attribute_type type_flags, + const char *prefix) +{ + struct index_storage_attribute_iter *iter; + struct dict *dict; + const char *mailbox_prefix; + + iter = i_new(struct index_storage_attribute_iter, 1); + iter->iter.box = box; + if (index_storage_get_dict(box, type_flags, &dict, &mailbox_prefix) < 0) { + if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTPOSSIBLE) + iter->dict_disabled = TRUE; + } else { + iter->prefix = i_strdup(key_get_prefixed(type_flags, mailbox_prefix, + prefix)); + iter->prefix_len = strlen(iter->prefix); + struct mail_user *user = mailbox_list_get_user(box->list); + const struct dict_op_settings *set = mail_user_get_dict_op_settings(user); + iter->diter = dict_iterate_init(dict, set, iter->prefix, + DICT_ITERATE_FLAG_RECURSE | + DICT_ITERATE_FLAG_NO_VALUE); + } + return &iter->iter; +} + +const char * +index_storage_attribute_iter_next(struct mailbox_attribute_iter *_iter) +{ + struct index_storage_attribute_iter *iter = + (struct index_storage_attribute_iter *)_iter; + const char *key, *value; + + if (iter->diter == NULL || !dict_iterate(iter->diter, &key, &value)) + return NULL; + + i_assert(strncmp(key, iter->prefix, iter->prefix_len) == 0); + key += iter->prefix_len; + return key; +} + +int index_storage_attribute_iter_deinit(struct mailbox_attribute_iter *_iter) +{ + struct index_storage_attribute_iter *iter = + (struct index_storage_attribute_iter *)_iter; + const char *error; + int ret; + + if (iter->diter == NULL) { + ret = iter->dict_disabled ? 0 : -1; + } else { + if ((ret = dict_iterate_deinit(&iter->diter, &error)) < 0) { + mailbox_set_critical(_iter->box, + "dict_iterate(%s) failed: %s", + iter->prefix, error); + } + } + i_free(iter->prefix); + i_free(iter); + return ret; +} diff --git a/src/lib-storage/index/index-mail-binary.c b/src/lib-storage/index/index-mail-binary.c new file mode 100644 index 0000000..80c319e --- /dev/null +++ b/src/lib-storage/index/index-mail-binary.c @@ -0,0 +1,598 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "safe-mkstemp.h" +#include "istream.h" +#include "istream-crlf.h" +#include "istream-seekable.h" +#include "istream-base64.h" +#include "istream-qp.h" +#include "istream-header-filter.h" +#include "ostream.h" +#include "message-binary-part.h" +#include "message-parser.h" +#include "message-decoder.h" +#include "mail-user.h" +#include "index-storage.h" +#include "index-mail.h" + +#define MAIL_BINARY_CACHE_EXPIRE_MSECS (60*1000) + +#define IS_CONVERTED_CTE(cte) \ + ((cte) == MESSAGE_CTE_QP || (cte) == MESSAGE_CTE_BASE64) + +struct binary_block { + struct istream *input; + uoff_t physical_pos; + unsigned int body_lines_count; + bool converted, converted_hdr; +}; + +struct binary_ctx { + struct mail *mail; + struct istream *input; + bool has_nuls, converted; + /* each block is its own input stream. basically each converted MIME + body has its own block and the parts between the MIME bodies are + unconverted blocks */ + ARRAY(struct binary_block) blocks; + + uoff_t copy_start_offset; +}; + +static void binary_copy_to(struct binary_ctx *ctx, uoff_t end_offset) +{ + struct binary_block *block; + struct istream *linput, *cinput; + uoff_t orig_offset, size; + + i_assert(end_offset >= ctx->copy_start_offset); + + if (end_offset == ctx->copy_start_offset) + return; + + size = end_offset - ctx->copy_start_offset; + orig_offset = ctx->input->v_offset; + + i_stream_seek(ctx->input, ctx->copy_start_offset); + linput = i_stream_create_limit(ctx->input, size); + cinput = i_stream_create_crlf(linput); + i_stream_unref(&linput); + + block = array_append_space(&ctx->blocks); + block->input = cinput; + + i_stream_seek(ctx->input, orig_offset); +} + +static void +binary_cte_filter_callback(struct header_filter_istream *input, + struct message_header_line *hdr, + bool *matched ATTR_UNUSED, void *context ATTR_UNUSED) +{ + static const char *cte_binary = "Content-Transfer-Encoding: binary\r\n"; + + if (hdr != NULL && hdr->eoh) { + i_stream_header_filter_add(input, cte_binary, + strlen(cte_binary)); + } +} + +static int +add_binary_part(struct binary_ctx *ctx, const struct message_part *part, + bool include_hdr) +{ + static const char *filter_headers[] = { + "Content-Transfer-Encoding", + }; + struct message_header_parser_ctx *parser; + struct message_header_line *hdr; + struct message_part *child; + struct message_size hdr_size; + struct istream *linput; + struct binary_block *block; + enum message_cte cte; + uoff_t part_end_offset; + int ret; + + /* first parse the header to find c-t-e. */ + i_stream_seek(ctx->input, part->physical_pos); + + cte = MESSAGE_CTE_78BIT; + parser = message_parse_header_init(ctx->input, &hdr_size, 0); + while ((ret = message_parse_header_next(parser, &hdr)) > 0) { + if (strcasecmp(hdr->name, "Content-Transfer-Encoding") == 0) + cte = message_decoder_parse_cte(hdr); + } + i_assert(ret < 0); + if (message_parse_header_has_nuls(parser)) { + /* we're not converting NULs to 0x80 when doing a binary fetch, + even if they're in the message header. */ + ctx->has_nuls = TRUE; + } + message_parse_header_deinit(&parser); + + if (ctx->input->stream_errno != 0) { + mail_set_critical(ctx->mail, + "read(%s) failed: %s", i_stream_get_name(ctx->input), + i_stream_get_error(ctx->input)); + return -1; + } + + if (cte == MESSAGE_CTE_UNKNOWN) { + mail_storage_set_error(ctx->mail->box->storage, + MAIL_ERROR_CONVERSION, + "Unknown Content-Transfer-Encoding."); + return -1; + } + + i_stream_seek(ctx->input, part->physical_pos); + if (!include_hdr) { + /* body only */ + } else if (IS_CONVERTED_CTE(cte)) { + /* write header with modified content-type */ + if (ctx->copy_start_offset != 0) + binary_copy_to(ctx, part->physical_pos); + block = array_append_space(&ctx->blocks); + block->physical_pos = part->physical_pos; + block->converted = TRUE; + block->converted_hdr = TRUE; + + linput = i_stream_create_limit(ctx->input, UOFF_T_MAX); + block->input = i_stream_create_header_filter(linput, + HEADER_FILTER_EXCLUDE | HEADER_FILTER_HIDE_BODY, + filter_headers, N_ELEMENTS(filter_headers), + binary_cte_filter_callback, NULL); + i_stream_unref(&linput); + } else { + /* copy everything as-is until the end of this header */ + binary_copy_to(ctx, part->physical_pos + + part->header_size.physical_size); + } + ctx->copy_start_offset = part->physical_pos + + part->header_size.physical_size; + part_end_offset = part->physical_pos + + part->header_size.physical_size + + part->body_size.physical_size; + + if (part->children != NULL) { + /* multipart */ + for (child = part->children; child != NULL; child = child->next) { + if (add_binary_part(ctx, child, TRUE) < 0) + return -1; + } + binary_copy_to(ctx, part_end_offset); + ctx->copy_start_offset = part_end_offset; + return 0; + } + if (part->body_size.physical_size == 0) { + /* no body */ + ctx->copy_start_offset = part_end_offset; + return 0; + } + + /* single part - write decoded data */ + block = array_append_space(&ctx->blocks); + block->physical_pos = part->physical_pos; + + i_stream_seek(ctx->input, part->physical_pos + + part->header_size.physical_size); + linput = i_stream_create_limit(ctx->input, part->body_size.physical_size); + switch (cte) { + case MESSAGE_CTE_UNKNOWN: + i_unreached(); + case MESSAGE_CTE_78BIT: + case MESSAGE_CTE_BINARY: + /* no conversion necessary */ + if ((part->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0) + ctx->has_nuls = TRUE; + block->input = i_stream_create_crlf(linput); + break; + case MESSAGE_CTE_QP: + block->input = i_stream_create_qp_decoder(linput); + ctx->converted = block->converted = TRUE; + break; + case MESSAGE_CTE_BASE64: + block->input = i_stream_create_base64_decoder(linput); + ctx->converted = block->converted = TRUE; + break; + } + i_stream_unref(&linput); + + ctx->copy_start_offset = part_end_offset; + return 0; +} + +static int fd_callback(const char **path_r, void *context) +{ + struct mail *_mail = context; + string_t *path; + int fd; + + path = t_str_new(256); + mail_user_set_get_temp_prefix(path, _mail->box->storage->user->set); + fd = safe_mkstemp_hostpid(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("Temp file creation to %s failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (i_unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_close_fd(&fd); + return -1; + } + *path_r = str_c(path); + return fd; +} + +static void binary_streams_free(struct binary_ctx *ctx) +{ + struct binary_block *block; + + array_foreach_modifiable(&ctx->blocks, block) + i_stream_unref(&block->input); +} + +static void +binary_parts_update(struct binary_ctx *ctx, const struct message_part *part, + struct message_binary_part **msg_bin_parts) +{ + struct index_mail *mail = INDEX_MAIL(ctx->mail); + struct binary_block *blocks; + struct message_binary_part bin_part; + unsigned int i, count; + uoff_t size; + bool found; + + blocks = array_get_modifiable(&ctx->blocks, &count); + for (; part != NULL; part = part->next) { + binary_parts_update(ctx, part->children, msg_bin_parts); + + i_zero(&bin_part); + /* default to unchanged header */ + bin_part.binary_hdr_size = part->header_size.virtual_size; + bin_part.physical_pos = part->physical_pos; + found = FALSE; + for (i = 0; i < count; i++) { + if (blocks[i].physical_pos != part->physical_pos || + !blocks[i].converted) + continue; + + size = blocks[i].input->v_offset; + if (blocks[i].converted_hdr) + bin_part.binary_hdr_size = size; + else + bin_part.binary_body_size = size; + found = TRUE; + } + if (found) { + bin_part.next = *msg_bin_parts; + *msg_bin_parts = p_new(mail->mail.data_pool, + struct message_binary_part, 1); + **msg_bin_parts = bin_part; + } + } +} + +static void binary_parts_cache(struct binary_ctx *ctx) +{ + struct index_mail *mail = INDEX_MAIL(ctx->mail); + buffer_t *buf; + + buf = t_buffer_create(128); + message_binary_part_serialize(mail->data.bin_parts, buf); + index_mail_cache_add(mail, MAIL_CACHE_BINARY_PARTS, + buf->data, buf->used); +} + +static struct istream **blocks_get_streams(struct binary_ctx *ctx) +{ + struct istream **streams; + const struct binary_block *blocks; + unsigned int i, count; + + blocks = array_get(&ctx->blocks, &count); + streams = t_new(struct istream *, count+1); + for (i = 0; i < count; i++) { + streams[i] = blocks[i].input; + i_assert(streams[i]->v_offset == 0); + } + return streams; +} + +static int +blocks_count_lines(struct binary_ctx *ctx, struct istream *full_input) +{ + struct binary_block *blocks, *cur_block; + unsigned int block_idx, block_count; + uoff_t cur_block_offset, cur_block_size; + const unsigned char *data, *p; + size_t size, skip; + ssize_t ret; + + blocks = array_get_modifiable(&ctx->blocks, &block_count); + cur_block = blocks; + cur_block_offset = 0; + block_idx = 0; + + /* count the number of lines each block contains */ + while ((ret = i_stream_read_more(full_input, &data, &size)) > 0) { + i_assert(cur_block_offset <= cur_block->input->v_offset); + if (cur_block->input->eof) { + /* this is the last input for this block. the input + may also contain the next block's data, which we + don't want to include in this block's line count. */ + cur_block_size = cur_block->input->v_offset + + i_stream_get_data_size(cur_block->input); + i_assert(size >= cur_block_size - cur_block_offset); + size = cur_block_size - cur_block_offset; + } + skip = size; + while ((p = memchr(data, '\n', size)) != NULL) { + size -= p-data+1; + data = p+1; + cur_block->body_lines_count++; + } + i_stream_skip(full_input, skip); + cur_block_offset += skip; + + if (i_stream_read_eof(cur_block->input)) { + /* go to the next block */ + if (block_idx+1 == block_count) { + i_assert(i_stream_read_eof(full_input)); + ret = -1; + break; + } + block_idx++; + cur_block++; + cur_block_offset = 0; + } + } + i_assert(ret == -1); + if (full_input->stream_errno != 0) + return -1; + i_assert(block_count == 0 || !i_stream_have_bytes_left(cur_block->input)); + i_assert(block_count == 0 || block_idx+1 == block_count); + return 0; +} + +static int +index_mail_read_binary_to_cache(struct mail *_mail, + const struct message_part *part, + bool include_hdr, const char *reason, + bool *binary_r, bool *converted_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct mail_binary_cache *cache = &_mail->box->storage->binary_cache; + struct binary_ctx ctx; + struct istream *is; + + i_zero(&ctx); + ctx.mail = _mail; + t_array_init(&ctx.blocks, 8); + + mail_storage_free_binary_cache(_mail->box->storage); + if (mail_get_stream_because(_mail, NULL, NULL, reason, &ctx.input) < 0) + return -1; + + if (add_binary_part(&ctx, part, include_hdr) < 0) { + binary_streams_free(&ctx); + return -1; + } + + if (array_count(&ctx.blocks) != 0) { + is = i_streams_merge(blocks_get_streams(&ctx), + IO_BLOCK_SIZE, + fd_callback, _mail); + } else { + is = i_stream_create_from_data("", 0); + } + i_stream_set_name(is, t_strdup_printf( + "<binary stream of mailbox %s UID %u>", + _mail->box->vname, _mail->uid)); + if (blocks_count_lines(&ctx, is) < 0) { + if (is->stream_errno == EINVAL) { + /* MIME part contains invalid data */ + mail_storage_set_error(_mail->box->storage, + MAIL_ERROR_INVALIDDATA, + "Invalid data in MIME part"); + } else { + mail_set_critical(_mail, "read(%s) failed: %s", + i_stream_get_name(is), + i_stream_get_error(is)); + } + i_stream_unref(&is); + binary_streams_free(&ctx); + return -1; + } + + if (_mail->uid > 0) { + cache->to = timeout_add(MAIL_BINARY_CACHE_EXPIRE_MSECS, + mail_storage_free_binary_cache, + _mail->box->storage); + cache->box = _mail->box; + cache->uid = _mail->uid; + cache->orig_physical_pos = part->physical_pos; + cache->include_hdr = include_hdr; + cache->input = is; + } + + i_assert(!i_stream_have_bytes_left(is)); + cache->size = is->v_offset; + i_stream_seek(is, 0); + + if (part->parent == NULL && include_hdr && + mail->data.bin_parts == NULL) { + binary_parts_update(&ctx, part, &mail->data.bin_parts); + if (_mail->uid > 0) + binary_parts_cache(&ctx); + } + binary_streams_free(&ctx); + + *binary_r = ctx.converted ? TRUE : ctx.has_nuls; + *converted_r = ctx.converted; + return 0; +} + +static bool get_cached_binary_parts(struct index_mail *mail) +{ + const unsigned int field_idx = + mail->ibox->cache_fields[MAIL_CACHE_BINARY_PARTS].idx; + buffer_t *part_buf; + int ret; + + if (mail->data.bin_parts != NULL) + return TRUE; + + part_buf = t_buffer_create(128); + ret = index_mail_cache_lookup_field(mail, part_buf, field_idx); + if (ret <= 0) + return FALSE; + + if (message_binary_part_deserialize(mail->mail.data_pool, + part_buf->data, part_buf->used, + &mail->data.bin_parts) < 0) { + mail_set_mail_cache_corrupted(&mail->mail.mail, + "Corrupted cached binary.parts data"); + return FALSE; + } + return TRUE; +} + +static struct message_part * +msg_part_find(struct message_part *parts, uoff_t physical_pos) +{ + struct message_part *part, *child; + + for (part = parts; part != NULL; part = part->next) { + if (part->physical_pos == physical_pos) + return part; + child = msg_part_find(part->children, physical_pos); + if (child != NULL) + return child; + } + return NULL; +} + +static int +index_mail_get_binary_size(struct mail *_mail, + const struct message_part *part, bool include_hdr, + uoff_t *size_r, unsigned int *lines_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct message_part *all_parts, *msg_part; + const struct message_binary_part *bin_part, *root_bin_part; + uoff_t size, end_offset; + unsigned int lines; + bool binary, converted; + + if (mail_get_parts(_mail, &all_parts) < 0) + return -1; + + /* first lookup from cache */ + if (!get_cached_binary_parts(mail)) { + /* not found. parse the whole message */ + if (index_mail_read_binary_to_cache(_mail, all_parts, TRUE, + "binary.size", &binary, &converted) < 0) + return -1; + } + + size = part->header_size.virtual_size + + part->body_size.virtual_size; + /* note that we assume here that binary translation doesn't change the + headers' line counts. this isn't true if the original message + contained duplicate Content-Transfer-Encoding lines, but since + that's invalid anyway we don't bother trying to handle it. */ + lines = part->header_size.lines + part->body_size.lines; + end_offset = part->physical_pos + size; + + bin_part = mail->data.bin_parts; root_bin_part = NULL; + for (; bin_part != NULL; bin_part = bin_part->next) { + msg_part = msg_part_find(all_parts, bin_part->physical_pos); + if (msg_part == NULL) { + /* either binary.parts or mime.parts is broken */ + mail_set_cache_corrupted(_mail, MAIL_FETCH_MESSAGE_PARTS, t_strdup_printf( + "BINARY part at offset %"PRIuUOFF_T" not found from MIME parts", + bin_part->physical_pos)); + return -1; + } + if (msg_part->physical_pos >= part->physical_pos && + msg_part->physical_pos < end_offset) { + if (msg_part->physical_pos == part->physical_pos) + root_bin_part = bin_part; + size -= msg_part->header_size.virtual_size + + msg_part->body_size.virtual_size; + size += bin_part->binary_hdr_size + + bin_part->binary_body_size; + lines -= msg_part->body_size.lines; + lines += bin_part->binary_body_lines_count; + } + } + if (!include_hdr) { + if (root_bin_part != NULL) + size -= root_bin_part->binary_hdr_size; + else + size -= part->header_size.virtual_size; + lines -= part->header_size.lines; + } + *size_r = size; + *lines_r = lines; + return 0; +} + +int index_mail_get_binary_stream(struct mail *_mail, + const struct message_part *part, + bool include_hdr, uoff_t *size_r, + unsigned int *lines_r, bool *binary_r, + struct istream **stream_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct mail_binary_cache *cache = &_mail->box->storage->binary_cache; + struct istream *input; + bool binary, converted; + + if (stream_r == NULL) { + return index_mail_get_binary_size(_mail, part, include_hdr, + size_r, lines_r); + } + /* current implementation doesn't bother implementing this, + because it's not needed by anything. */ + i_assert(lines_r == NULL); + + /* FIXME: always put the header to temp file. skip it when needed. */ + if (cache->box == _mail->box && cache->uid == _mail->uid && + cache->orig_physical_pos == part->physical_pos && + cache->include_hdr == include_hdr) { + /* we have this cached already */ + i_stream_seek(cache->input, 0); + timeout_reset(cache->to); + binary = TRUE; + converted = TRUE; + } else { + if (index_mail_read_binary_to_cache(_mail, part, include_hdr, + "binary stream", &binary, &converted) < 0) + return -1; + mail->data.cache_fetch_fields |= MAIL_FETCH_STREAM_BINARY; + } + *size_r = cache->size; + *binary_r = binary; + if (!converted) { + /* don't keep this cached. it's exactly the same as + the original stream */ + i_assert(mail->data.stream != NULL); + i_stream_seek(mail->data.stream, part->physical_pos + + (include_hdr ? 0 : + part->header_size.physical_size)); + input = i_stream_create_crlf(mail->data.stream); + *stream_r = i_stream_create_limit(input, *size_r); + i_stream_unref(&input); + mail_storage_free_binary_cache(_mail->box->storage); + } else { + *stream_r = cache->input; + i_stream_ref(cache->input); + } + return 0; +} diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c new file mode 100644 index 0000000..ce23e9d --- /dev/null +++ b/src/lib-storage/index/index-mail-headers.c @@ -0,0 +1,990 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "array.h" +#include "buffer.h" +#include "str.h" +#include "message-date.h" +#include "message-part-data.h" +#include "message-parser.h" +#include "message-header-decode.h" +#include "istream-tee.h" +#include "istream-header-filter.h" +#include "imap-envelope.h" +#include "imap-bodystructure.h" +#include "index-storage.h" +#include "index-mail.h" + +static const struct message_parser_settings msg_parser_set = { + .hdr_flags = MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP | + MESSAGE_HEADER_PARSER_FLAG_DROP_CR, + .flags = MESSAGE_PARSER_FLAG_SKIP_BODY_BLOCK, +}; + +static void index_mail_filter_stream_destroy(struct index_mail *mail); + +static int header_line_cmp(const struct index_mail_line *l1, + const struct index_mail_line *l2) +{ + int diff; + + diff = (int)l1->field_idx - (int)l2->field_idx; + return diff != 0 ? diff : + (int)l1->line_num - (int)l2->line_num; +} + +void index_mail_parse_header_deinit(struct index_mail *mail) +{ + mail->data.header_parser_initialized = FALSE; +} + +static void index_mail_parse_header_finish(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + const struct index_mail_line *lines; + const unsigned char *header; + const uint8_t *match; + buffer_t *buf; + unsigned int i, j, count, match_idx, match_count; + bool noncontiguous; + + /* sort it first so fields are grouped together and ordered by + line number */ + array_sort(&mail->header_lines, header_line_cmp); + + lines = array_get(&mail->header_lines, &count); + match = array_get(&mail->header_match, &match_count); + header = mail->header_data->data; + buf = t_buffer_create(256); + + /* go through all the header lines we found */ + for (i = match_idx = 0; i < count; i = j) { + /* matches and header lines are both sorted, all matches + until lines[i] weren't found */ + while (match_idx < lines[i].field_idx && + match_idx < match_count) { + if (HEADER_MATCH_USABLE(mail, match[match_idx]) && + mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, match_idx)) { + /* this header doesn't exist. remember that. */ + i_assert((match[match_idx] & + HEADER_MATCH_FLAG_FOUND) == 0); + index_mail_cache_add_idx(mail, match_idx, + "", 0); + } + match_idx++; + } + + if (match_idx < match_count) { + /* save index to first header line */ + i_assert(match_idx == lines[i].field_idx); + j = i + 1; + array_idx_set(&mail->header_match_lines, match_idx, &j); + match_idx++; + } + + if (!mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, lines[i].field_idx)) { + /* header is already cached. skip over all the + header lines. */ + for (j = i+1; j < count; j++) { + if (lines[j].field_idx != lines[i].field_idx) + break; + } + continue; + } + + /* buffer contains: { uint32_t line_num[], 0, header texts } + noncontiguous is just a small optimization.. */ + buffer_set_used_size(buf, 0); + buffer_append(buf, &lines[i].line_num, + sizeof(lines[i].line_num)); + + noncontiguous = FALSE; + for (j = i+1; j < count; j++) { + if (lines[j].field_idx != lines[i].field_idx) + break; + + if (lines[j].start_pos != lines[j-1].end_pos) + noncontiguous = TRUE; + buffer_append(buf, &lines[j].line_num, + sizeof(lines[j].line_num)); + } + buffer_append_zero(buf, sizeof(uint32_t)); + + if (noncontiguous) { + for (; i < j; i++) { + buffer_append(buf, header + lines[i].start_pos, + lines[i].end_pos - + lines[i].start_pos); + } + i--; + } else { + buffer_append(buf, header + lines[i].start_pos, + lines[j-1].end_pos - lines[i].start_pos); + } + + index_mail_cache_add_idx(mail, lines[i].field_idx, + buf->data, buf->used); + } + + for (; match_idx < match_count; match_idx++) { + if (HEADER_MATCH_USABLE(mail, match[match_idx]) && + mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, match_idx)) { + /* this header doesn't exist. remember that. */ + i_assert((match[match_idx] & + HEADER_MATCH_FLAG_FOUND) == 0); + index_mail_cache_add_idx(mail, match_idx, "", 0); + } + } + + mail->data.dont_cache_field_idx = UINT_MAX; + index_mail_parse_header_deinit(mail); +} + +static unsigned int +get_header_field_idx(struct mailbox *box, const char *field) +{ + struct mail_cache_field header_field; + + i_zero(&header_field); + header_field.type = MAIL_CACHE_FIELD_HEADER; + /* Always register with NO decision. The field should be added soon + with mail_cache_add(), which changes the decision to TEMP. Most + importantly doing it this way emits mail_cache_decision event. */ + header_field.decision = MAIL_CACHE_DECISION_NO; + T_BEGIN { + header_field.name = t_strconcat("hdr.", field, NULL); + mail_cache_register_fields(box->cache, &header_field, 1); + } T_END; + return header_field.idx; +} + +bool index_mail_want_parse_headers(struct index_mail *mail) +{ + if (mail->data.wanted_headers != NULL || + mail->data.save_bodystructure_header) + return TRUE; + + if ((mail->data.cache_fetch_fields & MAIL_FETCH_DATE) != 0 && + !mail->data.sent_date_parsed) + return TRUE; + return FALSE; +} + +static void index_mail_parse_header_register_all_wanted(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + const struct mail_cache_field *all_cache_fields; + unsigned int i, count; + + all_cache_fields = + mail_cache_register_get_list(_mail->box->cache, + pool_datastack_create(), &count); + for (i = 0; i < count; i++) { + if (strncasecmp(all_cache_fields[i].name, "hdr.", 4) != 0) + continue; + if (!mail_cache_field_want_add(_mail->transaction->cache_trans, + _mail->seq, i)) + continue; + + array_idx_set(&mail->header_match, all_cache_fields[i].idx, + &mail->header_match_value); + } +} + +void index_mail_parse_header_init(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers) +{ + struct index_mail_data *data = &mail->data; + const uint8_t *match; + unsigned int i, field_idx, match_count; + + index_mail_filter_stream_destroy(mail); + i_assert(!mail->data.header_parser_initialized); + + mail->header_seq = mail->mail.mail.seq; + if (mail->header_data == NULL) { + mail->header_data = buffer_create_dynamic(default_pool, 4096); + i_array_init(&mail->header_lines, 32); + i_array_init(&mail->header_match, 32); + i_array_init(&mail->header_match_lines, 32); + mail->header_match_value = HEADER_MATCH_SKIP_COUNT; + } else { + buffer_set_used_size(mail->header_data, 0); + array_clear(&mail->header_lines); + array_clear(&mail->header_match_lines); + + i_assert((mail->header_match_value & + (HEADER_MATCH_SKIP_COUNT-1)) == 0); + if (mail->header_match_value + HEADER_MATCH_SKIP_COUNT <= UINT8_MAX) + mail->header_match_value += HEADER_MATCH_SKIP_COUNT; + else { + /* wrapped, we'll have to clear the buffer */ + array_clear(&mail->header_match); + mail->header_match_value = HEADER_MATCH_SKIP_COUNT; + } + } + + if (headers != NULL) { + for (i = 0; i < headers->count; i++) { + array_idx_set(&mail->header_match, headers->idx[i], + &mail->header_match_value); + } + } + + if (data->wanted_headers != NULL && data->wanted_headers != headers) { + headers = data->wanted_headers; + for (i = 0; i < headers->count; i++) { + array_idx_set(&mail->header_match, headers->idx[i], + &mail->header_match_value); + } + } + + /* register also all the other headers that exist in cache file */ + T_BEGIN { + index_mail_parse_header_register_all_wanted(mail); + } T_END; + + /* if we want sent date, it doesn't mean that we also want to cache + Date: header. if we have Date field's index set at this point we + know that we want it. otherwise add it and remember that we don't + want it cached. */ + field_idx = get_header_field_idx(mail->mail.mail.box, "Date"); + match = array_get(&mail->header_match, &match_count); + if (field_idx < match_count && + match[field_idx] == mail->header_match_value) { + /* cache Date: header */ + } else if ((data->cache_fetch_fields & MAIL_FETCH_DATE) != 0 || + data->save_sent_date) { + /* parse Date: header, but don't cache it. */ + data->dont_cache_field_idx = field_idx; + array_idx_set(&mail->header_match, field_idx, + &mail->header_match_value); + } + mail->data.header_parser_initialized = TRUE; + mail->data.parse_line_num = 0; + i_zero(&mail->data.parse_line); +} + +static void index_mail_parse_finish_imap_envelope(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + const unsigned int cache_field_envelope = + mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx; + string_t *str; + + str = str_new(mail->mail.data_pool, 256); + imap_envelope_write(mail->data.envelope_data, str); + mail->data.envelope = str_c(str); + mail->data.save_envelope = FALSE; + + if (mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, cache_field_envelope)) { + index_mail_cache_add_idx(mail, cache_field_envelope, + str_data(str), str_len(str)); + } +} + +void index_mail_parse_header(struct message_part *part, + struct message_header_line *hdr, + struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + struct index_mail_data *data = &mail->data; + unsigned int field_idx, count; + uint8_t *match; + + i_assert(data->header_parser_initialized); + + data->parse_line_num++; + + if (data->save_bodystructure_header && + !data->parsed_bodystructure_header) { + i_assert(part != NULL); + message_part_data_parse_from_header(mail->mail.data_pool, part, hdr); + } + + if (data->save_envelope) { + message_part_envelope_parse_from_header(mail->mail.data_pool, + &data->envelope_data, hdr); + + if (hdr == NULL) + index_mail_parse_finish_imap_envelope(mail); + } + + if (hdr == NULL) { + /* end of headers */ + if (mail->data.save_sent_date) + mail->data.sent_date_parsed = TRUE; + T_BEGIN { + index_mail_parse_header_finish(mail); + } T_END; + if (data->save_bodystructure_header) { + i_assert(data->parser_ctx != NULL); + data->parsed_bodystructure_header = TRUE; + } + return; + } + + if (!hdr->continued) { + T_BEGIN { + const char *cache_field_name = + t_strconcat("hdr.", hdr->name, NULL); + data->parse_line.field_idx = + mail_cache_register_lookup(_mail->box->cache, + cache_field_name); + } T_END; + } + field_idx = data->parse_line.field_idx; + match = array_get_modifiable(&mail->header_match, &count); + if (field_idx >= count || + !HEADER_MATCH_USABLE(mail, match[field_idx])) { + /* we don't want this header. */ + return; + } + + if (!hdr->continued) { + /* beginning of a line. add the header name. */ + data->parse_line.start_pos = str_len(mail->header_data); + data->parse_line.line_num = data->parse_line_num; + str_append(mail->header_data, hdr->name); + str_append_data(mail->header_data, hdr->middle, hdr->middle_len); + + /* remember that we saw this header so we don't add it to + cache as nonexistent. */ + match[field_idx] |= HEADER_MATCH_FLAG_FOUND; + } + str_append_data(mail->header_data, hdr->value, hdr->value_len); + if (!hdr->no_newline) + str_append(mail->header_data, "\n"); + if (!hdr->continues) { + data->parse_line.end_pos = str_len(mail->header_data); + array_push_back(&mail->header_lines, &data->parse_line); + } +} + +static void +index_mail_parse_part_header_cb(struct message_part *part, + struct message_header_line *hdr, + struct index_mail *mail) +{ + index_mail_parse_header(part, hdr, mail); +} + +static void +index_mail_parse_header_cb(struct message_header_line *hdr, + struct index_mail *mail) +{ + index_mail_parse_header(mail->data.parts, hdr, mail); +} + +struct istream * +index_mail_cache_parse_init(struct mail *_mail, struct istream *input) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct istream *input2; + + i_assert(mail->data.tee_stream == NULL); + i_assert(mail->data.parser_ctx == NULL); + + /* we're doing everything for now, figure out later if we want to + save them. */ + mail->data.save_sent_date = TRUE; + mail->data.save_bodystructure_header = TRUE; + mail->data.save_bodystructure_body = TRUE; + /* Don't unnecessarily waste time generating a snippet, since it's + not as cheap as the others to generate. */ + if (index_mail_want_cache(mail, MAIL_CACHE_BODY_SNIPPET)) + mail->data.save_body_snippet = TRUE; + + mail->data.tee_stream = tee_i_stream_create(input); + input = tee_i_stream_create_child(mail->data.tee_stream); + input2 = tee_i_stream_create_child(mail->data.tee_stream); + + index_mail_parse_header_init(mail, NULL); + mail->data.parser_input = input; + mail->data.parser_ctx = + message_parser_init(mail->mail.data_pool, input, + &msg_parser_set); + i_stream_unref(&input); + return input2; +} + +static void index_mail_init_parser(struct index_mail *mail) +{ + struct index_mail_data *data = &mail->data; + struct message_part *parts; + const char *error; + + if (data->parser_ctx != NULL) { + data->parser_input = NULL; + if (message_parser_deinit_from_parts(&data->parser_ctx, &parts, &error) < 0) { + index_mail_set_message_parts_corrupted(&mail->mail.mail, error); + index_mail_parts_reset(mail); + } + if (data->parts == NULL || data->parts != parts) { + /* The previous parsing didn't finish, so we're + re-parsing the header. The new parts don't have data + filled anymore. */ + data->parsed_bodystructure_header = FALSE; + } + } + + /* make sure parsing starts from the beginning of the stream */ + i_stream_seek(mail->data.stream, 0); + if (data->parts == NULL) { + data->parser_input = data->stream; + data->parser_ctx = message_parser_init(mail->mail.data_pool, + data->stream, + &msg_parser_set); + } else { + data->parser_ctx = + message_parser_init_from_parts(data->parts, + data->stream, + &msg_parser_set); + } +} + +int index_mail_parse_headers_internal(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers) +{ + struct index_mail_data *data = &mail->data; + + i_assert(data->stream != NULL); + + index_mail_parse_header_init(mail, headers); + + if (data->parts == NULL || data->save_bodystructure_header || + (data->access_part & PARSE_BODY) != 0) { + /* initialize bodystructure parsing in case we read the whole + message. */ + index_mail_init_parser(mail); + message_parser_parse_header(data->parser_ctx, &data->hdr_size, + index_mail_parse_part_header_cb, + mail); + } else { + /* just read the header */ + i_assert(!data->save_bodystructure_body || + data->parser_ctx != NULL); + message_parse_header(data->stream, &data->hdr_size, + msg_parser_set.hdr_flags, + index_mail_parse_header_cb, mail); + } + if (index_mail_stream_check_failure(mail) < 0) { + index_mail_parse_header_deinit(mail); + return -1; + } + i_assert(!mail->data.header_parser_initialized); + data->hdr_size_set = TRUE; + data->access_part &= ENUM_NEGATE(PARSE_HDR); + return 0; +} + +int index_mail_parse_headers(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers, + const char *reason) +{ + struct index_mail_data *data = &mail->data; + struct istream *input; + uoff_t old_offset; + + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + + if (mail_get_hdr_stream_because(&mail->mail.mail, NULL, reason, &input) < 0) + return -1; + + int ret = index_mail_parse_headers_internal(mail, headers); + i_stream_seek(data->stream, old_offset); + return ret; +} + +static void +imap_envelope_parse_callback(struct message_header_line *hdr, + struct index_mail *mail) +{ + message_part_envelope_parse_from_header(mail->mail.data_pool, + &mail->data.envelope_data, hdr); + + if (hdr == NULL) + index_mail_parse_finish_imap_envelope(mail); +} + +int index_mail_headers_get_envelope(struct index_mail *mail) +{ + const unsigned int cache_field_envelope = + mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx; + struct mailbox_header_lookup_ctx *header_ctx; + struct istream *stream; + uoff_t old_offset; + string_t *str; + + str = str_new(mail->mail.data_pool, 256); + if (index_mail_cache_lookup_field(mail, str, + cache_field_envelope) > 0) { + mail->data.envelope = str_c(str); + return 0; + } + str_free(&str); + + old_offset = mail->data.stream == NULL ? 0 : + mail->data.stream->v_offset; + + /* Make sure header_cache_callback() isn't also parsing the ENVELOPE. + Otherwise two callbacks are doing it and mixing up results. */ + mail->data.save_envelope = FALSE; + + header_ctx = mailbox_header_lookup_init(mail->mail.mail.box, + message_part_envelope_headers); + if (mail_get_header_stream(&mail->mail.mail, header_ctx, &stream) < 0) { + mailbox_header_lookup_unref(&header_ctx); + return -1; + } + mailbox_header_lookup_unref(&header_ctx); + + if (mail->data.envelope == NULL) { + /* we got the headers from cache - parse them to get the + envelope */ + message_parse_header(stream, NULL, msg_parser_set.hdr_flags, + imap_envelope_parse_callback, mail); + if (stream->stream_errno != 0) { + index_mail_stream_log_failure_for(mail, stream); + return -1; + } + i_assert(mail->data.envelope != NULL); + } + + if (mail->data.stream != NULL) + i_stream_seek(mail->data.stream, old_offset); + return 0; +} + +static size_t get_header_size(buffer_t *buffer, size_t pos) +{ + const unsigned char *data = buffer->data; + size_t i, size = buffer->used; + + i_assert(pos <= size); + + for (i = pos; i < size; i++) { + if (data[i] == '\n') { + if (i+1 == size || + (data[i+1] != ' ' && data[i+1] != '\t')) + return i - pos; + } + } + return size - pos; +} + +static int index_mail_header_is_parsed(struct index_mail *mail, + unsigned int field_idx) +{ + const uint8_t *match; + unsigned int count; + + match = array_get(&mail->header_match, &count); + if (field_idx < count && HEADER_MATCH_USABLE(mail, match[field_idx])) + return (match[field_idx] & HEADER_MATCH_FLAG_FOUND) != 0 ? 1 : 0; + return -1; +} + +static bool skip_header(const unsigned char **data, size_t len) +{ + const unsigned char *p = *data; + size_t i; + + for (i = 0; i < len; i++) { + if (p[i] == ':') + break; + } + if (i == len) + return FALSE; + + for (i++; i < len; i++) { + if (!IS_LWSP(p[i])) + break; + } + + *data = p + i; + return TRUE; +} + +static const char *const * +index_mail_get_parsed_header(struct index_mail *mail, unsigned int field_idx) +{ + ARRAY(const char *) header_values; + const struct index_mail_line *lines; + const unsigned char *header, *value_start, *value_end; + const unsigned int *line_idx; + const char *value; + unsigned int i, lines_count, first_line_idx; + + line_idx = array_idx(&mail->header_match_lines, field_idx); + i_assert(*line_idx != 0); + first_line_idx = *line_idx - 1; + + p_array_init(&header_values, mail->mail.data_pool, 4); + header = mail->header_data->data; + + lines = array_get(&mail->header_lines, &lines_count); + for (i = first_line_idx; i < lines_count; i++) { + if (lines[i].field_idx != lines[first_line_idx].field_idx) + break; + + /* skip header: and drop ending LF */ + value_start = header + lines[i].start_pos; + value_end = header + lines[i].end_pos; + if (skip_header(&value_start, value_end - value_start)) { + if (value_start != value_end && value_end[-1] == '\n') + value_end--; + value = message_header_strdup(mail->mail.data_pool, + value_start, + value_end - value_start); + array_push_back(&header_values, &value); + } + } + + array_append_zero(&header_values); + return array_front(&header_values); +} + +static int +index_mail_get_raw_headers(struct index_mail *mail, const char *field, + const char *const **value_r) +{ + struct mail *_mail = &mail->mail.mail; + const char *headers[2], *value; + struct mailbox_header_lookup_ctx *headers_ctx; + const unsigned char *data; + unsigned int field_idx; + string_t *dest; + size_t i, len, len2; + int ret; + ARRAY(const char *) header_values; + + i_assert(field != NULL); + + field_idx = get_header_field_idx(_mail->box, field); + + dest = t_str_new(128); + if (mail_cache_lookup_headers(_mail->transaction->cache_view, dest, + _mail->seq, &field_idx, 1) <= 0) { + /* not in cache / error - first see if it's already parsed */ + p_free(mail->mail.data_pool, dest); + if (mail->data.header_parser_initialized) { + /* don't try to parse headers recursively. we're here + because message size was wrong and istream-mail + wants to log some cached headers. */ + i_assert(mail->mail.mail.lookup_abort >= MAIL_LOOKUP_ABORT_NOT_IN_CACHE); + mail_set_aborted(&mail->mail.mail); + return -1; + } + if (mail->header_seq != mail->mail.mail.seq || + index_mail_header_is_parsed(mail, field_idx) < 0) { + /* parse */ + const char *reason = index_mail_cache_reason(_mail, + t_strdup_printf("header %s", field)); + headers[0] = field; headers[1] = NULL; + headers_ctx = mailbox_header_lookup_init(_mail->box, + headers); + ret = index_mail_parse_headers(mail, headers_ctx, reason); + mailbox_header_lookup_unref(&headers_ctx); + if (ret < 0) + return -1; + } + + if ((ret = index_mail_header_is_parsed(mail, field_idx)) <= 0) { + /* not found */ + i_assert(ret != -1); + *value_r = p_new(mail->mail.data_pool, const char *, 1); + return 0; + } + *value_r = index_mail_get_parsed_header(mail, field_idx); + return 0; + } + _mail->transaction->stats.cache_hit_count++; + data = buffer_get_data(dest, &len); + + if (len == 0) { + /* cached as nonexistent. */ + *value_r = p_new(mail->mail.data_pool, const char *, 1); + return 0; + } + + p_array_init(&header_values, mail->mail.data_pool, 4); + + /* cached. skip "header name: " parts in dest. */ + for (i = 0; i < len; i++) { + if (data[i] == ':') { + i++; + while (i < len && IS_LWSP(data[i])) i++; + + /* @UNSAFE */ + len2 = get_header_size(dest, i); + value = message_header_strdup(mail->mail.data_pool, + data + i, len2); + i += len2 + 1; + + array_push_back(&header_values, &value); + } + } + + array_append_zero(&header_values); + *value_r = array_front(&header_values); + return 0; +} + +static int unfold_header(pool_t pool, const char **_str) +{ + const char *str = *_str; + char *new_str; + unsigned int i, j; + + for (i = 0; str[i] != '\0'; i++) { + if (str[i] == '\n') + break; + } + if (str[i] == '\0') + return 0; + + /* @UNSAFE */ + new_str = p_malloc(pool, i + strlen(str+i) + 1); + memcpy(new_str, str, i); + for (j = i; str[i] != '\0'; i++) { + if (str[i] == '\n') { + new_str[j++] = ' '; + i++; + if (str[i] == '\0') + break; + + if (str[i] != ' ' && str[i] != '\t') { + /* corrupted */ + return -1; + } + } else { + new_str[j++] = str[i]; + } + } + new_str[j] = '\0'; + *_str = new_str; + return 0; +} + +static void str_replace_nuls(string_t *str) +{ + char *data = str_c_modifiable(str); + size_t i, len = str_len(str); + + for (i = 0; i < len; i++) { + if (data[i] == '\0') + data[i] = ' '; + } +} + +static int +index_mail_headers_decode(struct index_mail *mail, const char *const **_list, + unsigned int max_count) +{ + const char *const *list = *_list; + const char **decoded_list, *input; + unsigned int i, count; + string_t *str; + + count = str_array_length(list); + if (count > max_count) + count = max_count; + decoded_list = p_new(mail->mail.data_pool, const char *, count + 1); + + str = t_str_new(512); + for (i = 0; i < count; i++) { + str_truncate(str, 0); + input = list[i]; + /* unfold all lines into a single line */ + if (unfold_header(mail->mail.data_pool, &input) < 0) + return -1; + + /* decode MIME encoded-words. decoding may also add new LFs. */ + message_header_decode_utf8((const unsigned char *)input, + strlen(input), str, NULL); + if (strcmp(str_c(str), input) != 0) { + if (strlen(str_c(str)) != str_len(str)) { + /* replace NULs with spaces */ + str_replace_nuls(str); + } + input = p_strdup(mail->mail.data_pool, str_c(str)); + } + decoded_list[i] = input; + } + *_list = decoded_list; + return 0; +} + +int index_mail_get_headers(struct mail *_mail, const char *field, + bool decode_to_utf8, const char *const **value_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + bool retry = TRUE; + int ret; + + for (;; retry = FALSE) { + if (index_mail_get_raw_headers(mail, field, value_r) < 0) + return -1; + if (**value_r == NULL) + return 0; + if (!decode_to_utf8) + return 1; + + T_BEGIN { + ret = index_mail_headers_decode(mail, value_r, UINT_MAX); + } T_END; + + if (ret < 0 && retry) { + mail_set_mail_cache_corrupted(_mail, "Broken header %s", + field); + } else { + break; + } + } + if (ret < 0) { + i_panic("BUG: Broken header %s for mail UID %u " + "wasn't fixed by re-parsing the header", + field, _mail->uid); + } + return 1; +} + +int index_mail_get_first_header(struct mail *_mail, const char *field, + bool decode_to_utf8, const char **value_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + const char *const *list; + bool retry = TRUE; + int ret; + + for (;; retry = FALSE) { + if (index_mail_get_raw_headers(mail, field, &list) < 0) + return -1; + if (!decode_to_utf8 || list[0] == NULL) { + ret = 0; + break; + } + + T_BEGIN { + ret = index_mail_headers_decode(mail, &list, 1); + } T_END; + + if (ret < 0 && retry) { + mail_set_mail_cache_corrupted(_mail, "Broken header %s", + field); + /* retry by parsing the full header */ + } else { + break; + } + } + if (ret < 0) { + i_panic("BUG: Broken header %s for mail UID %u " + "wasn't fixed by re-parsing the header", + field, _mail->uid); + } + *value_r = list[0]; + return list[0] != NULL ? 1 : 0; +} + +static void +header_cache_callback(struct header_filter_istream *input ATTR_UNUSED, + struct message_header_line *hdr, + bool *matched ATTR_UNUSED, struct index_mail *mail) +{ + index_mail_parse_header(NULL, hdr, mail); +} + +static void index_mail_filter_stream_destroy(struct index_mail *mail) +{ + if (mail->data.filter_stream == NULL) + return; + + const unsigned char *data; + size_t size; + + /* read through the previous filter_stream. this makes sure that the + fields are added to cache, and most importantly it resets + header_parser_initialized=FALSE so we don't assert on it. */ + while (i_stream_read_more(mail->data.filter_stream, &data, &size) > 0) + i_stream_skip(mail->data.filter_stream, size); + if (mail->data.header_parser_initialized) { + /* istream failed while reading the header */ + i_assert(mail->data.filter_stream->stream_errno != 0); + index_mail_parse_header_deinit(mail); + } + i_stream_destroy(&mail->data.filter_stream); +} + +int index_mail_get_header_stream(struct mail *_mail, + struct mailbox_header_lookup_ctx *headers, + struct istream **stream_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct istream *input; + string_t *dest; + + index_mail_filter_stream_destroy(mail); + + if (mail->data.save_bodystructure_header) { + /* we have to parse the header. */ + const char *reason = + index_mail_cache_reason(_mail, "bodystructure"); + mail->data.access_reason_code = "mail:header_fields"; + if (index_mail_parse_headers(mail, headers, reason) < 0) + return -1; + } + + dest = str_new(mail->mail.data_pool, 256); + if (mail_cache_lookup_headers(_mail->transaction->cache_view, dest, + _mail->seq, headers->idx, + headers->count) > 0) { + str_append(dest, "\n"); + _mail->transaction->stats.cache_hit_count++; + mail->data.filter_stream = + i_stream_create_from_data(str_data(dest), + str_len(dest)); + *stream_r = mail->data.filter_stream; + return 0; + } + /* not in cache / error */ + p_free(mail->mail.data_pool, dest); + + unsigned int first_not_found = UINT_MAX, not_found_count = 0; + for (unsigned int i = 0; i < headers->count; i++) { + if (mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, headers->idx[i]) <= 0) { + if (not_found_count++ == 0) + first_not_found = i; + } + } + + const char *reason; + if (not_found_count == 0) + reason = "BUG: all headers seem to exist in cache"; + else { + i_assert(first_not_found != UINT_MAX); + reason = index_mail_cache_reason(_mail, t_strdup_printf( + "%u/%u headers not cached (first=%s)", + not_found_count, headers->count, headers->name[first_not_found])); + } + mail->data.access_reason_code = "mail:header_fields"; + if (mail_get_hdr_stream_because(_mail, NULL, reason, &input) < 0) + return -1; + + index_mail_parse_header_init(mail, headers); + mail->data.filter_stream = + i_stream_create_header_filter(mail->data.stream, + HEADER_FILTER_INCLUDE | + HEADER_FILTER_ADD_MISSING_EOH | + HEADER_FILTER_HIDE_BODY, + headers->name, headers->count, + header_cache_callback, mail); + *stream_r = mail->data.filter_stream; + return 0; +} diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c new file mode 100644 index 0000000..a5a280d --- /dev/null +++ b/src/lib-storage/index/index-mail.c @@ -0,0 +1,2625 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "ioloop.h" +#include "istream.h" +#include "hex-binary.h" +#include "str.h" +#include "mailbox-recent-flags.h" +#include "message-date.h" +#include "message-part-data.h" +#include "message-part-serialize.h" +#include "message-parser.h" +#include "message-snippet.h" +#include "imap-bodystructure.h" +#include "imap-envelope.h" +#include "mail-cache.h" +#include "mail-index-modseq.h" +#include "index-storage.h" +#include "istream-mail.h" +#include "index-mail.h" + +#include <fcntl.h> + +#define BODY_SNIPPET_ALGO_V1 "1" +#define BODY_SNIPPET_MAX_CHARS 200 + +struct mail_cache_field global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT] = { + { .name = "flags", + .type = MAIL_CACHE_FIELD_BITMASK, + .field_size = sizeof(uint32_t) }, + { .name = "date.sent", + .type = MAIL_CACHE_FIELD_FIXED_SIZE, + .field_size = sizeof(struct mail_sent_date) }, + { .name = "date.received", + .type = MAIL_CACHE_FIELD_FIXED_SIZE, + .field_size = sizeof(uint32_t) }, + { .name = "date.save", + .type = MAIL_CACHE_FIELD_FIXED_SIZE, + .field_size = sizeof(uint32_t) }, + { .name = "size.virtual", + .type = MAIL_CACHE_FIELD_FIXED_SIZE, + .field_size = sizeof(uoff_t) }, + { .name = "size.physical", + .type = MAIL_CACHE_FIELD_FIXED_SIZE, + .field_size = sizeof(uoff_t) }, + { .name = "imap.body", + .type = MAIL_CACHE_FIELD_STRING }, + { .name = "imap.bodystructure", + .type = MAIL_CACHE_FIELD_STRING }, + { .name = "imap.envelope", + .type = MAIL_CACHE_FIELD_STRING }, + { .name = "pop3.uidl", + .type = MAIL_CACHE_FIELD_STRING }, + { .name = "pop3.order", + .type = MAIL_CACHE_FIELD_FIXED_SIZE, + .field_size = sizeof(uint32_t) }, + { .name = "guid", + .type = MAIL_CACHE_FIELD_STRING }, + { .name = "mime.parts", + .type = MAIL_CACHE_FIELD_VARIABLE_SIZE }, + { .name = "binary.parts", + .type = MAIL_CACHE_FIELD_VARIABLE_SIZE }, + { .name = "body.snippet", + .type = MAIL_CACHE_FIELD_VARIABLE_SIZE } + /* FIXME: for now need to update get_metadata_precache_fields() in + index-status.c when adding more fields. those fields should probably + just be moved here to the same struct. */ +}; + +static void index_mail_init_data(struct index_mail *mail); +static int index_mail_parse_body(struct index_mail *mail, + enum index_cache_field field); +static int index_mail_write_body_snippet(struct index_mail *mail); + +int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf, + unsigned int field_idx) +{ + struct mail *_mail = &mail->mail.mail; + int ret; + + ret = mail_cache_lookup_field(mail->mail.mail.transaction->cache_view, + buf, mail->mail.mail.seq, field_idx); + if (ret > 0) + mail->mail.mail.transaction->stats.cache_hit_count++; + + /* If the request was lazy mark the field as cache wanted. */ + if (_mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE_START_CACHING && + mail_cache_field_get_decision(_mail->box->cache, field_idx) == + MAIL_CACHE_DECISION_NO) { + mail_cache_decision_add(_mail->transaction->cache_view, + _mail->seq, field_idx); + } + + return ret; +} + +static void index_mail_try_set_attachment_keywords(struct index_mail *mail) +{ + if (mail->data.attachment_flags_updating) { + /* We can get here from mail_get_parts() */ + return; + } + mail->data.attachment_flags_updating = TRUE; + enum mail_lookup_abort orig_lookup_abort = mail->mail.mail.lookup_abort; + mail->mail.mail.lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + (void)mail_set_attachment_keywords(&mail->mail.mail); + mail->mail.mail.lookup_abort = orig_lookup_abort; + mail->data.attachment_flags_updating = FALSE; +} + +static bool +index_mail_want_attachment_keywords_on_fetch(struct index_mail *mail) +{ + const struct mail_storage_settings *mail_set = + mailbox_get_settings(mail->mail.mail.box); + + return mail_set->parsed_mail_attachment_detection_add_flags && + !mail_set->parsed_mail_attachment_detection_no_flags_on_fetch && + !mail_has_attachment_keywords(&mail->mail.mail); +} + +static int get_serialized_parts(struct index_mail *mail, buffer_t **part_buf_r) +{ + const unsigned int field_idx = + mail->ibox->cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx; + + *part_buf_r = t_buffer_create(128); + return index_mail_cache_lookup_field(mail, *part_buf_r, field_idx); +} + +static struct message_part *get_unserialized_parts(struct index_mail *mail) +{ + struct message_part *parts; + buffer_t *part_buf; + const char *error; + + if (get_serialized_parts(mail, &part_buf) <= 0) + return NULL; + + parts = message_part_deserialize(mail->mail.data_pool, part_buf->data, + part_buf->used, &error); + if (parts == NULL) { + mail_set_mail_cache_corrupted(&mail->mail.mail, + "Corrupted cached mime.parts data: %s (parts=%s)", + error, binary_to_hex(part_buf->data, part_buf->used)); + } + return parts; +} + +static bool message_parts_have_nuls(const struct message_part *part) +{ + for (; part != NULL; part = part->next) { + if ((part->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0) + return TRUE; + if (part->children != NULL) { + if (message_parts_have_nuls(part->children)) + return TRUE; + } + } + return FALSE; +} + +static bool get_cached_parts(struct index_mail *mail) +{ + struct message_part *part; + + if (mail->data.parts != NULL) + return TRUE; + if (mail->data.parser_ctx != NULL) { + /* Message is already being parsed. Get the message parts by + finishing its parsing, so there won't be any confusion about + whether e.g. data->parsed_bodystructure=TRUE match data->parts */ + return FALSE; + } + + T_BEGIN { + part = get_unserialized_parts(mail); + } T_END; + if (part == NULL) + return FALSE; + + /* we know the NULs now, update them */ + if (message_parts_have_nuls(part)) { + mail->mail.mail.has_nuls = TRUE; + mail->mail.mail.has_no_nuls = FALSE; + } else { + mail->mail.mail.has_nuls = FALSE; + mail->mail.mail.has_no_nuls = TRUE; + } + + mail->data.parts = part; + if (index_mail_want_attachment_keywords_on_fetch(mail)) + index_mail_try_set_attachment_keywords(mail); + return TRUE; +} + +void index_mail_set_message_parts_corrupted(struct mail *mail, const char *error) +{ + buffer_t *part_buf; + const char *parts_str; + + if (get_serialized_parts(INDEX_MAIL(mail), &part_buf) <= 0) + parts_str = ""; + else + parts_str = binary_to_hex(part_buf->data, part_buf->used); + + mail_set_cache_corrupted(mail, + MAIL_FETCH_MESSAGE_PARTS, t_strdup_printf( + "Cached MIME parts don't match message during parsing: %s (parts=%s)", + error, parts_str)); +} + +static bool index_mail_get_fixed_field(struct index_mail *mail, + enum index_cache_field field, + void *data, size_t data_size) +{ + const unsigned int field_idx = mail->ibox->cache_fields[field].idx; + buffer_t buf; + bool ret; + + buffer_create_from_data(&buf, data, data_size); + if (index_mail_cache_lookup_field(mail, &buf, field_idx) <= 0) + ret = FALSE; + else { + i_assert(buf.used == data_size); + ret = TRUE; + } + return ret; +} + +bool index_mail_get_cached_uoff_t(struct index_mail *mail, + enum index_cache_field field, uoff_t *size_r) +{ + return index_mail_get_fixed_field(mail, field, + size_r, sizeof(*size_r)); +} + +static bool index_mail_get_pvt(struct mail *_mail) +{ + struct mail_private *mail = (struct mail_private *)_mail; + + if (mail->seq_pvt != 0) + return TRUE; + if (_mail->box->view_pvt == NULL) { + /* no private view (set by view syncing) -> no private flags */ + return FALSE; + } + if (_mail->saving) { + /* mail is still being saved, it has no private flags yet */ + return FALSE; + } + i_assert(_mail->uid != 0); + + index_transaction_init_pvt(_mail->transaction); + if (!mail_index_lookup_seq(_mail->transaction->view_pvt, _mail->uid, + &mail->seq_pvt)) + mail->seq_pvt = 0; + return mail->seq_pvt != 0; +} + +enum mail_flags index_mail_get_flags(struct mail *_mail) +{ + struct mail_private *mail = (struct mail_private *)_mail; + const struct mail_index_record *rec; + enum mail_flags flags, pvt_flags_mask; + + rec = mail_index_lookup(_mail->transaction->view, _mail->seq); + flags = rec->flags & (MAIL_FLAGS_NONRECENT | + MAIL_INDEX_MAIL_FLAG_BACKEND); + + if (mailbox_recent_flags_have_uid(_mail->box, _mail->uid)) + flags |= MAIL_RECENT; + + if (index_mail_get_pvt(_mail)) { + /* mailbox has private flags */ + pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box); + flags &= ENUM_NEGATE(pvt_flags_mask); + rec = mail_index_lookup(_mail->transaction->view_pvt, + mail->seq_pvt); + flags |= rec->flags & pvt_flags_mask; + } + return flags; +} + +uint64_t index_mail_get_modseq(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (mail->data.modseq != 0) + return mail->data.modseq; + + mail_index_modseq_enable(_mail->box->index); + mail->data.modseq = + mail_index_modseq_lookup(_mail->transaction->view, _mail->seq); + return mail->data.modseq; +} + +uint64_t index_mail_get_pvt_modseq(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (mail->data.pvt_modseq != 0) + return mail->data.pvt_modseq; + + if (mailbox_open_index_pvt(_mail->box) <= 0) + return 0; + index_transaction_init_pvt(_mail->transaction); + + mail_index_modseq_enable(_mail->box->index_pvt); + mail->data.pvt_modseq = + mail_index_modseq_lookup(_mail->transaction->view_pvt, + _mail->seq); + return mail->data.pvt_modseq; +} + +const char *const *index_mail_get_keywords(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + const char *const *names; + const unsigned int *keyword_indexes; + unsigned int i, count, names_count; + + if (array_is_created(&data->keywords)) + return array_front(&data->keywords); + + (void)index_mail_get_keyword_indexes(_mail); + + keyword_indexes = array_get(&data->keyword_indexes, &count); + names = array_get(mail->ibox->keyword_names, &names_count); + p_array_init(&data->keywords, mail->mail.data_pool, count + 1); + for (i = 0; i < count; i++) { + const char *name; + i_assert(keyword_indexes[i] < names_count); + + name = names[keyword_indexes[i]]; + array_push_back(&data->keywords, &name); + } + + /* end with NULL */ + array_append_zero(&data->keywords); + return array_front(&data->keywords); +} + +const ARRAY_TYPE(keyword_indexes) * +index_mail_get_keyword_indexes(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + if (!array_is_created(&data->keyword_indexes)) { + p_array_init(&data->keyword_indexes, mail->mail.data_pool, 32); + mail_index_lookup_keywords(_mail->transaction->view, + mail->mail.mail.seq, + &data->keyword_indexes); + } + return &data->keyword_indexes; +} + +int index_mail_get_parts(struct mail *_mail, struct message_part **parts_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + data->cache_fetch_fields |= MAIL_FETCH_MESSAGE_PARTS; + if (data->parts != NULL || get_cached_parts(mail)) { + *parts_r = data->parts; + return 0; + } + + if (data->parser_ctx == NULL) { + const char *reason = + index_mail_cache_reason(_mail, "mime parts"); + if (index_mail_parse_headers(mail, NULL, reason) < 0) + return -1; + /* parts may be set now as a result of some plugin */ + } + + if (data->parts == NULL) { + data->save_message_parts = TRUE; + if (index_mail_parse_body(mail, 0) < 0) + return -1; + } + + *parts_r = data->parts; + return 0; +} + +int index_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + data->cache_fetch_fields |= MAIL_FETCH_RECEIVED_DATE; + if (data->received_date == (time_t)-1) { + uint32_t t; + + if (index_mail_get_fixed_field(mail, MAIL_CACHE_RECEIVED_DATE, + &t, sizeof(t))) + data->received_date = t; + } + + *date_r = data->received_date; + return *date_r == (time_t)-1 ? -1 : 0; +} + +int index_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + data->cache_fetch_fields |= MAIL_FETCH_SAVE_DATE; + if (data->save_date == (time_t)-1) { + uint32_t t; + + if (index_mail_get_fixed_field(mail, MAIL_CACHE_SAVE_DATE, + &t, sizeof(t))) + data->save_date = t; + } + + *date_r = data->save_date; + return *date_r == (time_t)-1 ? -1 : 1; +} + +static int index_mail_cache_sent_date(struct index_mail *mail) +{ + struct index_mail_data *data = &mail->data; + const char *str; + time_t t; + int ret, tz; + + if (data->sent_date.time != (uint32_t)-1) + return 0; + + if ((ret = mail_get_first_header(&mail->mail.mail, "Date", &str)) < 0) + return ret; + + if (ret == 0 || + !message_date_parse((const unsigned char *)str, + strlen(str), &t, &tz)) { + /* 0 = not found / invalid */ + t = 0; + tz = 0; + } + data->sent_date.time = t; + data->sent_date.timezone = tz; + index_mail_cache_add(mail, MAIL_CACHE_SENT_DATE, + &data->sent_date, sizeof(data->sent_date)); + return 0; +} + +int index_mail_get_date(struct mail *_mail, time_t *date_r, int *timezone_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct mail_sent_date sentdate; + + data->cache_fetch_fields |= MAIL_FETCH_DATE; + if (data->sent_date.time != (uint32_t)-1) { + *timezone_r = data->sent_date.timezone; + *date_r = data->sent_date.time; + return 0; + } + + if (index_mail_get_fixed_field(mail, MAIL_CACHE_SENT_DATE, + &sentdate, sizeof(sentdate))) + data->sent_date = sentdate; + + if (index_mail_cache_sent_date(mail) < 0) + return -1; + + *timezone_r = data->sent_date.timezone; + *date_r = data->sent_date.time; + return 0; +} + +static bool get_cached_msgpart_sizes(struct index_mail *mail) +{ + struct index_mail_data *data = &mail->data; + + if (data->parts == NULL) + (void)get_cached_parts(mail); + + if (data->parts != NULL) { + data->hdr_size_set = TRUE; + data->hdr_size = data->parts->header_size; + data->body_size = data->parts->body_size; + data->body_size_set = TRUE; + data->virtual_size = data->parts->header_size.virtual_size + + data->body_size.virtual_size; + data->physical_size = data->parts->header_size.physical_size + + data->body_size.physical_size; + } + + return data->parts != NULL; +} + +const uint32_t *index_mail_get_vsize_extension(struct mail *_mail) +{ + const void *idata; + bool expunged ATTR_UNUSED; + + mail_index_lookup_ext(_mail->transaction->view, _mail->seq, + _mail->box->mail_vsize_ext_id, &idata, &expunged); + const uint32_t *vsize = idata; + return vsize; +} + +static void index_mail_try_set_body_size(struct index_mail *mail) +{ + struct index_mail_data *data = &mail->data; + + if (data->hdr_size_set && !data->inexact_total_sizes && + data->physical_size != UOFF_T_MAX && + data->virtual_size != UOFF_T_MAX) { + /* We know the total size of this mail and we know the + header size, so we can calculate also the body size. + However, don't do this if there's a possibility that + physical_size or virtual_size don't actually match the + mail stream's size (e.g. buggy imapc servers). */ + if (data->physical_size < data->hdr_size.physical_size) { + mail_set_cache_corrupted(&mail->mail.mail, + MAIL_FETCH_PHYSICAL_SIZE, t_strdup_printf( + "Cached physical size smaller than header size " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", + data->physical_size, data->hdr_size.physical_size)); + } else if (data->virtual_size < data->hdr_size.virtual_size) { + mail_set_cache_corrupted(&mail->mail.mail, + MAIL_FETCH_VIRTUAL_SIZE, t_strdup_printf( + "Cached virtual size smaller than header size " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", + data->virtual_size, data->hdr_size.virtual_size)); + } else { + data->body_size.physical_size = data->physical_size - + data->hdr_size.physical_size; + data->body_size.virtual_size = data->virtual_size - + data->hdr_size.virtual_size; + data->body_size_set = TRUE; + } + } +} + +bool index_mail_get_cached_virtual_size(struct index_mail *mail, uoff_t *size_r) +{ + struct index_mail_data *data = &mail->data; + struct mail *_mail = &mail->mail.mail; + uoff_t size; + unsigned int idx ATTR_UNUSED; + + /* see if we can get it from index */ + const uint32_t *vsize = index_mail_get_vsize_extension(_mail); + + data->cache_fetch_fields |= MAIL_FETCH_VIRTUAL_SIZE; + if (data->virtual_size == UOFF_T_MAX && vsize != NULL && *vsize > 0) + data->virtual_size = (*vsize)-1; + if (data->virtual_size == UOFF_T_MAX) { + if (index_mail_get_cached_uoff_t(mail, + MAIL_CACHE_VIRTUAL_FULL_SIZE, + &size)) + data->virtual_size = size; + else { + if (!get_cached_msgpart_sizes(mail)) + return FALSE; + } + } + index_mail_try_set_body_size(mail); + *size_r = data->virtual_size; + + /* if vsize is present and wanted for index, but missing from index + add it to index. */ + if (vsize != NULL && *vsize == 0 && + data->virtual_size < (uint32_t)-1) { + uint32_t vsize = data->virtual_size+1; + mail_index_update_ext(_mail->transaction->itrans, _mail->seq, + _mail->box->mail_vsize_ext_id, &vsize, NULL); + } + + return TRUE; +} + +static void index_mail_get_cached_body_size(struct index_mail *mail) +{ + struct index_mail_data *data = &mail->data; + uoff_t tmp; + + if (!data->hdr_size_set) + return; + + /* we've already called get_cached_msgpart_sizes() and it didn't work. + try to do this by using cached virtual size and a quick physical + size lookup. */ + if (!index_mail_get_cached_virtual_size(mail, &tmp)) + return; + + if (!data->body_size_set) { + enum mail_lookup_abort old_abort = mail->mail.mail.lookup_abort; + + /* get the physical size, but not if it requires reading + through the whole message */ + if (mail->mail.mail.lookup_abort < MAIL_LOOKUP_ABORT_READ_MAIL) + mail->mail.mail.lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + if (mail_get_physical_size(&mail->mail.mail, &tmp) == 0) { + /* we should have everything now. try again. */ + (void)index_mail_get_cached_virtual_size(mail, &tmp); + } + mail->mail.mail.lookup_abort = old_abort; + } +} + +int index_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct message_size hdr_size, body_size; + struct istream *input; + uoff_t old_offset; + + if (index_mail_get_cached_virtual_size(mail, size_r)) + return 0; + + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream_because(_mail, &hdr_size, &body_size, + index_mail_cache_reason(_mail, "virtual size"), &input) < 0) + return -1; + i_stream_seek(data->stream, old_offset); + + i_assert(data->virtual_size != UOFF_T_MAX); + *size_r = data->virtual_size; + return 0; +} + +int index_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + uoff_t size; + + if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NOT_IN_CACHE && + _mail->lookup_abort != MAIL_LOOKUP_ABORT_READ_MAIL) { + /* If size.physical isn't in cache yet, add it. Do this only + when the caller appears to actually want it to be cached. + We don't want to cache the size when coming in here from + i_stream_mail_try_get_cached_size() or + index_mail_get_cached_body_size(). */ + data->cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE; + } + if (data->physical_size == UOFF_T_MAX) { + if (index_mail_get_cached_uoff_t(mail, + MAIL_CACHE_PHYSICAL_FULL_SIZE, + &size)) + data->physical_size = size; + else + (void)get_cached_msgpart_sizes(mail); + } + *size_r = data->physical_size; + return *size_r == UOFF_T_MAX ? -1 : 0; +} + +void index_mail_cache_add(struct index_mail *mail, enum index_cache_field field, + const void *data, size_t data_size) +{ + index_mail_cache_add_idx(mail, mail->ibox->cache_fields[field].idx, + data, data_size); +} + +void index_mail_cache_add_idx(struct index_mail *mail, unsigned int field_idx, + const void *data, size_t data_size) +{ + struct mail *_mail = &mail->mail.mail; + const struct mail_storage_settings *set = _mail->box->storage->set; + const struct mail_index_header *hdr; + + if (set->mail_cache_min_mail_count > 0) { + /* First check if we've configured caching not to be used with + low enough message count. */ + hdr = mail_index_get_header(_mail->transaction->view); + if (hdr->messages_count < set->mail_cache_min_mail_count) + return; + } + + if (!mail->data.no_caching && + mail->data.dont_cache_field_idx != field_idx && + !_mail->box->mail_cache_disabled) { + mail_cache_add(_mail->transaction->cache_trans, _mail->seq, + field_idx, data, data_size); + } +} + +void index_mail_cache_pop3_data(struct mail *_mail, + const char *uidl, uint32_t order) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (uidl != NULL) + index_mail_cache_add(mail, MAIL_CACHE_POP3_UIDL, + uidl, strlen(uidl)); + + if (order != 0) + index_mail_cache_add(mail, MAIL_CACHE_POP3_ORDER, + &order, sizeof(order)); +} + +static void parse_bodystructure_part_header(struct message_part *part, + struct message_header_line *hdr, + pool_t pool) +{ + message_part_data_parse_from_header(pool, part, hdr); +} + +static bool want_plain_bodystructure_cached(struct index_mail *mail) +{ + const unsigned int cache_field_body = + mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODY].idx; + const unsigned int cache_field_bodystructure = + mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx; + struct mail *_mail = &mail->mail.mail; + + if ((mail->data.wanted_fields & (MAIL_FETCH_IMAP_BODY | + MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0) + return TRUE; + + if (mail_cache_field_want_add(_mail->transaction->cache_trans, + _mail->seq, cache_field_body)) + return TRUE; + if (mail_cache_field_want_add(_mail->transaction->cache_trans, + _mail->seq, cache_field_bodystructure)) + return TRUE; + return FALSE; +} + +static void index_mail_body_parsed_cache_flags(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + struct index_mail_data *data = &mail->data; + unsigned int cache_flags_idx; + uint32_t cache_flags = data->cache_flags; + bool want_cached; + + cache_flags_idx = mail->ibox->cache_fields[MAIL_CACHE_FLAGS].idx; + want_cached = mail_cache_field_want_add(_mail->transaction->cache_trans, + _mail->seq, cache_flags_idx); + + if (data->parsed_bodystructure && + message_part_data_is_plain_7bit(data->parts) && + (want_cached || want_plain_bodystructure_cached(mail))) { + cache_flags |= MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII; + /* we need message_parts cached to be able to + actually use it in BODY/BODYSTRUCTURE reply */ + want_cached = TRUE; + data->save_message_parts = TRUE; + } + + /* cache flags should never get unset as long as the message doesn't + change, but try to handle it anyway */ + cache_flags &= ENUM_NEGATE(MAIL_CACHE_FLAG_BINARY_HEADER | + MAIL_CACHE_FLAG_BINARY_BODY | + MAIL_CACHE_FLAG_HAS_NULS | + MAIL_CACHE_FLAG_HAS_NO_NULS); + if (message_parts_have_nuls(data->parts)) { + _mail->has_nuls = TRUE; + _mail->has_no_nuls = FALSE; + cache_flags |= MAIL_CACHE_FLAG_HAS_NULS; + } else { + _mail->has_nuls = FALSE; + _mail->has_no_nuls = TRUE; + cache_flags |= MAIL_CACHE_FLAG_HAS_NO_NULS; + } + + if (data->hdr_size.virtual_size == data->hdr_size.physical_size) + cache_flags |= MAIL_CACHE_FLAG_BINARY_HEADER; + if (data->body_size.virtual_size == data->body_size.physical_size) + cache_flags |= MAIL_CACHE_FLAG_BINARY_BODY; + + if (cache_flags != data->cache_flags && want_cached) { + index_mail_cache_add_idx(mail, cache_flags_idx, + &cache_flags, sizeof(cache_flags)); + } + data->cache_flags = cache_flags; +} + +static void index_mail_body_parsed_cache_message_parts(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + struct index_mail_data *data = &mail->data; + const unsigned int cache_field = + mail->ibox->cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx; + enum mail_cache_decision_type decision; + buffer_t *buffer; + + if (data->messageparts_saved_to_cache || + mail_cache_field_exists(_mail->transaction->cache_view, _mail->seq, + cache_field) != 0) { + /* already cached */ + return; + } + + decision = mail_cache_field_get_decision(_mail->box->cache, + cache_field); + if (decision == (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED)) { + /* we never want it cached */ + return; + } + if (decision == MAIL_CACHE_DECISION_NO && + !data->save_message_parts && + (data->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) == 0) { + /* we didn't really care about the message parts themselves, + just wanted to use something that depended on it */ + return; + } + + T_BEGIN { + buffer = t_buffer_create(1024); + message_part_serialize(mail->data.parts, buffer); + index_mail_cache_add_idx(mail, cache_field, + buffer->data, buffer->used); + } T_END; + + data->messageparts_saved_to_cache = TRUE; +} + +static int +index_mail_write_bodystructure(struct index_mail *mail, string_t *str, + bool extended) +{ + const char *error; + + if (imap_bodystructure_write(mail->data.parts, str, extended, + &error) < 0) { + mail_set_cache_corrupted(&mail->mail.mail, + MAIL_FETCH_MESSAGE_PARTS, error); + return -1; + } + return 0; +} + +static void +index_mail_body_parsed_cache_bodystructure(struct index_mail *mail, + enum index_cache_field field) +{ + struct mail *_mail = &mail->mail.mail; + struct index_mail_data *data = &mail->data; + const unsigned int cache_field_parts = + mail->ibox->cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx; + const unsigned int cache_field_body = + mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODY].idx; + const unsigned int cache_field_bodystructure = + mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx; + enum mail_cache_decision_type dec; + string_t *str; + bool bodystructure_cached = FALSE; + bool plain_bodystructure = FALSE; + bool cache_bodystructure, cache_body; + + if ((data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) != 0) { + if (data->messageparts_saved_to_cache || + mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, cache_field_parts) > 0) { + /* cached it as flag + message_parts */ + plain_bodystructure = TRUE; + } + } + + if (!data->parsed_bodystructure) + return; + i_assert(data->parts != NULL); + + /* If BODY is fetched first but BODYSTRUCTURE is also wanted, we don't + normally want to first cache BODY and then BODYSTRUCTURE. So check + the wanted_fields also in here. */ + if (plain_bodystructure) + cache_bodystructure = FALSE; + else if (field == MAIL_CACHE_IMAP_BODYSTRUCTURE || + (data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0) { + cache_bodystructure = + mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, cache_field_bodystructure); + } else { + cache_bodystructure = + mail_cache_field_want_add(_mail->transaction->cache_trans, + _mail->seq, cache_field_bodystructure); + } + if (cache_bodystructure) { + str = str_new(mail->mail.data_pool, 128); + if (index_mail_write_bodystructure(mail, str, TRUE) == 0) { + data->bodystructure = str_c(str); + index_mail_cache_add(mail, MAIL_CACHE_IMAP_BODYSTRUCTURE, + str_c(str), str_len(str)); + bodystructure_cached = TRUE; + } + } else { + bodystructure_cached = + mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, cache_field_bodystructure) > 0; + } + + /* normally don't cache both BODY and BODYSTRUCTURE, but do it + if BODY is forced to be cached */ + dec = mail_cache_field_get_decision(_mail->box->cache, + cache_field_body); + if (plain_bodystructure || + (bodystructure_cached && + (dec != (MAIL_CACHE_DECISION_FORCED | MAIL_CACHE_DECISION_YES)))) + cache_body = FALSE; + else if (field == MAIL_CACHE_IMAP_BODY) { + cache_body = + mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, cache_field_body); + } else { + cache_body = + mail_cache_field_want_add(_mail->transaction->cache_trans, + _mail->seq, cache_field_body); + } + + if (cache_body) { + str = str_new(mail->mail.data_pool, 128); + if (index_mail_write_bodystructure(mail, str, FALSE) == 0) { + data->body = str_c(str); + index_mail_cache_add(mail, MAIL_CACHE_IMAP_BODY, + str_c(str), str_len(str)); + } + } +} + +bool index_mail_want_cache(struct index_mail *mail, enum index_cache_field field) +{ + struct mail *_mail = &mail->mail.mail; + enum mail_fetch_field fetch_field; + unsigned int cache_field; + + switch (field) { + case MAIL_CACHE_SENT_DATE: + fetch_field = MAIL_FETCH_DATE; + break; + case MAIL_CACHE_RECEIVED_DATE: + fetch_field = MAIL_FETCH_RECEIVED_DATE; + break; + case MAIL_CACHE_SAVE_DATE: + fetch_field = MAIL_FETCH_SAVE_DATE; + break; + case MAIL_CACHE_VIRTUAL_FULL_SIZE: + fetch_field = MAIL_FETCH_VIRTUAL_SIZE; + break; + case MAIL_CACHE_PHYSICAL_FULL_SIZE: + fetch_field = MAIL_FETCH_PHYSICAL_SIZE; + break; + case MAIL_CACHE_BODY_SNIPPET: + fetch_field = MAIL_FETCH_BODY_SNIPPET; + break; + default: + i_unreached(); + } + + if ((mail->data.dont_cache_fetch_fields & fetch_field) != 0) + return FALSE; + + /* If a field has been explicitly requested to be fetched, it's + included in data.cache_fetch_fields. In that case use _can_add() to + add it to the cache file if at all possible. Otherwise, use + _want_add() to use previous caching decisions. */ + cache_field = mail->ibox->cache_fields[field].idx; + if ((mail->data.cache_fetch_fields & fetch_field) != 0) { + return mail_cache_field_can_add(_mail->transaction->cache_trans, + _mail->seq, cache_field); + } else { + return mail_cache_field_want_add(_mail->transaction->cache_trans, + _mail->seq, cache_field); + } +} + +static void index_mail_save_finish_make_snippet(struct index_mail *mail) +{ + if (mail->data.save_body_snippet) { + if (index_mail_write_body_snippet(mail) < 0) + return; + mail->data.save_body_snippet = FALSE; + } + + if (mail->data.body_snippet != NULL && + index_mail_want_cache(mail, MAIL_CACHE_BODY_SNIPPET)) { + index_mail_cache_add(mail, MAIL_CACHE_BODY_SNIPPET, + mail->data.body_snippet, + strlen(mail->data.body_snippet)); + } +} + +static void index_mail_cache_sizes(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + struct mail_index_view *view = _mail->transaction->view; + + static enum index_cache_field size_fields[] = { + MAIL_CACHE_VIRTUAL_FULL_SIZE, + MAIL_CACHE_PHYSICAL_FULL_SIZE + }; + uoff_t sizes[N_ELEMENTS(size_fields)]; + unsigned int i; + uint32_t vsize; + uint32_t idx ATTR_UNUSED; + + sizes[0] = mail->data.virtual_size; + sizes[1] = mail->data.physical_size; + + /* store the virtual size in index if + extension for it exists or + extension for box virtual size exists and + size fits and is present and + size is not cached or + cached size differs + */ + if ((mail_index_map_get_ext_idx(view->index->map, _mail->box->mail_vsize_ext_id, &idx) || + mail_index_map_get_ext_idx(view->index->map, _mail->box->vsize_hdr_ext_id, &idx)) && + (sizes[0] != UOFF_T_MAX && + sizes[0] < (uint32_t)-1)) { + const uint32_t *vsize_ext = + index_mail_get_vsize_extension(_mail); + /* vsize = 0 means it's not present in index, consult cache. + we store vsize for every +4GB-1 mail to cache because + index can only hold 2^32-1 size. Cache will not be used + when vsize is stored in index. */ + vsize = sizes[0] + 1; + if (vsize_ext == NULL || vsize != *vsize_ext) { + mail_index_update_ext(_mail->transaction->itrans, _mail->seq, + _mail->box->mail_vsize_ext_id, &vsize, NULL); + } + /* it's already in index, so don't update cache */ + sizes[0] = UOFF_T_MAX; + } + + for (i = 0; i < N_ELEMENTS(size_fields); i++) { + if (sizes[i] != UOFF_T_MAX && + index_mail_want_cache(mail, size_fields[i])) { + index_mail_cache_add(mail, size_fields[i], + &sizes[i], sizeof(sizes[i])); + } + } +} + +static void index_mail_cache_dates(struct index_mail *mail) +{ + static enum index_cache_field date_fields[] = { + MAIL_CACHE_RECEIVED_DATE, + MAIL_CACHE_SAVE_DATE + }; + time_t dates[N_ELEMENTS(date_fields)]; + unsigned int i; + uint32_t t; + + dates[0] = mail->data.received_date; + dates[1] = mail->mail.mail.saving ? ioloop_time : + mail->data.save_date; + + for (i = 0; i < N_ELEMENTS(date_fields); i++) { + if (dates[i] != (time_t)-1 && + index_mail_want_cache(mail, date_fields[i])) { + t = dates[i]; + index_mail_cache_add(mail, date_fields[i], + &t, sizeof(t)); + } + } + + if (mail->data.sent_date_parsed && + index_mail_want_cache(mail, MAIL_CACHE_SENT_DATE)) + (void)index_mail_cache_sent_date(mail); +} + +static struct message_part * +index_mail_find_first_text_mime_part(struct message_part *parts) +{ + struct message_part_data *body_data = parts->data; + struct message_part *part; + + i_assert(body_data != NULL); + + if (body_data->content_type == NULL || + strcasecmp(body_data->content_type, "text") == 0) { + /* use any text/ part, even if we don't know what exactly + it is. */ + return parts; + } + if (strcasecmp(body_data->content_type, "multipart") != 0) { + /* for now we support only text Content-Types */ + return NULL; + } + + if (strcasecmp(body_data->content_subtype, "alternative") == 0) { + /* text/plain > text/html > text/ */ + struct message_part *html_part = NULL, *text_part = NULL; + + for (part = parts->children; part != NULL; part = part->next) { + struct message_part_data *sub_body_data = + part->data; + + i_assert(sub_body_data != NULL); + + if (sub_body_data->content_type == NULL || + strcasecmp(sub_body_data->content_type, "text") == 0) { + if (sub_body_data->content_subtype == NULL || + strcasecmp(sub_body_data->content_subtype, "plain") == 0) + return part; + if (strcasecmp(sub_body_data->content_subtype, "html") == 0) + html_part = part; + else + text_part = part; + } + } + return html_part != NULL ? html_part : text_part; + } + /* find the first usable MIME part */ + for (part = parts->children; part != NULL; part = part->next) { + struct message_part *subpart = + index_mail_find_first_text_mime_part(part); + if (subpart != NULL) + return subpart; + } + return NULL; +} + +static int index_mail_write_body_snippet(struct index_mail *mail) +{ + struct message_part *part; + struct istream *input; + uoff_t old_offset; + string_t *str; + int ret; + + i_assert(mail->data.parsed_bodystructure); + + part = index_mail_find_first_text_mime_part(mail->data.parts); + if (part == NULL) { + mail->data.body_snippet = BODY_SNIPPET_ALGO_V1; + return 0; + } + + old_offset = mail->data.stream == NULL ? 0 : mail->data.stream->v_offset; + const char *reason = index_mail_cache_reason(&mail->mail.mail, "snippet"); + if (mail_get_stream_because(&mail->mail.mail, NULL, NULL, reason, &input) < 0) + return -1; + i_assert(mail->data.stream != NULL); + + i_stream_seek(input, part->physical_pos); + input = i_stream_create_limit(input, part->header_size.physical_size + + part->body_size.physical_size); + + str = str_new(mail->mail.data_pool, 128); + str_append(str, BODY_SNIPPET_ALGO_V1); + ret = message_snippet_generate(input, BODY_SNIPPET_MAX_CHARS, str); + if (ret == 0) + mail->data.body_snippet = str_c(str); + i_stream_destroy(&input); + + i_stream_seek(mail->data.stream, old_offset); + return ret; +} + +void index_mail_parts_reset(struct index_mail *mail) +{ + mail->data.parts = NULL; + mail->data.parsed_bodystructure_header = FALSE; + mail->data.parsed_bodystructure = FALSE; +} + +static int +index_mail_parse_body_finish(struct index_mail *mail, + enum index_cache_field field, bool success) +{ + struct istream *parser_input = mail->data.parser_input; + const struct mail_storage_settings *mail_set = + mailbox_get_settings(mail->mail.mail.box); + const char *error = NULL; + int ret; + + if (parser_input == NULL) { + ret = message_parser_deinit_from_parts(&mail->data.parser_ctx, + &mail->data.parts, &error) < 0 ? 0 : 1; + } else { + mail->data.parser_input = NULL; + i_stream_ref(parser_input); + ret = message_parser_deinit_from_parts(&mail->data.parser_ctx, + &mail->data.parts, &error) < 0 ? 0 : 1; + if (success && (parser_input->stream_errno == 0 || + parser_input->stream_errno == EPIPE)) { + /* do one final read, which verifies that the message + size is correct. */ + if (i_stream_read(parser_input) != -1 || + i_stream_have_bytes_left(parser_input)) + i_unreached(); + } + /* EPIPE = input already closed. allow the caller to + decide if that is an error or not. (for example we + could be coming here from IMAP APPEND when IMAP + client has closed the connection too early. we + don't want to log an error in that case.) + Note that EPIPE may also come from istream-mail which + detects a corrupted message size. Either way, the + body wasn't successfully parsed. */ + if (parser_input->stream_errno == 0) + ; + else if (parser_input->stream_errno == EPIPE) + ret = -1; + else { + index_mail_stream_log_failure_for(mail, parser_input); + ret = -1; + } + i_stream_unref(&parser_input); + } + if (ret <= 0) { + if (ret == 0) { + i_assert(error != NULL); + index_mail_set_message_parts_corrupted(&mail->mail.mail, error); + } + index_mail_parts_reset(mail); + if (mail->data.save_bodystructure_body) + mail->data.save_bodystructure_header = TRUE; + if (mail->data.header_parser_initialized) + index_mail_parse_header_deinit(mail); + return -1; + } + if (mail->data.header_parser_initialized) { + i_assert(!success); + index_mail_parse_header_deinit(mail); + } + + if (mail->data.save_bodystructure_body) { + mail->data.parsed_bodystructure = TRUE; + mail->data.save_bodystructure_header = FALSE; + mail->data.save_bodystructure_body = FALSE; + i_assert(mail->data.parts != NULL); + } + + if (mail->data.no_caching) { + /* if we're here because we aborted parsing, don't get any + further or we may crash while generating output from + incomplete data */ + return 0; + } + + (void)get_cached_msgpart_sizes(mail); + + index_mail_body_parsed_cache_flags(mail); + index_mail_body_parsed_cache_message_parts(mail); + index_mail_body_parsed_cache_bodystructure(mail, field); + index_mail_cache_sizes(mail); + index_mail_cache_dates(mail); + if (mail_set->parsed_mail_attachment_detection_add_flags && + !mail_has_attachment_keywords(&mail->mail.mail)) + index_mail_try_set_attachment_keywords(mail); + return 0; +} + +static void index_mail_stream_log_failure(struct index_mail *mail) +{ + index_mail_stream_log_failure_for(mail, mail->data.stream); +} + +int index_mail_stream_check_failure(struct index_mail *mail) +{ + if (mail->data.stream->stream_errno == 0) + return 0; + index_mail_stream_log_failure(mail); + return -1; +} + +void index_mail_refresh_expunged(struct mail *mail) +{ + mail_index_refresh(mail->box->index); + if (mail_index_is_expunged(mail->transaction->view, mail->seq)) + mail_set_expunged(mail); +} + +void index_mail_stream_log_failure_for(struct index_mail *mail, + struct istream *input) +{ + struct mail *_mail = &mail->mail.mail; + + i_assert(input->stream_errno != 0); + + if (input->stream_errno == ENOENT) { + /* was the mail just expunged? we could get here especially if + external attachments are used and the attachment is deleted + before we've opened the file. */ + index_mail_refresh_expunged(_mail); + if (_mail->expunged) + return; + } + + const char *old_error = + mailbox_get_last_internal_error(_mail->box, NULL); + const char *new_error = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); + + if (mail->data.istream_error_logged && + strstr(old_error, new_error) != NULL) { + /* Avoid logging the same istream error multiple times + (even if the read reason is different). The old_error begins + with the UID=n prefix, which we can ignore since we know + that this mail already logged a critical error, so it has + to be about this same mail. */ + return; + } + mail->data.istream_error_logged = TRUE; + mail_set_critical(_mail, "%s (read reason=%s)", new_error, + mail->mail.get_stream_reason == NULL ? "" : + mail->mail.get_stream_reason); +} + +static int index_mail_parse_body(struct index_mail *mail, + enum index_cache_field field) +{ + struct index_mail_data *data = &mail->data; + uoff_t old_offset; + int ret; + + i_assert(data->parser_ctx != NULL); + + old_offset = data->stream->v_offset; + i_stream_seek(data->stream, data->hdr_size.physical_size); + + if (data->save_bodystructure_body) { + /* bodystructure header is parsed, we want the body's mime + headers too */ + i_assert(data->parsed_bodystructure_header); + message_parser_parse_body(data->parser_ctx, + parse_bodystructure_part_header, + mail->mail.data_pool); + } else { + message_parser_parse_body(data->parser_ctx, + *null_message_part_header_callback, NULL); + } + ret = index_mail_stream_check_failure(mail); + if (index_mail_parse_body_finish(mail, field, TRUE) < 0) + ret = -1; + + i_stream_seek(data->stream, old_offset); + return ret; +} + +static void index_mail_stream_destroy_callback(struct index_mail *mail) +{ + i_assert(mail->data.destroying_stream); + + mail->data.destroying_stream = FALSE; +} + +void index_mail_set_read_buffer_size(struct mail *_mail, struct istream *input) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + unsigned int block_size; + + i_stream_set_max_buffer_size(input, MAIL_READ_FULL_BLOCK_SIZE); + block_size = (mail->data.access_part & (READ_BODY | PARSE_BODY)) != 0 ? + MAIL_READ_FULL_BLOCK_SIZE : MAIL_READ_HDR_BLOCK_SIZE; + i_stream_set_init_buffer_size(input, block_size); +} + +int index_mail_init_stream(struct index_mail *mail, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) +{ + struct mail *_mail = &mail->mail.mail; + struct index_mail_data *data = &mail->data; + struct istream *input; + bool has_nuls, body_size_from_stream = FALSE; + int ret; + + i_assert(_mail->mail_stream_accessed); + + if (!data->initialized_wrapper_stream && + _mail->transaction->stats_track) { + input = i_stream_create_mail(_mail, data->stream, + !data->stream_has_only_header); + i_stream_unref(&data->stream); + data->stream = input; + data->initialized_wrapper_stream = TRUE; + } + + if (!data->destroy_callback_set) { + /* do this only once in case a plugin changes the stream. + otherwise the check would break. */ + data->destroy_callback_set = TRUE; + i_stream_add_destroy_callback(data->stream, + index_mail_stream_destroy_callback, mail); + } + + bool want_attachment_kw = + index_mail_want_attachment_keywords_on_fetch(mail); + if (want_attachment_kw) { + data->access_part |= PARSE_HDR | PARSE_BODY; + data->access_reason_code = "mail:attachment_keywords"; + } + + if (hdr_size != NULL || body_size != NULL) + (void)get_cached_msgpart_sizes(mail); + + bool want_body_parsing = want_attachment_kw || + (body_size != NULL && !data->body_size_set && + (data->access_part & PARSE_BODY) != 0); + + if (hdr_size != NULL || body_size != NULL || want_body_parsing) { + i_stream_seek(data->stream, 0); + if (!data->hdr_size_set || want_body_parsing) { + if ((data->access_part & (PARSE_HDR | PARSE_BODY)) != 0) { + (void)get_cached_parts(mail); + if (index_mail_parse_headers_internal(mail, NULL) < 0) + return -1; + } else { + if (message_get_header_size(data->stream, + &data->hdr_size, + &has_nuls) < 0) { + index_mail_stream_log_failure(mail); + return -1; + } + data->hdr_size_set = TRUE; + } + } + + if (hdr_size != NULL) + *hdr_size = data->hdr_size; + } + + if (body_size != NULL || want_body_parsing) { + if (!data->body_size_set && body_size != NULL) + index_mail_get_cached_body_size(mail); + if (!data->body_size_set || want_body_parsing) { + i_stream_seek(data->stream, + data->hdr_size.physical_size); + if ((data->access_part & PARSE_BODY) != 0) { + if (index_mail_parse_body(mail, 0) < 0) + return -1; + } else { + if (message_get_body_size(data->stream, + &data->body_size, + &has_nuls) < 0) { + index_mail_stream_log_failure(mail); + return -1; + } + data->body_size_set = TRUE; + } + body_size_from_stream = TRUE; + } + + if (body_size != NULL) + *body_size = data->body_size; + } + + if (data->hdr_size_set && data->body_size_set) { + data->virtual_size = data->hdr_size.virtual_size + + data->body_size.virtual_size; + data->physical_size = data->hdr_size.physical_size + + data->body_size.physical_size; + if (body_size_from_stream) { + /* the sizes were just calculated */ + data->inexact_total_sizes = FALSE; + } + } else { + /* If body_size==NULL, the caller doesn't care about it. + However, try to set it anyway if it can be calculated. */ + index_mail_try_set_body_size(mail); + } + ret = index_mail_stream_check_failure(mail); + + i_stream_seek(data->stream, 0); + if (ret < 0) + return -1; + *stream_r = data->stream; + return 0; +} + +static int +index_mail_parse_bodystructure_full(struct index_mail *mail, + enum index_cache_field field) +{ + struct index_mail_data *data = &mail->data; + + if ((data->save_bodystructure_header && + !data->parsed_bodystructure_header) || + !data->save_bodystructure_body || + field == MAIL_CACHE_BODY_SNIPPET) { + /* we haven't parsed the header yet */ + const char *reason = + index_mail_cache_reason(&mail->mail.mail, "bodystructure"); + bool orig_bodystructure_header = + data->save_bodystructure_header; + bool orig_bodystructure_body = + data->save_bodystructure_body; + data->save_bodystructure_header = TRUE; + data->save_bodystructure_body = TRUE; + (void)get_cached_parts(mail); + if (index_mail_parse_headers(mail, NULL, reason) < 0) { + data->save_bodystructure_header = + orig_bodystructure_header; + data->save_bodystructure_body = + orig_bodystructure_body; + return -1; + } + i_assert(data->parser_ctx != NULL); + } + + return index_mail_parse_body(mail, field); +} + +static int index_mail_parse_bodystructure(struct index_mail *mail, + enum index_cache_field field) +{ + struct index_mail_data *data = &mail->data; + string_t *str; + + if (data->parsed_bodystructure && field != MAIL_CACHE_BODY_SNIPPET) { + /* we have everything parsed already, but just not written to + a string */ + index_mail_body_parsed_cache_bodystructure(mail, field); + } else { + if (index_mail_parse_bodystructure_full(mail, field) < 0) + return -1; + if (data->parts == NULL) { + /* Corrupted mime.parts detected. Retry by parsing + the mail. */ + data->parsed_bodystructure = FALSE; + data->parsed_bodystructure_header = FALSE; + data->save_bodystructure_header = TRUE; + data->save_bodystructure_body = TRUE; + if (index_mail_parse_bodystructure_full(mail, field) < 0) + return -1; + } + } + i_assert(data->parts != NULL); + + /* if we didn't want to have the body(structure) cached, + it's still not written. */ + switch (field) { + case MAIL_CACHE_IMAP_BODY: + if (data->body == NULL) { + str = str_new(mail->mail.data_pool, 128); + if (index_mail_write_bodystructure(mail, str, FALSE) < 0) + return -1; + data->body = str_c(str); + } + break; + case MAIL_CACHE_IMAP_BODYSTRUCTURE: + if (data->bodystructure == NULL) { + str = str_new(mail->mail.data_pool, 128); + if (index_mail_write_bodystructure(mail, str, TRUE) < 0) + return -1; + data->bodystructure = str_c(str); + } + break; + case MAIL_CACHE_BODY_SNIPPET: + if (data->body_snippet == NULL) { + if (index_mail_write_body_snippet(mail) < 0) + return -1; + + if (index_mail_want_cache(mail, MAIL_CACHE_BODY_SNIPPET)) + index_mail_cache_add(mail, MAIL_CACHE_BODY_SNIPPET, + mail->data.body_snippet, + strlen(mail->data.body_snippet)); + } + i_assert(data->body_snippet != NULL && + data->body_snippet[0] != '\0'); + break; + default: + i_unreached(); + } + return 0; +} + +static void +index_mail_get_plain_bodystructure(struct index_mail *mail, string_t *str, + bool extended) +{ + str_printfa(str, IMAP_BODY_PLAIN_7BIT_ASCII" %"PRIuUOFF_T" %u", + mail->data.parts->body_size.virtual_size, + mail->data.parts->body_size.lines); + if (extended) + str_append(str, " NIL NIL NIL NIL"); +} + +static int +index_mail_fetch_body_snippet(struct index_mail *mail, const char **value_r) +{ + const struct mail_cache_field *cache_fields = mail->ibox->cache_fields; + const unsigned int cache_field = + cache_fields[MAIL_CACHE_BODY_SNIPPET].idx; + string_t *str; + + mail->data.cache_fetch_fields |= MAIL_FETCH_BODY_SNIPPET; + if (mail->data.body_snippet == NULL) { + str = str_new(mail->mail.data_pool, 128); + if (index_mail_cache_lookup_field(mail, str, cache_field) > 0 && + str_len(str) > 0) + mail->data.body_snippet = str_c(str); + } + if (mail->data.body_snippet != NULL) { + *value_r = mail->data.body_snippet; + return 0; + } + + /* reuse the IMAP bodystructure parsing code to get all the useful + headers that we need. */ + mail->data.save_body_snippet = TRUE; + if (index_mail_parse_bodystructure(mail, MAIL_CACHE_BODY_SNIPPET) < 0) + return -1; + i_assert(mail->data.body_snippet != NULL); + *value_r = mail->data.body_snippet; + return 0; +} + +bool index_mail_get_cached_body(struct index_mail *mail, const char **value_r) +{ + const struct mail_cache_field *cache_fields = mail->ibox->cache_fields; + const unsigned int body_cache_field = + cache_fields[MAIL_CACHE_IMAP_BODY].idx; + const unsigned int bodystructure_cache_field = + cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx; + struct index_mail_data *data = &mail->data; + string_t *str; + const char *error; + + if (data->body != NULL) { + *value_r = data->body; + return TRUE; + } + + str = str_new(mail->mail.data_pool, 128); + if ((data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) != 0 && + get_cached_parts(mail)) { + index_mail_get_plain_bodystructure(mail, str, FALSE); + *value_r = data->body = str_c(str); + return TRUE; + } + + /* 2) get BODY if it exists */ + if (index_mail_cache_lookup_field(mail, str, body_cache_field) > 0) { + *value_r = data->body = str_c(str); + return TRUE; + } + /* 3) get it using BODYSTRUCTURE if it exists */ + if (index_mail_cache_lookup_field(mail, str, bodystructure_cache_field) > 0) { + data->bodystructure = + p_strdup(mail->mail.data_pool, str_c(str)); + str_truncate(str, 0); + + if (imap_body_parse_from_bodystructure(data->bodystructure, + str, &error) < 0) { + /* broken, continue.. */ + mail_set_cache_corrupted(&mail->mail.mail, + MAIL_FETCH_IMAP_BODYSTRUCTURE, t_strdup_printf( + "Invalid BODYSTRUCTURE %s: %s", + data->bodystructure, error)); + } else { + *value_r = data->body = str_c(str); + return TRUE; + } + } + + str_free(&str); + return FALSE; +} + +bool index_mail_get_cached_bodystructure(struct index_mail *mail, + const char **value_r) +{ + const struct mail_cache_field *cache_fields = mail->ibox->cache_fields; + const unsigned int bodystructure_cache_field = + cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx; + struct index_mail_data *data = &mail->data; + string_t *str; + + if (data->bodystructure != NULL) { + *value_r = data->bodystructure; + return TRUE; + } + + str = str_new(mail->mail.data_pool, 128); + if ((data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) != 0 && + get_cached_parts(mail)) + index_mail_get_plain_bodystructure(mail, str, TRUE); + else if (index_mail_cache_lookup_field(mail, str, + bodystructure_cache_field) <= 0) { + str_free(&str); + return FALSE; + } + + *value_r = data->bodystructure = str_c(str); + if (index_mail_want_attachment_keywords_on_fetch(mail)) + index_mail_try_set_attachment_keywords(mail); + return TRUE; +} + +int index_mail_get_special(struct mail *_mail, + enum mail_fetch_field field, const char **value_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + switch (field) { + case MAIL_FETCH_IMAP_BODY: + if (index_mail_get_cached_body(mail, value_r)) + return 0; + + /* parse body structure, and save BODY/BODYSTRUCTURE + depending on what we want cached */ + if (index_mail_parse_bodystructure(mail, MAIL_CACHE_IMAP_BODY) < 0) + return -1; + i_assert(data->body != NULL); + *value_r = data->body; + return 0; + case MAIL_FETCH_IMAP_BODYSTRUCTURE: + if (index_mail_get_cached_bodystructure(mail, value_r)) + return 0; + + if (index_mail_parse_bodystructure(mail, MAIL_CACHE_IMAP_BODYSTRUCTURE) < 0) + return -1; + i_assert(data->bodystructure != NULL); + *value_r = data->bodystructure; + return 0; + case MAIL_FETCH_IMAP_ENVELOPE: + if (data->envelope == NULL) { + if (index_mail_headers_get_envelope(mail) < 0) + return -1; + } + *value_r = data->envelope; + return 0; + case MAIL_FETCH_FROM_ENVELOPE: + *value_r = data->from_envelope != NULL ? + data->from_envelope : ""; + return 0; + case MAIL_FETCH_BODY_SNIPPET: + return index_mail_fetch_body_snippet(mail, value_r); + case MAIL_FETCH_STORAGE_ID: + case MAIL_FETCH_UIDL_BACKEND: + case MAIL_FETCH_SEARCH_RELEVANCY: + case MAIL_FETCH_GUID: + case MAIL_FETCH_HEADER_MD5: + case MAIL_FETCH_POP3_ORDER: + case MAIL_FETCH_REFCOUNT: + case MAIL_FETCH_REFCOUNT_ID: + *value_r = ""; + return 0; + case MAIL_FETCH_MAILBOX_NAME: + *value_r = _mail->box->vname; + return 0; + default: + i_unreached(); + } +} + +int index_mail_get_backend_mail(struct mail *mail, + struct mail **real_mail_r) +{ + *real_mail_r = mail; + return 0; +} + +struct mail * +index_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct index_mail *mail; + pool_t pool; + + pool = pool_alloconly_create("mail", 2048); + mail = p_new(pool, struct index_mail, 1); + + index_mail_init(mail, t, wanted_fields, wanted_headers, pool, NULL); + return &mail->mail.mail; +} + +void index_mail_init(struct index_mail *mail, + struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers, + struct pool *mail_pool, + struct pool *data_pool) +{ + mail->mail.pool = mail_pool; + array_create(&mail->mail.module_contexts, mail->mail.pool, + sizeof(void *), 5); + + mail->mail.v = *t->box->mail_vfuncs; + mail->mail.mail.box = t->box; + mail->mail.mail.transaction = t; + t->mail_ref_count++; + if (data_pool != NULL) + mail->mail.data_pool = data_pool; + else + mail->mail.data_pool = pool_alloconly_create("index_mail", 16384); + mail->ibox = INDEX_STORAGE_CONTEXT(t->box); + mail->mail.wanted_fields = wanted_fields; + if (wanted_headers != NULL) { + mail->mail.wanted_headers = wanted_headers; + mailbox_header_lookup_ref(wanted_headers); + } + index_mail_init_data(mail); +} + +static void index_mail_close_streams_full(struct index_mail *mail, bool closing) +{ + struct index_mail_data *data = &mail->data; + struct message_part *parts; + const char *error; + + if (data->parser_ctx != NULL) { + if (message_parser_deinit_from_parts(&data->parser_ctx, &parts, &error) < 0) + index_mail_set_message_parts_corrupted(&mail->mail.mail, error); + mail->data.parser_input = NULL; + if (mail->data.save_bodystructure_body) + mail->data.save_bodystructure_header = TRUE; + } + i_stream_unref(&data->filter_stream); + if (data->stream != NULL) { + struct istream *orig_stream = data->stream; + + data->destroying_stream = TRUE; + if (!closing && data->destroy_callback_set) { + /* we're replacing the stream with a new one. it's + allowed to have references until the mail is closed + (but we can't really check that) */ + i_stream_remove_destroy_callback(data->stream, + index_mail_stream_destroy_callback); + } + i_stream_unref(&data->stream); + /* there must be no references to the mail when the + mail is being closed. */ + if (!closing) + data->destroying_stream = FALSE; + else if (mail->data.destroying_stream) { + i_panic("Input stream %s unexpectedly has references", + i_stream_get_name(orig_stream)); + } + + data->initialized_wrapper_stream = FALSE; + data->destroy_callback_set = FALSE; + } +} + +void index_mail_close_streams(struct index_mail *mail) +{ + index_mail_close_streams_full(mail, FALSE); +} + +static void index_mail_init_data(struct index_mail *mail) +{ + struct index_mail_data *data = &mail->data; + + data->virtual_size = UOFF_T_MAX; + data->physical_size = UOFF_T_MAX; + data->save_date = (time_t)-1; + data->received_date = (time_t)-1; + data->sent_date.time = (uint32_t)-1; + data->dont_cache_field_idx = UINT_MAX; + + data->wanted_fields = mail->mail.wanted_fields; + if (mail->mail.wanted_headers != NULL) { + data->wanted_headers = mail->mail.wanted_headers; + mailbox_header_lookup_ref(data->wanted_headers); + } +} + +static void index_mail_reset_data(struct index_mail *mail) +{ + i_zero(&mail->data); + p_clear(mail->mail.data_pool); + + index_mail_init_data(mail); + + mail->mail.mail.seq = 0; + mail->mail.mail.uid = 0; + mail->mail.seq_pvt = 0; + mail->mail.mail.expunged = FALSE; + mail->mail.mail.has_nuls = FALSE; + mail->mail.mail.has_no_nuls = FALSE; + mail->mail.mail.saving = FALSE; + mail->mail.mail.mail_stream_accessed = FALSE; + mail->mail.mail.mail_metadata_accessed = FALSE; +} + +void index_mail_close(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (mail->mail.mail.seq == 0) { + /* mail_set_seq*() hasn't been called yet, or is being called + right now. Don't reset anything yet. We especially don't + want to reset wanted_fields or wanted_headers so that + mail_add_temp_wanted_fields() can be called by plugins + before mail_set_seq_saving() for + mail_save_context.dest_mail. */ + return; + } + + /* make sure old mail isn't visible in the event anymore even if it's + attempted to be used. */ + event_unref(&mail->mail._event); + + /* If uid == 0 but seq != 0, we came here from saving a (non-mbox) + message. If that happens, don't bother checking if anything should + be cached since it was already checked. Also by now the transaction + may have already been rollbacked and seq point to a nonexistent + message. */ + if (mail->mail.mail.uid != 0) { + index_mail_cache_sizes(mail); + index_mail_cache_dates(mail); + } + + index_mail_close_streams_full(mail, TRUE); + /* Notify cache that the mail is no longer open. This mainly helps + with INDEX=MEMORY to keep all data added with mail_cache_add() in + memory until this point. */ + mail_cache_close_mail(_mail->transaction->cache_trans, _mail->seq); + + mailbox_header_lookup_unref(&mail->data.wanted_headers); + if (!mail->freeing) + index_mail_reset_data(mail); +} + +static void check_envelope(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + const unsigned int cache_field_envelope = + mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx; + unsigned int cache_field_hdr; + + if ((mail->data.access_part & PARSE_HDR) != 0) { + mail->data.save_envelope = TRUE; + return; + } + + /* if "imap.envelope" is cached, that's all we need */ + if (mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, cache_field_envelope) > 0) + return; + + /* don't waste time doing full checks for all required + headers. assume that if we have "hdr.message-id" cached, + we don't need to parse the header. */ + cache_field_hdr = mail_cache_register_lookup(_mail->box->cache, + "hdr.message-id"); + if (cache_field_hdr == UINT_MAX || + mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, cache_field_hdr) <= 0) { + mail->data.access_reason_code = "mail:imap_envelope"; + mail->data.access_part |= PARSE_HDR; + } + mail->data.save_envelope = TRUE; +} + +void index_mail_update_access_parts_pre(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct mail_storage *storage = _mail->box->storage; + const struct mail_cache_field *cache_fields = mail->ibox->cache_fields; + struct mail_cache_view *cache_view = _mail->transaction->cache_view; + const struct mail_storage_settings *mail_set = _mail->box->storage->set; + + if (_mail->seq == 0) { + /* mail_add_temp_wanted_fields() called before mail_set_seq*(). + We'll allow this, since it can be useful for plugins to + call it for mail_save_context.dest_mail. This function + is called again in mail_set_seq*(). */ + return; + } + + if ((data->wanted_fields & (MAIL_FETCH_NUL_STATE | + MAIL_FETCH_IMAP_BODY | + MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0 && + !_mail->has_nuls && !_mail->has_no_nuls) { + (void)index_mail_get_fixed_field(mail, MAIL_CACHE_FLAGS, + &data->cache_flags, + sizeof(data->cache_flags)); + _mail->has_nuls = + (data->cache_flags & MAIL_CACHE_FLAG_HAS_NULS) != 0; + _mail->has_no_nuls = + (data->cache_flags & MAIL_CACHE_FLAG_HAS_NO_NULS) != 0; + /* we currently don't forcibly set the nul state. if it's not + already cached, the caller can figure out itself what to + do when neither is set */ + } + + /* see if wanted_fields can tell us if we need to read/parse + header/body */ + if ((data->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) != 0 && + (storage->nonbody_access_fields & MAIL_FETCH_MESSAGE_PARTS) == 0 && + data->parts == NULL) { + const unsigned int cache_field = + cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx; + + if (mail_cache_field_exists(cache_view, _mail->seq, + cache_field) <= 0) { + data->access_reason_code = "mail:mime_parts"; + data->access_part |= PARSE_HDR | PARSE_BODY; + data->save_message_parts = TRUE; + } + } + + if ((data->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0 && + (storage->nonbody_access_fields & MAIL_FETCH_IMAP_ENVELOPE) == 0 && + data->envelope == NULL) + check_envelope(mail); + + if ((data->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0 && + (data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) == 0 && + (storage->nonbody_access_fields & MAIL_FETCH_IMAP_BODY) == 0 && + data->body == NULL) { + /* we need either imap.body or imap.bodystructure */ + const unsigned int cache_field1 = + cache_fields[MAIL_CACHE_IMAP_BODY].idx; + const unsigned int cache_field2 = + cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx; + + if (mail_cache_field_exists(cache_view, _mail->seq, + cache_field1) <= 0 && + mail_cache_field_exists(cache_view, _mail->seq, + cache_field2) <= 0) { + data->access_reason_code = "mail:imap_bodystructure"; + data->access_part |= PARSE_HDR | PARSE_BODY; + data->save_bodystructure_header = TRUE; + data->save_bodystructure_body = TRUE; + } + } + + if ((data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0 && + (data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) == 0 && + (storage->nonbody_access_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) == 0 && + data->bodystructure == NULL) { + const unsigned int cache_field = + cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx; + + if (mail_cache_field_exists(cache_view, _mail->seq, + cache_field) <= 0) { + data->access_reason_code = "mail:imap_bodystructure"; + data->access_part |= PARSE_HDR | PARSE_BODY; + data->save_bodystructure_header = TRUE; + data->save_bodystructure_body = TRUE; + } + } + + if ((data->wanted_fields & MAIL_FETCH_DATE) != 0 && + (storage->nonbody_access_fields & MAIL_FETCH_DATE) == 0 && + data->sent_date.time == (uint32_t)-1) { + const unsigned int cache_field = + cache_fields[MAIL_CACHE_SENT_DATE].idx; + + if (mail_cache_field_exists(cache_view, _mail->seq, + cache_field) <= 0) { + data->access_reason_code = "mail:date"; + data->access_part |= PARSE_HDR; + data->save_sent_date = TRUE; + } + } + if ((data->wanted_fields & MAIL_FETCH_BODY_SNIPPET) != 0 && + (storage->nonbody_access_fields & MAIL_FETCH_BODY_SNIPPET) == 0) { + const unsigned int cache_field = + cache_fields[MAIL_CACHE_BODY_SNIPPET].idx; + + if (mail_cache_field_exists(cache_view, _mail->seq, + cache_field) <= 0) { + data->access_reason_code = "mail:snippet"; + data->access_part |= PARSE_HDR | PARSE_BODY; + data->save_body_snippet = TRUE; + } + } + if ((data->wanted_fields & (MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY)) != 0) { + /* Clear reason_code if set. The mail is going to be read + in any case, so the previous reason for deciding to open + the mail won't matter. */ + data->access_reason_code = NULL; + if ((data->wanted_fields & MAIL_FETCH_STREAM_HEADER) != 0) + data->access_part |= READ_HDR; + if ((data->wanted_fields & MAIL_FETCH_STREAM_BODY) != 0) + data->access_part |= READ_BODY; + } + + /* NOTE: Keep this attachment detection the last, so that the + access_part check works correctly. + + The attachment flag detection is done while parsing BODYSTRUCTURE. + We want to do this for mails that are being saved, but also when + we need to open the mail body anyway. */ + if (mail_set->parsed_mail_attachment_detection_add_flags && + (_mail->saving || data->access_part != 0) && + !mail_has_attachment_keywords(&mail->mail.mail)) { + data->save_bodystructure_header = TRUE; + data->save_bodystructure_body = TRUE; + } +} + +void index_mail_update_access_parts_post(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + const struct mail_index_header *hdr; + struct istream *input; + + if (_mail->seq == 0) { + /* see index_mail_update_access_parts_pre() */ + return; + } + + /* when mail_prefetch_count>1, at this point we've started the + prefetching to all the mails and we're now starting to access the + first mail. */ + + if (data->access_part != 0) { + /* open stream immediately to set expunged flag if + it's already lost */ + + /* open the stream only if we didn't get here from + mailbox_save_init() */ + hdr = mail_index_get_header(_mail->transaction->view); + if (!_mail->saving && _mail->uid < hdr->next_uid) { + if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0) + (void)mail_get_stream_because(_mail, NULL, NULL, "access", &input); + else + (void)mail_get_hdr_stream(_mail, NULL, &input); + } + } +} + +void index_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + const struct mail_index_record *rec; + struct mail_index_map *map; + bool expunged; + + if (mail->mail.mail.seq == seq) { + if (!saving) + return; + /* we started saving a mail, aborted it, and now we're saving + another mail with the same sequence. make sure the mail + gets reset. */ + } + + mail->mail.v.close(&mail->mail.mail); + + mail->mail.mail.seq = seq; + mail->mail.mail.saving = saving; + + rec = mail_index_lookup_full(_mail->transaction->view, seq, + &map, &expunged); + mail->mail.mail.uid = rec->uid; + + /* Recreate the mail event when changing mails. Even though the same + mail struct is reused, they are practically different mails. The + event should have already been freed by close(). */ + i_assert(mail->mail._event == NULL); + + if (mail_index_view_is_inconsistent(_mail->transaction->view)) { + mail_set_expunged(&mail->mail.mail); + return; + } + /* Allow callers to easily find out if this mail was already expunged + by another session. It's possible that it could still be + successfully accessed. */ + if (expunged) + mail_set_expunged(&mail->mail.mail); + + if (!mail->mail.search_mail) { + index_mail_update_access_parts_pre(_mail); + index_mail_update_access_parts_post(_mail); + } else { + /* searching code will call the + index_mail_update_access_parts_*() after we know the mail is + actually wanted to be fetched. */ + } + mail->data.initialized = TRUE; +} + +bool index_mail_prefetch(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); +/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */ +#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + struct mail_storage *storage = _mail->box->storage; + struct istream *input; + off_t len; + int fd; + + if ((storage->class_flags & MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG) == 0) { + /* we're handling only file-per-msg storages for now. */ + return TRUE; + } + if (mail->data.access_part == 0) { + /* everything we need is cached */ + return TRUE; + } + + if (mail->data.stream == NULL) { + (void)mail_get_stream_because(_mail, NULL, NULL, "prefetch", &input); + if (mail->data.stream == NULL) + return TRUE; + } + + /* tell OS to start reading the file into memory */ + fd = i_stream_get_fd(mail->data.stream); + if (fd != -1) { + if ((mail->data.access_part & (READ_BODY | PARSE_BODY)) != 0) + len = 0; + else + len = MAIL_READ_HDR_BLOCK_SIZE; + if (posix_fadvise(fd, 0, len, POSIX_FADV_WILLNEED) < 0) { + i_error("posix_fadvise(%s) failed: %m", + i_stream_get_name(mail->data.stream)); + } + mail->data.prefetch_sent = TRUE; + } +#endif + return !mail->data.prefetch_sent; +} + +bool index_mail_set_uid(struct mail *_mail, uint32_t uid) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + uint32_t seq; + + if (mail_index_lookup_seq(_mail->transaction->view, uid, &seq)) { + index_mail_set_seq(_mail, seq, FALSE); + return TRUE; + } else { + mail->mail.v.close(&mail->mail.mail); + mail->mail.mail.uid = uid; + mail_set_expunged(&mail->mail.mail); + return FALSE; + } +} + +void index_mail_add_temp_wanted_fields(struct mail *_mail, + enum mail_fetch_field fields, + struct mailbox_header_lookup_ctx *headers) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct mailbox_header_lookup_ctx *new_wanted_headers; + + data->wanted_fields |= fields; + if (headers == NULL) { + /* keep old ones */ + } else if (data->wanted_headers == NULL) { + data->wanted_headers = headers; + mailbox_header_lookup_ref(headers); + } else { + /* merge headers */ + new_wanted_headers = mailbox_header_lookup_merge(data->wanted_headers, + headers); + mailbox_header_lookup_unref(&data->wanted_headers); + data->wanted_headers = new_wanted_headers; + } + index_mail_update_access_parts_pre(_mail); + /* Don't call _post(), which would try to open the stream. It should be + enough to delay the opening until it happens anyway. + + Otherwise there's not really any good place to call this in the + plugins: set_seq() call get_stream() internally, which can already + start parsing the headers, so it's too late. If we use get_stream() + and there's a _post() call here, it gets into infinite loop. The + loop could probably be prevented in some way, but it's probably + better to eventually try to remove the _post() call entirely + everywhere. */ +} + +void index_mail_set_uid_cache_updates(struct mail *_mail, bool set) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + mail->data.no_caching = set || mail->data.forced_no_caching; +} + +void index_mail_free(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + mail->freeing = TRUE; + mail->mail.v.close(_mail); + + i_assert(_mail->transaction->mail_ref_count > 0); + _mail->transaction->mail_ref_count--; + + buffer_free(&mail->header_data); + if (array_is_created(&mail->header_lines)) + array_free(&mail->header_lines); + if (array_is_created(&mail->header_match)) + array_free(&mail->header_match); + if (array_is_created(&mail->header_match_lines)) + array_free(&mail->header_match_lines); + + mailbox_header_lookup_unref(&mail->data.wanted_headers); + mailbox_header_lookup_unref(&mail->mail.wanted_headers); + event_unref(&mail->mail._event); + pool_unref(&mail->mail.data_pool); + pool_unref(&mail->mail.pool); +} + +void index_mail_cache_parse_continue(struct mail *_mail) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct message_block block; + + while (message_parser_parse_next_block(mail->data.parser_ctx, + &block) > 0) { + if (block.size != 0) + continue; + + if (!mail->data.header_parsed) { + index_mail_parse_header(block.part, block.hdr, mail); + if (block.hdr == NULL) + mail->data.header_parsed = TRUE; + } else { + message_part_data_parse_from_header(mail->mail.data_pool, + block.part, block.hdr); + } + } +} + +void index_mail_cache_parse_deinit(struct mail *_mail, time_t received_date, + bool success) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (!success) { + /* we're going to delete this mail anyway, + don't bother trying to update cache file */ + mail->data.no_caching = TRUE; + mail->data.forced_no_caching = TRUE; + + if (mail->data.parser_ctx == NULL) { + /* we didn't even start cache parsing */ + i_assert(!mail->data.header_parser_initialized); + return; + } + } + + /* This is needed with 0 byte mails to get hdr=NULL call done. */ + index_mail_cache_parse_continue(_mail); + + if (mail->data.received_date == (time_t)-1) + mail->data.received_date = received_date; + if (mail->data.save_date == (time_t)-1) { + /* this save_date may not be exactly the same as what we get + in future, but then again neither mbox nor maildir + guarantees it anyway. */ + mail->data.save_date = ioloop_time; + } + + (void)index_mail_parse_body_finish(mail, 0, success); +} + +static bool +index_mail_update_pvt_flags(struct mail *_mail, enum modify_type modify_type, + enum mail_flags pvt_flags) +{ + struct mail_private *mail = (struct mail_private *)_mail; + const struct mail_index_record *rec; + enum mail_flags old_pvt_flags; + + if (!index_mail_get_pvt(_mail)) + return FALSE; + if (pvt_flags == 0 && modify_type != MODIFY_REPLACE) + return FALSE; + + /* see if the flags actually change anything */ + rec = mail_index_lookup(_mail->transaction->view_pvt, mail->seq_pvt); + old_pvt_flags = rec->flags & mailbox_get_private_flags_mask(_mail->box); + + switch (modify_type) { + case MODIFY_ADD: + return (old_pvt_flags & pvt_flags) != pvt_flags; + case MODIFY_REPLACE: + return old_pvt_flags != pvt_flags; + case MODIFY_REMOVE: + return (old_pvt_flags & pvt_flags) != 0; + } + i_unreached(); +} + +void index_mail_update_flags(struct mail *_mail, enum modify_type modify_type, + enum mail_flags flags) +{ + struct mail_private *mail = (struct mail_private *)_mail; + enum mail_flags pvt_flags_mask, pvt_flags = 0; + bool update_modseq = FALSE; + + flags &= MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND; + + if (_mail->box->view_pvt != NULL) { + /* mailbox has private flags */ + pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box); + pvt_flags = flags & pvt_flags_mask; + flags &= ENUM_NEGATE(pvt_flags_mask); + if (index_mail_update_pvt_flags(_mail, modify_type, pvt_flags)) { + mail_index_update_flags(_mail->transaction->itrans_pvt, + mail->seq_pvt, + modify_type, pvt_flags); + update_modseq = TRUE; + } + } + + if (!update_modseq) { + /* no forced modseq update */ + } else if (modify_type == MODIFY_REMOVE) { + /* add the modseq update separately */ + mail_index_update_flags(_mail->transaction->itrans, _mail->seq, + MODIFY_ADD, (enum mail_flags )MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ); + } else { + /* add as part of the flag updates */ + flags |= MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ; + } + mail_index_update_flags(_mail->transaction->itrans, _mail->seq, + modify_type, flags); +} + +void index_mail_update_keywords(struct mail *mail, enum modify_type modify_type, + struct mail_keywords *keywords) +{ + struct index_mail *imail = INDEX_MAIL(mail); + + if (array_is_created(&imail->data.keyword_indexes)) + array_free(&imail->data.keyword_indexes); + if (array_is_created(&imail->data.keywords)) { + /* clear the keywords array so the next mail_get_keywords() + returns the updated keywords. don't free the array, because + then any existing mail_get_keywords() return values would + point to broken data. this won't leak memory because the + array is allocated from mail's memory pool. */ + memset(&imail->data.keywords, 0, + sizeof(imail->data.keywords)); + } + + mail_index_update_keywords(mail->transaction->itrans, mail->seq, + modify_type, keywords); +} + +void index_mail_update_modseq(struct mail *mail, uint64_t min_modseq) +{ + mail_index_update_modseq(mail->transaction->itrans, mail->seq, + min_modseq); +} + +void index_mail_update_pvt_modseq(struct mail *mail, uint64_t min_pvt_modseq) +{ + if (mail->box->view_pvt == NULL) + return; + index_transaction_init_pvt(mail->transaction); + mail_index_update_modseq(mail->transaction->itrans_pvt, mail->seq, + min_pvt_modseq); +} + +void index_mail_expunge(struct mail *mail) +{ + enum mail_lookup_abort old_abort = mail->lookup_abort; + const char *value; + guid_128_t guid_128; + + mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + if (mail_get_special(mail, MAIL_FETCH_GUID, &value) < 0) + mail_index_expunge(mail->transaction->itrans, mail->seq); + else { + mail_generate_guid_128_hash(value, guid_128); + mail_index_expunge_guid(mail->transaction->itrans, + mail->seq, guid_128); + } + mail->lookup_abort = old_abort; +} + +static void index_mail_parse(struct mail *mail, bool parse_body) +{ + struct index_mail *imail = INDEX_MAIL(mail); + + imail->data.access_part |= PARSE_HDR; + if (index_mail_parse_headers(imail, NULL, "precache") == 0) { + if (parse_body) { + imail->data.access_part |= PARSE_BODY; + (void)index_mail_parse_body(imail, 0); + } + } +} + +int index_mail_precache(struct mail *mail) +{ + struct index_mail *imail = INDEX_MAIL(mail); + enum mail_fetch_field cache; + time_t date; + uoff_t size; + const char *str; + + if (mail_cache_field_exists_any(mail->transaction->cache_view, + mail->seq)) { + /* already cached this mail (we should get here only if FTS + plugin decreased the first precached seq) */ + return 0; + } + + cache = imail->data.wanted_fields; + if ((cache & (MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY)) != 0) + index_mail_parse(mail, (cache & MAIL_FETCH_STREAM_BODY) != 0); + if ((cache & MAIL_FETCH_RECEIVED_DATE) != 0) + (void)mail_get_received_date(mail, &date); + if ((cache & MAIL_FETCH_SAVE_DATE) != 0) + (void)mail_get_save_date(mail, &date); + if ((cache & MAIL_FETCH_VIRTUAL_SIZE) != 0) + (void)mail_get_virtual_size(mail, &size); + if ((cache & MAIL_FETCH_PHYSICAL_SIZE) != 0) + (void)mail_get_physical_size(mail, &size); + if ((cache & MAIL_FETCH_UIDL_BACKEND) != 0) + (void)mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &str); + if ((cache & MAIL_FETCH_POP3_ORDER) != 0) + (void)mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str); + if ((cache & MAIL_FETCH_GUID) != 0) + (void)mail_get_special(mail, MAIL_FETCH_GUID, &str); + return 0; +} + +static void +index_mail_reset_vsize_ext(struct mail *mail) +{ + unsigned int idx; + uint32_t vsize = 0; + struct mail_index_view *view = mail->transaction->view; + if (mail_index_map_get_ext_idx(view->map, mail->box->mail_vsize_ext_id, + &idx)) { + mail_index_update_ext(mail->transaction->itrans, mail->seq, + mail->box->mail_vsize_ext_id, &vsize, NULL); + } +} + +void index_mail_set_cache_corrupted(struct mail *mail, + enum mail_fetch_field field, + const char *reason) +{ + struct index_mail *imail = INDEX_MAIL(mail); + const char *field_name; + + switch ((int)field) { + case 0: + field_name = "fields"; + break; + case MAIL_FETCH_PHYSICAL_SIZE: + field_name = "physical size"; + imail->data.physical_size = UOFF_T_MAX; + imail->data.virtual_size = UOFF_T_MAX; + index_mail_parts_reset(imail); + index_mail_reset_vsize_ext(mail); + break; + case MAIL_FETCH_VIRTUAL_SIZE: + field_name = "virtual size"; + imail->data.physical_size = UOFF_T_MAX; + imail->data.virtual_size = UOFF_T_MAX; + index_mail_parts_reset(imail); + index_mail_reset_vsize_ext(mail); + break; + case MAIL_FETCH_MESSAGE_PARTS: + field_name = "MIME parts"; + index_mail_parts_reset(imail); + break; + case MAIL_FETCH_IMAP_BODY: + field_name = "IMAP BODY"; + imail->data.body = NULL; + imail->data.bodystructure = NULL; + break; + case MAIL_FETCH_IMAP_BODYSTRUCTURE: + field_name = "IMAP BODYSTRUCTURE"; + imail->data.body = NULL; + imail->data.bodystructure = NULL; + break; + default: + field_name = t_strdup_printf("#%x", field); + } + + /* make sure we don't cache invalid values */ + mail_cache_transaction_reset(mail->transaction->cache_trans); + imail->data.no_caching = TRUE; + imail->data.forced_no_caching = TRUE; + + if (mail->saving) { + mail_set_critical(mail, + "BUG: Broken %s found while saving a new mail: %s", + field_name, reason); + } else if (reason[0] == '\0') { + mail_set_mail_cache_corrupted(mail, + "Broken %s in mailbox %s", + field_name, mail->box->vname); + } else { + mail_set_mail_cache_corrupted(mail, + "Broken %s in mailbox %s: %s", + field_name, mail->box->vname, reason); + } +} + +int index_mail_opened(struct mail *mail, + struct istream **stream ATTR_UNUSED) +{ + struct index_mail *imail = + container_of(mail, struct index_mail, mail.mail); + struct event_reason *reason = NULL; + + if (imail->data.access_reason_code != NULL) + reason = event_reason_begin(imail->data.access_reason_code); + mail_opened_event(mail); + event_reason_end(&reason); + return 0; +} + +void index_mail_save_finish(struct mail_save_context *ctx) +{ + struct index_mail *imail = INDEX_MAIL(ctx->dest_mail); + + index_mail_save_finish_make_snippet(imail); + + if (ctx->data.from_envelope != NULL && + imail->data.from_envelope == NULL) { + imail->data.from_envelope = + p_strdup(imail->mail.data_pool, ctx->data.from_envelope); + } +} + +const char *index_mail_cache_reason(struct mail *mail, const char *reason) +{ + const char *cache_reason = + mail_cache_get_missing_reason(mail->transaction->cache_view, mail->seq); + return t_strdup_printf("%s (%s)", reason, cache_reason); +} diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h new file mode 100644 index 0000000..deb2831 --- /dev/null +++ b/src/lib-storage/index/index-mail.h @@ -0,0 +1,293 @@ +#ifndef INDEX_MAIL_H +#define INDEX_MAIL_H + +#include "message-size.h" +#include "mail-cache.h" +#include "mail-storage-private.h" + +enum index_cache_field { + /* fixed size fields */ + MAIL_CACHE_FLAGS = 0, + MAIL_CACHE_SENT_DATE, + MAIL_CACHE_RECEIVED_DATE, + MAIL_CACHE_SAVE_DATE, + MAIL_CACHE_VIRTUAL_FULL_SIZE, + MAIL_CACHE_PHYSICAL_FULL_SIZE, + + /* variable sized field */ + MAIL_CACHE_IMAP_BODY, + MAIL_CACHE_IMAP_BODYSTRUCTURE, + MAIL_CACHE_IMAP_ENVELOPE, + MAIL_CACHE_POP3_UIDL, + MAIL_CACHE_POP3_ORDER, + MAIL_CACHE_GUID, + MAIL_CACHE_MESSAGE_PARTS, + MAIL_CACHE_BINARY_PARTS, + MAIL_CACHE_BODY_SNIPPET, + + MAIL_INDEX_CACHE_FIELD_COUNT +}; +extern struct mail_cache_field + global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT]; + +#define IMAP_BODY_PLAIN_7BIT_ASCII \ + "\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\"" + +enum mail_cache_record_flag { + /* If binary flags are set, it's not checked whether mail is + missing CRs. So this flag may be set as an optimization for + regular non-binary mails as well if it's known that it contains + valid CR+LF line breaks. */ + MAIL_CACHE_FLAG_BINARY_HEADER = 0x0001, + MAIL_CACHE_FLAG_BINARY_BODY = 0x0002, + + /* Mail header or body is known to contain NUL characters. */ + MAIL_CACHE_FLAG_HAS_NULS = 0x0004, + /* Mail header or body is known to not contain NUL characters. */ + MAIL_CACHE_FLAG_HAS_NO_NULS = 0x0020, + /* obsolete _HAS_NO_NULS flag, which was being set incorrectly */ + MAIL_CACHE_FLAG_HAS_NO_NULS_BROKEN = 0x0008, + + /* BODY is IMAP_BODY_PLAIN_7BIT_ASCII and rest of BODYSTRUCTURE + fields are NIL */ + MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII = 0x0010 +}; + +enum index_mail_access_part { + READ_HDR = 0x01, + READ_BODY = 0x02, + PARSE_HDR = 0x04, + PARSE_BODY = 0x08 +}; + +struct mail_sent_date { + uint32_t time; + int32_t timezone; +}; + +struct index_mail_line { + unsigned int field_idx; + uint32_t start_pos, end_pos; + uint32_t line_num; +}; + +struct message_header_line; + +struct index_mail_data { + time_t date, received_date, save_date; + uoff_t virtual_size, physical_size; + + struct mail_sent_date sent_date; + struct index_mail_line parse_line; + uint32_t parse_line_num; + + struct message_part *parts; + struct message_binary_part *bin_parts; + const char *envelope, *body, *bodystructure, *guid, *filename; + const char *from_envelope, *body_snippet; + struct message_part_envelope *envelope_data; + + uint32_t cache_flags; + uint64_t modseq, pvt_modseq; + enum index_mail_access_part access_part; + const char *access_reason_code; + /* dont_cache_fields overrides cache_fields */ + enum mail_fetch_field cache_fetch_fields, dont_cache_fetch_fields; + unsigned int dont_cache_field_idx; + enum mail_fetch_field wanted_fields; + struct mailbox_header_lookup_ctx *wanted_headers; + + buffer_t *search_results; + + struct istream *stream, *filter_stream; + struct tee_istream *tee_stream; + struct message_size hdr_size, body_size; + struct istream *parser_input; + struct message_parser_ctx *parser_ctx; + int parsing_count; + ARRAY_TYPE(keywords) keywords; + ARRAY_TYPE(keyword_indexes) keyword_indexes; + + bool initialized:1; + bool save_sent_date:1; + bool sent_date_parsed:1; + bool save_envelope:1; + bool save_bodystructure_header:1; + bool save_bodystructure_body:1; + bool save_message_parts:1; + bool save_body_snippet:1; + bool stream_has_only_header:1; + bool parsed_bodystructure:1; + bool parsed_bodystructure_header:1; + bool hdr_size_set:1; + bool body_size_set:1; + bool messageparts_saved_to_cache:1; + bool header_parsed:1; + bool no_caching:1; + bool forced_no_caching:1; + bool istream_error_logged:1; + bool destroying_stream:1; + bool initialized_wrapper_stream:1; + bool destroy_callback_set:1; + bool prefetch_sent:1; + bool header_parser_initialized:1; + bool attachment_flags_updating:1; + /* virtual_size and physical_size may not match the stream size. + Try to avoid trusting them too much. */ + bool inexact_total_sizes:1; +}; + +struct index_mail { + struct mail_private mail; + struct index_mail_data data; + struct index_mailbox_context *ibox; + + int pop3_state; + + /* per-mail variables, here for performance reasons: */ + uint32_t header_seq; + string_t *header_data; + ARRAY(struct index_mail_line) header_lines; +#define HEADER_MATCH_FLAG_FOUND 1 +#define HEADER_MATCH_SKIP_COUNT 2 +#define HEADER_MATCH_USABLE(mail, num) \ + ((num & ~1U) == (mail)->header_match_value) + ARRAY(uint8_t) header_match; + ARRAY(unsigned int) header_match_lines; + uint8_t header_match_value; + + bool pop3_state_set:1; + /* close() is being called from mail_free() */ + bool freeing:1; +}; + +#define INDEX_MAIL(s) container_of(s, struct index_mail, mail.mail) + +struct mail * +index_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); +void index_mail_init(struct index_mail *mail, + struct mailbox_transaction_context *_t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *_wanted_headers, + struct pool *mail_pool, + struct pool *data_pool); + +void index_mail_set_seq(struct mail *mail, uint32_t seq, bool saving); +bool index_mail_set_uid(struct mail *mail, uint32_t uid); +void index_mail_set_uid_cache_updates(struct mail *mail, bool set); +bool index_mail_prefetch(struct mail *mail); +void index_mail_add_temp_wanted_fields(struct mail *mail, + enum mail_fetch_field fields, + struct mailbox_header_lookup_ctx *headers); +void index_mail_update_access_parts_pre(struct mail *mail); +void index_mail_update_access_parts_post(struct mail *_mail); +void index_mail_close(struct mail *mail); +void index_mail_close_streams(struct index_mail *mail); +void index_mail_free(struct mail *mail); +void index_mail_set_message_parts_corrupted(struct mail *mail, const char *error); + +bool index_mail_want_parse_headers(struct index_mail *mail); +void index_mail_parse_header_init(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers) + ATTR_NULL(2); +void index_mail_parse_header(struct message_part *part, + struct message_header_line *hdr, + struct index_mail *mail) ATTR_NULL(1); +int index_mail_parse_headers(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers, + const char *reason) + ATTR_NULL(2); +void index_mail_parse_header_deinit(struct index_mail *mail); +/* Same as index_mail_parse_headers(), but assume that the stream is + already opened. */ +int index_mail_parse_headers_internal(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers) + ATTR_NULL(2); +int index_mail_headers_get_envelope(struct index_mail *mail); +void index_mail_parts_reset(struct index_mail *mail); + +int index_mail_get_first_header(struct mail *_mail, const char *field, + bool decode_to_utf8, const char **value_r); +int index_mail_get_headers(struct mail *_mail, const char *field, + bool decode_to_utf8, const char *const **value_r); +int index_mail_get_header_stream(struct mail *_mail, + struct mailbox_header_lookup_ctx *headers, + struct istream **stream_r); +void index_mail_set_read_buffer_size(struct mail *mail, struct istream *input); + +enum mail_flags index_mail_get_flags(struct mail *_mail); +uint64_t index_mail_get_modseq(struct mail *_mail); +uint64_t index_mail_get_pvt_modseq(struct mail *_mail); +const char *const *index_mail_get_keywords(struct mail *_mail); +const ARRAY_TYPE(keyword_indexes) * +index_mail_get_keyword_indexes(struct mail *_mail); +int index_mail_get_parts(struct mail *_mail, struct message_part **parts_r); +int index_mail_get_received_date(struct mail *_mail, time_t *date_r); +int index_mail_get_save_date(struct mail *_mail, time_t *date_r); +int index_mail_get_date(struct mail *_mail, time_t *date_r, int *timezone_r); +int index_mail_get_virtual_size(struct mail *mail, uoff_t *size_r); +int index_mail_get_physical_size(struct mail *mail, uoff_t *size_r); +int index_mail_init_stream(struct index_mail *mail, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) ATTR_NULL(2, 3); +int index_mail_get_binary_stream(struct mail *_mail, + const struct message_part *part, + bool include_hdr, uoff_t *size_r, + unsigned int *body_lines_r, bool *binary_r, + struct istream **stream_r); +int index_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r); +int index_mail_get_backend_mail(struct mail *mail, struct mail **real_mail_r); + +void index_mail_update_flags(struct mail *mail, enum modify_type modify_type, + enum mail_flags flags); +void index_mail_update_keywords(struct mail *mail, enum modify_type modify_type, + struct mail_keywords *keywords); +void index_mail_update_modseq(struct mail *mail, uint64_t min_modseq); +void index_mail_update_pvt_modseq(struct mail *mail, uint64_t min_pvt_modseq); +void index_mail_expunge(struct mail *mail); +int index_mail_precache(struct mail *mail); +void index_mail_set_cache_corrupted(struct mail *mail, + enum mail_fetch_field field, + const char *reason); +int index_mail_opened(struct mail *mail, struct istream **stream); +int index_mail_stream_check_failure(struct index_mail *mail); +void index_mail_stream_log_failure_for(struct index_mail *mail, + struct istream *input); +void index_mail_refresh_expunged(struct mail *mail); +struct index_mail *index_mail_get_index_mail(struct mail *mail); + +bool index_mail_get_cached_uoff_t(struct index_mail *mail, + enum index_cache_field field, uoff_t *size_r); +bool index_mail_get_cached_virtual_size(struct index_mail *mail, + uoff_t *size_r); +bool index_mail_get_cached_body(struct index_mail *mail, const char **value_r); +bool index_mail_get_cached_bodystructure(struct index_mail *mail, + const char **value_r); +const uint32_t *index_mail_get_vsize_extension(struct mail *_mail); + +bool index_mail_want_cache(struct index_mail *mail, enum index_cache_field field); +void index_mail_cache_add(struct index_mail *mail, enum index_cache_field field, + const void *data, size_t data_size); +void index_mail_cache_add_idx(struct index_mail *mail, unsigned int field_idx, + const void *data, size_t data_size); + +void index_mail_cache_pop3_data(struct mail *_mail, + const char *uidl, uint32_t order); + +struct istream *index_mail_cache_parse_init(struct mail *mail, + struct istream *input); +void index_mail_cache_parse_continue(struct mail *mail); +void index_mail_cache_parse_deinit(struct mail *mail, time_t received_date, + bool success); + +int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf, + unsigned int field_idx); +void index_mail_save_finish(struct mail_save_context *ctx); + +const char *index_mail_cache_reason(struct mail *mail, const char *reason); + +#endif diff --git a/src/lib-storage/index/index-mailbox-size.c b/src/lib-storage/index/index-mailbox-size.c new file mode 100644 index 0000000..35b1081 --- /dev/null +++ b/src/lib-storage/index/index-mailbox-size.c @@ -0,0 +1,502 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strescape.h" +#include "net.h" +#include "write-full.h" +#include "mail-search-build.h" +#include "index-storage.h" +#include "index-mailbox-size.h" + +/* + Saving new mails: After transaction is committed and synced, trigger + vsize updating. Lock vsize updates. Check if the message count + + last-indexed-uid are still valid. If they are, add all the missing new + mails. Unlock. + + Fetching vsize: Lock vsize updates. Check if the message count + + last-indexed-uid are still valid. If not, set them to zero. Add all + the missing mails. Unlock. + + Expunging mails: Check if syncing would expunge any mails. If so, lock the + vsize updates before locking syncing (to avoid deadlocks). Check if the + message count + last-indexed-uid are still valid. If not, unlock vsize and + do nothing else. Otherwise, for each expunged mail whose UID <= + last-indexed-uid, decrease the message count and the vsize in memory. After + syncing is successfully committed, write the changes to header. Unlock. + + Note that the final expunge handling with some mailbox formats is done while + syncing is no longer locked. Because of this we need to have the vsize + locking. The final vsize header update requires committing a transaction, + which internally is the same as a sync lock. So to avoid deadlocks we always + need to lock vsize updates before sync. +*/ + +#define VSIZE_LOCK_SUFFIX "dovecot-vsize.lock" +#define VSIZE_UPDATE_MAX_LOCK_SECS 10 + +#define INDEXER_SOCKET_NAME "indexer" +#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n" + +struct mailbox_vsize_update { + struct mailbox *box; + struct mail_index_view *view; + struct mailbox_index_vsize vsize_hdr, orig_vsize_hdr; + + struct file_lock *lock; + bool lock_failed; + bool skip_write; + bool rebuild; + bool written; + bool finish_in_background; +}; + +static void vsize_header_refresh(struct mailbox_vsize_update *update) +{ + const void *data; + size_t size; + + if (update->view != NULL) + mail_index_view_close(&update->view); + (void)mail_index_refresh(update->box->index); + update->view = mail_index_view_open(update->box->index); + + mail_index_get_header_ext(update->view, update->box->vsize_hdr_ext_id, + &data, &size); + if (size > 0) { + memcpy(&update->orig_vsize_hdr, data, + I_MIN(size, sizeof(update->orig_vsize_hdr))); + } + if (size == sizeof(update->vsize_hdr)) + memcpy(&update->vsize_hdr, data, sizeof(update->vsize_hdr)); + else { + if (size != 0) { + mailbox_set_critical(update->box, + "vsize-hdr has invalid size: %zu", + size); + } + update->rebuild = TRUE; + i_zero(&update->vsize_hdr); + } +} + +static void +index_mailbox_vsize_check_rebuild(struct mailbox_vsize_update *update) +{ + uint32_t seq1, seq2; + + if (update->vsize_hdr.highest_uid == 0) + return; + if (!mail_index_lookup_seq_range(update->view, 1, + update->vsize_hdr.highest_uid, + &seq1, &seq2)) + seq2 = 0; + + if (update->vsize_hdr.message_count != seq2) { + if (update->vsize_hdr.message_count < seq2) { + mailbox_set_critical(update->box, + "vsize-hdr has invalid message-count (%u < %u)", + update->vsize_hdr.message_count, seq2); + } else { + /* some messages have been expunged, rescan */ + } + i_zero(&update->vsize_hdr); + update->rebuild = TRUE; + } +} + +struct mailbox_vsize_update * +index_mailbox_vsize_update_init(struct mailbox *box) +{ + struct mailbox_vsize_update *update; + + i_assert(box->opened); + + update = i_new(struct mailbox_vsize_update, 1); + update->box = box; + + vsize_header_refresh(update); + return update; +} + +static bool vsize_update_lock_full(struct mailbox_vsize_update *update, + unsigned int lock_secs) +{ + struct mailbox *box = update->box; + const char *error; + int ret; + + if (update->lock != NULL) + return TRUE; + if (update->lock_failed) + return FALSE; + if (MAIL_INDEX_IS_IN_MEMORY(box->index)) + return FALSE; + + ret = mailbox_lock_file_create(box, VSIZE_LOCK_SUFFIX, lock_secs, + &update->lock, &error); + if (ret <= 0) { + /* don't log lock timeouts, because we're somewhat expecting + them. Especially when lock_secs is 0. */ + if (ret < 0) + mailbox_set_critical(box, "%s", error); + update->lock_failed = TRUE; + return FALSE; + } + update->rebuild = FALSE; + vsize_header_refresh(update); + index_mailbox_vsize_check_rebuild(update); + return TRUE; +} + +bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update) +{ + return vsize_update_lock_full(update, 0); +} + +bool index_mailbox_vsize_update_wait_lock(struct mailbox_vsize_update *update) +{ + return vsize_update_lock_full(update, VSIZE_UPDATE_MAX_LOCK_SECS); +} + +bool index_mailbox_vsize_want_updates(struct mailbox_vsize_update *update) +{ + return update->vsize_hdr.highest_uid > 0; +} + +static void +index_mailbox_vsize_update_write_to_index(struct mailbox_vsize_update *update) +{ + struct mail_index_transaction *trans; + + trans = mail_index_transaction_begin(update->view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_header_ext(trans, update->box->vsize_hdr_ext_id, + 0, &update->vsize_hdr, + sizeof(update->vsize_hdr)); + (void)mail_index_transaction_commit(&trans); +} + +static void +index_mailbox_vsize_update_write(struct mailbox_vsize_update *update) +{ + if (update->written) + return; + update->written = TRUE; + + if (update->rebuild == FALSE && + memcmp(&update->orig_vsize_hdr, &update->vsize_hdr, + sizeof(update->vsize_hdr)) == 0) { + /* no changes */ + return; + } + index_mailbox_vsize_update_write_to_index(update); +} + +static void index_mailbox_vsize_notify_indexer(struct mailbox *box) +{ + string_t *str = t_str_new(256); + const char *path; + int fd; + + path = t_strconcat(box->storage->user->set->base_dir, + "/"INDEXER_SOCKET_NAME, NULL); + fd = net_connect_unix(path); + if (fd == -1) { + mailbox_set_critical(box, + "Can't start vsize building on background: " + "net_connect_unix(%s) failed: %m", path); + return; + } + str_append(str, INDEXER_HANDSHAKE); + str_append(str, "APPEND\t0\t"); + str_append_tabescaped(str, box->storage->user->username); + str_append_c(str, '\t'); + str_append_tabescaped(str, box->vname); + str_append_c(str, '\n'); + + if (write_full(fd, str_data(str), str_len(str)) < 0) { + mailbox_set_critical(box, + "Can't start vsize building on background: " + "write(%s) failed: %m", path); + } + i_close_fd(&fd); +} + +void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update) +{ + struct mailbox_vsize_update *update = *_update; + + *_update = NULL; + + if ((update->lock != NULL || update->rebuild) && !update->skip_write) + index_mailbox_vsize_update_write(update); + file_lock_free(&update->lock); + if (update->finish_in_background) + index_mailbox_vsize_notify_indexer(update->box); + + mail_index_view_close(&update->view); + i_free(update); +} + +void index_mailbox_vsize_hdr_expunge(struct mailbox_vsize_update *update, + uint32_t uid, uoff_t vsize) +{ + i_assert(update->lock != NULL); + + if (uid > update->vsize_hdr.highest_uid) + return; + if (update->vsize_hdr.message_count == 0) { + mailbox_set_critical(update->box, + "vsize-hdr's message_count shrank below 0"); + i_zero(&update->vsize_hdr); + return; + } + update->vsize_hdr.message_count--; + if (update->vsize_hdr.vsize < vsize) { + mailbox_set_critical(update->box, + "vsize-hdr's vsize shrank below 0"); + i_zero(&update->vsize_hdr); + return; + } + update->vsize_hdr.vsize -= vsize; +} + +static void +index_mailbox_vsize_finish_bg(struct mailbox_vsize_update *update, + bool require_result) +{ + mail_storage_set_error(update->box->storage, MAIL_ERROR_INUSE, + "Finishing vsize calculation on background"); + if (require_result) + update->finish_in_background = TRUE; +} + +static int +index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update, + bool require_result) +{ + struct mailbox_index_vsize *vsize_hdr = &update->vsize_hdr; + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mailbox_status status; + struct mail *mail; + unsigned int idx, mails_left; + uint32_t seq1, seq2; + uoff_t vsize; + int ret = 0; + + mailbox_get_open_status(update->box, STATUS_UIDNEXT, &status); + if (vsize_hdr->highest_uid + 1 >= status.uidnext) { + /* nothing to do - we should have usually caught this already + before locking */ + return 0; + } + + /* note that update->view may be more up-to-date than box->view. + we'll just add whatever new mails are in box->view. if we'll notice + that some of the new mails are missing, we'll need to stop there + since that expunge will be applied later on to the vsize header. */ + search_args = mail_search_build_init(); + if (!mail_index_lookup_seq_range(update->box->view, + vsize_hdr->highest_uid + 1, + status.uidnext-1, &seq1, &seq2)) { + /* nothing existed, but update uidnext */ + vsize_hdr->highest_uid = status.uidnext - 1; + mail_search_args_unref(&search_args); + return 0; + } + mail_search_build_add_seqset(search_args, seq1, seq2); + + if (!mail_index_map_get_ext_idx(update->box->view->map, + update->box->vsize_hdr_ext_id, &idx)) { + /* vsize header doesn't exist yet. Create it here early so + that vsize mail records get created (instead of adding + size.virtuals to cache). */ + index_mailbox_vsize_update_write_to_index(update); + } + + trans = mailbox_transaction_begin(update->box, 0, "vsize update"); + search_ctx = mailbox_search_init(trans, search_args, NULL, + MAIL_FETCH_VIRTUAL_SIZE, NULL); + if (!require_result) + mails_left = 0; + else if (update->box->storage->set->mail_vsize_bg_after_count == 0) + mails_left = UINT_MAX; + else + mails_left = update->box->storage->set->mail_vsize_bg_after_count; + + while (mailbox_search_next(search_ctx, &mail)) { + if (mails_left == 0) { + if (mail->mail_stream_accessed) { + /* Seems stream is opened by mailbox search, so we + will stop here, and finish it on background. */ + index_mailbox_vsize_finish_bg(update, + require_result); + ret = -1; + break; + } + /* if there are any more mails whose vsize can't be + looked up from cache, abort and finish on + background. */ + mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + } + ret = mail_get_virtual_size(mail, &vsize); + mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER; + + if (ret < 0 && + mailbox_get_last_mail_error(update->box) == MAIL_ERROR_LOOKUP_ABORTED) { + /* abort and finish on background */ + i_assert(mails_left == 0); + index_mailbox_vsize_finish_bg(update, require_result); + break; + } + if (mail->mail_stream_accessed || + mail->mail_metadata_accessed) { + /* slow vsize lookup */ + i_assert(mails_left > 0); + mails_left--; + } + + if (ret < 0) { + if (mail->expunged) + continue; + ret = -1; + break; + } + vsize_hdr->vsize += vsize; + vsize_hdr->highest_uid = mail->uid; + vsize_hdr->message_count++; + } + if (mailbox_search_deinit(&search_ctx) < 0) + ret = -1; + mail_search_args_unref(&search_args); + + if (ret == 0) { + /* success, cache all */ + vsize_hdr->highest_uid = status.uidnext - 1; + } else { + /* search failed, cache only up to highest seen uid */ + } + (void)mailbox_transaction_commit(&trans); + return ret; +} + +int index_mailbox_get_virtual_size(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + struct mailbox_vsize_update *update; + struct mailbox_status status; + int ret; + + mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, &status); + update = index_mailbox_vsize_update_init(box); + if (update->vsize_hdr.highest_uid + 1 == status.uidnext && + update->vsize_hdr.message_count == status.messages) { + /* up to date */ + metadata_r->virtual_size = update->vsize_hdr.vsize; + index_mailbox_vsize_update_deinit(&update); + return 0; + } + + /* we need to update it - lock it if possible. if not, update it + anyway internally even though we won't be saving the result. */ + (void)index_mailbox_vsize_update_wait_lock(update); + + struct event_reason *reason = event_reason_begin("mailbox:vsize"); + ret = index_mailbox_vsize_hdr_add_missing(update, TRUE); + event_reason_end(&reason); + metadata_r->virtual_size = update->vsize_hdr.vsize; + index_mailbox_vsize_update_deinit(&update); + return ret; +} + +int index_mailbox_get_physical_size(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + struct mailbox_transaction_context *trans; + struct mail_search_context *ctx; + struct mail *mail; + struct mail_search_args *search_args; + uoff_t size; + int ret = 0; + + /* if physical size = virtual size always for the storage, we can + use the optimized vsize code for this */ + if (box->mail_vfuncs->get_physical_size == + box->mail_vfuncs->get_virtual_size) { + if (index_mailbox_get_virtual_size(box, metadata_r) < 0) + return -1; + metadata_r->physical_size = metadata_r->virtual_size; + return 0; + } + /* do it the slow way (we could implement similar logic as for vsize, + but for now it's not really needed) */ + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) + return -1; + + trans = mailbox_transaction_begin(box, 0, "mailbox physical size"); + + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + ctx = mailbox_search_init(trans, search_args, NULL, + MAIL_FETCH_PHYSICAL_SIZE, NULL); + mail_search_args_unref(&search_args); + + metadata_r->physical_size = 0; + while (mailbox_search_next(ctx, &mail)) { + if (mail_get_physical_size(mail, &size) == 0) + metadata_r->physical_size += size; + else { + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_internal_error(box, &error); + if (error != MAIL_ERROR_EXPUNGED) { + i_error("Couldn't get size of mail UID %u in %s: %s", + mail->uid, box->vname, errstr); + ret = -1; + break; + } + } + } + if (mailbox_search_deinit(&ctx) < 0) { + i_error("Listing mails in %s failed: %s", + box->vname, mailbox_get_last_internal_error(box, NULL)); + ret = -1; + } + (void)mailbox_transaction_commit(&trans); + return ret; +} + +void index_mailbox_vsize_update_appends(struct mailbox *box) +{ + struct mailbox_vsize_update *update; + struct mailbox_status status; + + update = index_mailbox_vsize_update_init(box); + if (update->rebuild) { + /* The vsize header doesn't exist. Don't create it. */ + update->skip_write = TRUE; + } + + /* update here only if we don't need to rebuild the whole vsize. */ + index_mailbox_vsize_check_rebuild(update); + if (index_mailbox_vsize_want_updates(update)) { + /* Get the UIDNEXT only after checking that vsize updating is + even potentially wanted for this mailbox. We especially + don't want to do this with imapc, because it could trigger + a remote STATUS (UIDNEXT) call. */ + mailbox_get_open_status(update->box, STATUS_UIDNEXT, &status); + if (update->vsize_hdr.highest_uid + 1 != status.uidnext && + index_mailbox_vsize_update_try_lock(update)) { + struct event_reason *reason = + event_reason_begin("mailbox:vsize"); + (void)index_mailbox_vsize_hdr_add_missing(update, FALSE); + event_reason_end(&reason); + } + } + index_mailbox_vsize_update_deinit(&update); +} diff --git a/src/lib-storage/index/index-mailbox-size.h b/src/lib-storage/index/index-mailbox-size.h new file mode 100644 index 0000000..002da22 --- /dev/null +++ b/src/lib-storage/index/index-mailbox-size.h @@ -0,0 +1,20 @@ +#ifndef INDEX_MAILBOX_SIZE_H +#define INDEX_MAILBOX_SIZE_H + +struct mailbox; + +struct mailbox_vsize_update * +index_mailbox_vsize_update_init(struct mailbox *box); +void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **update); + +void index_mailbox_vsize_hdr_expunge(struct mailbox_vsize_update *update, + uint32_t uid, uoff_t vsize); + +bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update); +bool index_mailbox_vsize_update_wait_lock(struct mailbox_vsize_update *update); +/* Returns TRUE if expunges & appends should be updating the header. */ +bool index_mailbox_vsize_want_updates(struct mailbox_vsize_update *update); + +void index_mailbox_vsize_update_appends(struct mailbox *box); + +#endif diff --git a/src/lib-storage/index/index-pop3-uidl.c b/src/lib-storage/index/index-pop3-uidl.c new file mode 100644 index 0000000..e537e9f --- /dev/null +++ b/src/lib-storage/index/index-pop3-uidl.c @@ -0,0 +1,104 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "index-storage.h" +#include "index-mail.h" +#include "index-pop3-uidl.h" + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid) +{ + struct mailbox_index_pop3_uidl uidl; + + i_zero(&uidl); + uidl.max_uid_with_pop3_uidl = uid; + + mail_index_update_header_ext(trans, box->pop3_uidl_hdr_ext_id, + 0, &uidl, sizeof(uidl)); +} + +bool index_pop3_uidl_can_exist(struct mail *mail) +{ + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + + /* We'll assume that if the header exists, it's up-to-date. normally + UIDLs are set only during migration, so this value never changes. + Also even if it does, it becomes out-of-date only when the mailbox + is modified with old Dovecot versions. To fix that we'd have to + add and keep updating "max tracked uid" in this header for every + saved mail, which isn't worth it. */ + mail_index_get_header_ext(mail->transaction->view, + mail->box->pop3_uidl_hdr_ext_id, + &data, &size); + if (size < sizeof(uidl)) { + /* this header isn't set yet */ + return TRUE; + } + memcpy(&uidl, data, sizeof(uidl)); + return mail->uid <= uidl.max_uid_with_pop3_uidl; +} + +void index_pop3_uidl_update_exists(struct mail *mail, bool exists) +{ + struct mailbox_transaction_context *trans = mail->transaction; + + if (exists) { + if (trans->highest_pop3_uidl_uid < mail->uid) { + trans->highest_pop3_uidl_uid = mail->uid; + trans->prev_pop3_uidl_tracking_seq = mail->seq; + } + } else if (mail->seq == trans->prev_pop3_uidl_tracking_seq+1) { + trans->prev_pop3_uidl_tracking_seq++; + } else { + /* skipping mails. we don't know the state. */ + } +} + +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans) +{ + struct mail_index_view *view; + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + bool seen_all_msgs; + + mail_index_get_header_ext(trans->view, trans->box->pop3_uidl_hdr_ext_id, + &data, &size); + + if (trans->highest_pop3_uidl_uid == 0 && size >= sizeof(uidl)) { + /* header already set and nothing to change */ + return; + } + + /* First check that we actually looked at UIDL for all messages. + Otherwise we can't say for sure if the newest messages had UIDLs. */ + if (trans->prev_pop3_uidl_tracking_seq != + mail_index_view_get_messages_count(trans->view)) + return; + + /* Just to be sure: Refresh the index and check again. POP3 keeps + transactions open for duration of the entire session. Maybe another + process already added new mails (and already updated this header). + This check is racy, but normally UIDLs aren't added after migration + so it's a bit questionable if it's even worth having this check in + there. */ + view = mail_index_view_open(trans->box->index); + seen_all_msgs = mail_index_refresh(trans->box->index) == 0 && + trans->prev_pop3_uidl_tracking_seq == + mail_index_view_get_messages_count(view); + mail_index_view_close(&view); + if (!seen_all_msgs) + return; + + /* check if we have already the same header */ + if (size >= sizeof(uidl)) { + memcpy(&uidl, data, sizeof(uidl)); + if (trans->highest_pop3_uidl_uid == uidl.max_uid_with_pop3_uidl) + return; + } + index_pop3_uidl_set_max_uid(trans->box, trans->itrans, + trans->highest_pop3_uidl_uid); +} diff --git a/src/lib-storage/index/index-pop3-uidl.h b/src/lib-storage/index/index-pop3-uidl.h new file mode 100644 index 0000000..956ab7d --- /dev/null +++ b/src/lib-storage/index/index-pop3-uidl.h @@ -0,0 +1,16 @@ +#ifndef INDEX_POP3_H +#define INDEX_POP3_H + +struct mail_index_transaction; +struct mail; +struct mailbox; +struct mailbox_transaction_context; + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid); +bool index_pop3_uidl_can_exist(struct mail *mail); +void index_pop3_uidl_update_exists(struct mail *mail, bool exists); +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans); + +#endif diff --git a/src/lib-storage/index/index-rebuild.c b/src/lib-storage/index/index-rebuild.c new file mode 100644 index 0000000..0eaaab5 --- /dev/null +++ b/src/lib-storage/index/index-rebuild.c @@ -0,0 +1,257 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-cache.h" +#include "mail-index-modseq.h" +#include "mailbox-list-private.h" +#include "mailbox-recent-flags.h" +#include "index-storage.h" +#include "index-rebuild.h" + +static void +index_index_copy_vsize(struct index_rebuild_context *ctx, + struct mail_index_view *view, + uint32_t old_seq, uint32_t new_seq) +{ + const void *data; + bool expunged; + + mail_index_lookup_ext(view, old_seq, ctx->box->mail_vsize_ext_id, + &data, &expunged); + if (data != NULL && !expunged) { + mail_index_update_ext(ctx->trans, new_seq, + ctx->box->mail_vsize_ext_id, data, NULL); + } +} + +static void +index_index_copy_cache(struct index_rebuild_context *ctx, + struct mail_index_view *view, + uint32_t old_seq, uint32_t new_seq) +{ + struct mail_index_map *map; + const void *data; + uint32_t reset_id = 0; + bool expunged; + + if (ctx->cache_ext_id == (uint32_t)-1) + return; + + mail_index_lookup_ext_full(view, old_seq, ctx->cache_ext_id, + &map, &data, &expunged); + if (expunged) + return; + + if (!mail_index_ext_get_reset_id(view, map, ctx->cache_ext_id, + &reset_id) || reset_id == 0) + return; + + if (!ctx->cache_used) { + /* set reset id */ + ctx->cache_used = TRUE; + ctx->cache_reset_id = reset_id; + mail_index_ext_reset(ctx->trans, ctx->cache_ext_id, + ctx->cache_reset_id, TRUE); + } + if (ctx->cache_reset_id == reset_id) { + mail_index_update_ext(ctx->trans, new_seq, + ctx->cache_ext_id, data, NULL); + } +} + +static void +index_index_copy_from_old(struct index_rebuild_context *ctx, + struct mail_index_view *view, + uint32_t old_seq, uint32_t new_seq) +{ + struct mail_index *index = mail_index_view_get_index(view); + const struct mail_index_record *rec; + ARRAY_TYPE(keyword_indexes) old_keywords; + struct mail_keywords *kw; + uint64_t modseq; + + /* copy flags */ + rec = mail_index_lookup(view, old_seq); + mail_index_update_flags(ctx->trans, new_seq, + MODIFY_REPLACE, rec->flags); + + /* copy keywords */ + t_array_init(&old_keywords, 32); + mail_index_lookup_keywords(view, old_seq, &old_keywords); + kw = mail_index_keywords_create_from_indexes(index, &old_keywords); + mail_index_update_keywords(ctx->trans, new_seq, MODIFY_REPLACE, kw); + mail_index_keywords_unref(&kw); + + /* copy modseq */ + modseq = mail_index_modseq_lookup(view, old_seq); + mail_index_update_modseq(ctx->trans, new_seq, modseq); + + index_index_copy_vsize(ctx, view, old_seq, new_seq); + index_index_copy_cache(ctx, view, old_seq, new_seq); +} + +void index_rebuild_index_metadata(struct index_rebuild_context *ctx, + uint32_t new_seq, uint32_t uid) +{ + uint32_t old_seq; + + if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) { + /* the message exists in the old index. + copy the metadata from it. */ + index_index_copy_from_old(ctx, ctx->view, old_seq, new_seq); + } else if (ctx->backup_view != NULL && + mail_index_lookup_seq(ctx->backup_view, uid, &old_seq)) { + /* copy the metadata from backup index. */ + index_index_copy_from_old(ctx, ctx->backup_view, + old_seq, new_seq); + } +} + +static void +index_rebuild_header(struct index_rebuild_context *ctx, + index_rebuild_generate_uidvalidity_t *gen_uidvalidity) +{ + const struct mail_index_header *hdr, *backup_hdr, *trans_hdr; + struct mail_index *index = mail_index_view_get_index(ctx->view); + struct mail_index_modseq_header modseq_hdr; + struct mail_index_view *trans_view; + uint32_t uid_validity, next_uid, first_recent_uid; + uint64_t modseq; + + hdr = mail_index_get_header(ctx->view); + backup_hdr = ctx->backup_view == NULL ? NULL : + mail_index_get_header(ctx->backup_view); + trans_view = mail_index_transaction_open_updated_view(ctx->trans); + trans_hdr = mail_index_get_header(trans_view); + + /* set uidvalidity */ + if (hdr->uid_validity != 0) + uid_validity = hdr->uid_validity; + else if (backup_hdr != NULL && backup_hdr->uid_validity != 0) + uid_validity = backup_hdr->uid_validity; + else + uid_validity = gen_uidvalidity(ctx->box->list); + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + + /* set next-uid */ + if (hdr->next_uid != 0) + next_uid = hdr->next_uid; + else if (backup_hdr != NULL && backup_hdr->next_uid != 0) + next_uid = backup_hdr->next_uid; + else + next_uid = 1; + if (next_uid > trans_hdr->next_uid) { + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, next_uid), + &next_uid, sizeof(next_uid), FALSE); + } + + /* set first_recent_uid */ + first_recent_uid = hdr->first_recent_uid; + if (backup_hdr != NULL && + backup_hdr->first_recent_uid > first_recent_uid && + backup_hdr->first_recent_uid <= next_uid) + first_recent_uid = backup_hdr->first_recent_uid; + first_recent_uid = I_MIN(first_recent_uid, next_uid); + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + + /* set highest-modseq */ + i_zero(&modseq_hdr); + modseq_hdr.highest_modseq = mail_index_modseq_get_highest(ctx->view); + if (ctx->backup_view != NULL) { + modseq = mail_index_modseq_get_highest(ctx->backup_view); + if (modseq_hdr.highest_modseq < modseq) + modseq_hdr.highest_modseq = modseq; + } + mail_index_update_header_ext(ctx->trans, index->modseq_ext_id, + 0, &modseq_hdr, sizeof(modseq_hdr)); + mail_index_view_close(&trans_view); +} + +static void +index_rebuild_box_preserve_header(struct index_rebuild_context *ctx, + uint32_t ext_id) +{ + const void *hdr; + size_t hdr_size; + + mail_index_get_header_ext(ctx->view, ext_id, &hdr, &hdr_size); + if (hdr_size == 0 && ctx->backup_view != NULL) { + mail_index_get_header_ext(ctx->backup_view, ext_id, + &hdr, &hdr_size); + } + if (hdr_size == 0) + return; + mail_index_update_header_ext(ctx->trans, ext_id, 0, hdr, hdr_size); +} + +struct index_rebuild_context * +index_index_rebuild_init(struct mailbox *box, struct mail_index_view *view, + struct mail_index_transaction *trans) +{ + struct index_rebuild_context *ctx; + const char *index_dir, *backup_path; + enum mail_index_open_flags open_flags = MAIL_INDEX_OPEN_FLAG_READONLY; + + /* Rebuilding really should be done locked so multiple processes won't + try to rebuild concurrently. Also at the end of rebiuld cache + purging requires this lock. */ + i_assert(mail_index_is_locked(view->index)); + + ctx = i_new(struct index_rebuild_context, 1); + ctx->box = box; + ctx->view = view; + ctx->trans = trans; + mail_index_reset(ctx->trans); + mailbox_recent_flags_reset(box); + (void)mail_index_ext_lookup(box->index, "cache", &ctx->cache_ext_id); + + /* open cache and read the caching decisions. */ + (void)mail_cache_open_and_verify(ctx->box->cache); + + /* if backup index file exists, try to use it */ + index_dir = mailbox_get_index_path(box); + backup_path = t_strconcat(box->index_prefix, ".backup", NULL); + ctx->backup_index = mail_index_alloc(box->event, + index_dir, backup_path); + +#ifndef MMAP_CONFLICTS_WRITE + if (box->storage->set->mmap_disable) +#endif + open_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE; + mail_index_set_lock_method(ctx->backup_index, + box->storage->set->parsed_lock_method, + UINT_MAX); + if (mail_index_open(ctx->backup_index, open_flags) <= 0) + mail_index_free(&ctx->backup_index); + else + ctx->backup_view = mail_index_view_open(ctx->backup_index); + return ctx; +} + +void index_index_rebuild_deinit(struct index_rebuild_context **_ctx, + index_rebuild_generate_uidvalidity_t *cb) +{ + struct index_rebuild_context *ctx = *_ctx; + + *_ctx = NULL; + + /* initialize cache file with the old field decisions */ + (void)mail_cache_purge_with_trans(ctx->box->cache, ctx->trans, + (uint32_t)-1, "rebuilding index"); + index_rebuild_header(ctx, cb); + index_rebuild_box_preserve_header(ctx, ctx->box->box_name_hdr_ext_id); + index_rebuild_box_preserve_header(ctx, ctx->box->box_last_rename_stamp_ext_id); + index_rebuild_box_preserve_header(ctx, ctx->box->pop3_uidl_hdr_ext_id); + if (ctx->backup_index != NULL) { + mail_index_view_close(&ctx->backup_view); + mail_index_close(ctx->backup_index); + mail_index_free(&ctx->backup_index); + } + i_free(ctx); +} diff --git a/src/lib-storage/index/index-rebuild.h b/src/lib-storage/index/index-rebuild.h new file mode 100644 index 0000000..183da9d --- /dev/null +++ b/src/lib-storage/index/index-rebuild.h @@ -0,0 +1,32 @@ +#ifndef INDEX_REBUILD_H +#define INDEX_REBUILD_H + +struct mailbox_list; + +struct index_rebuild_context { + struct mailbox *box; + + struct mail_index_view *view; + struct mail_index_transaction *trans; + uint32_t cache_ext_id; + uint32_t cache_reset_id; + + struct mail_index *backup_index; + struct mail_index_view *backup_view; + + bool cache_used:1; +}; + +typedef unsigned int +index_rebuild_generate_uidvalidity_t(struct mailbox_list *list); + +struct index_rebuild_context * +index_index_rebuild_init(struct mailbox *box, struct mail_index_view *view, + struct mail_index_transaction *trans); +void index_index_rebuild_deinit(struct index_rebuild_context **ctx, + index_rebuild_generate_uidvalidity_t *cb); + +void index_rebuild_index_metadata(struct index_rebuild_context *ctx, + uint32_t new_seq, uint32_t uid); + +#endif diff --git a/src/lib-storage/index/index-search-mime.c b/src/lib-storage/index/index-search-mime.c new file mode 100644 index 0000000..da7e5e1 --- /dev/null +++ b/src/lib-storage/index/index-search-mime.c @@ -0,0 +1,624 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "message-date.h" +#include "message-address.h" +#include "message-part-data.h" +#include "imap-bodystructure.h" +#include "mail-search.h" +#include "mail-search-mime.h" +#include "index-search-private.h" + +struct search_mimepart_stack { + unsigned int index; +}; + +struct search_mimepart_context { + pool_t pool; + struct index_search_context *index_ctx; + + /* message parts parsed from BODYSTRUCTURE */ + struct message_part *mime_parts, *mime_part; + + string_t *buf; + + unsigned int depth, index; + ARRAY(struct search_mimepart_stack) stack; +}; + +static void search_mime_arg(struct mail_search_mime_arg *arg, + struct search_mimepart_context *mpctx); + +static int seach_arg_mime_parent_match(struct search_mimepart_context *mpctx, + struct mail_search_mime_arg *args) +{ + struct message_part *part = mpctx->mime_part; + unsigned int prev_depth, prev_index; + struct search_mimepart_stack *level; + int ret; + + if (args->value.subargs == NULL) { + /* PARENT EXISTS: matches if this part has a parent. + */ + return (part->parent != NULL ? 1 : 0); + } + + /* PARENT <mpart-key>: matches if this part's parent matches the + mpart-key (subargs). + */ + + prev_depth = mpctx->depth; + prev_index = mpctx->index; + + level = array_idx_modifiable + (&mpctx->stack, mpctx->depth-1); + + mpctx->mime_part = part->parent; + mail_search_mime_args_reset(args->value.subargs, TRUE); + + mpctx->index = level->index; + mpctx->depth = mpctx->depth-1; + ret = mail_search_mime_args_foreach + (args->value.subargs, search_mime_arg, mpctx); + + mpctx->mime_part = part; + mpctx->index = prev_index; + mpctx->depth = prev_depth; + return ret; +} + +static int seach_arg_mime_child_match(struct search_mimepart_context *mpctx, + struct mail_search_mime_arg *args) +{ + struct message_part *part, *prev_part; + unsigned int prev_depth, prev_index, depth; + struct search_mimepart_stack *level; + int ret = 0; + + part = mpctx->mime_part; + if (args->value.subargs == NULL) { + /* CHILD EXISTS: matches if this part has any children; i.e., it is + multipart. + */ + return (part->children != NULL ? 1 : 0); + } + + /* CHILD <mpart-key>: matches if this part has any child that mathes + the mpart-key (subargs). + */ + + prev_part = part; + prev_depth = mpctx->depth; + prev_index = mpctx->index; + + depth = mpctx->depth; + T_BEGIN { + ARRAY(struct search_mimepart_stack) prev_stack; + + /* preserve current stack for any nested CHILD PARENT nastiness */ + t_array_init(&prev_stack, 16); + array_copy(&prev_stack.arr, 0, &mpctx->stack.arr, 0, + array_count(&mpctx->stack)); + + depth++; + if (depth < array_count(&mpctx->stack)) + level = array_idx_modifiable(&mpctx->stack, depth); + else { + i_assert(depth == array_count(&mpctx->stack)); + level = array_append_space(&mpctx->stack); + } + level->index = 1; + + part = part->children; + while (part != NULL) { + mpctx->mime_part = part; + mail_search_mime_args_reset(args->value.subargs, TRUE); + + mpctx->depth = depth - prev_depth; + mpctx->index = level->index; + if ((ret=mail_search_mime_args_foreach + (args->value.subargs, search_mime_arg, mpctx)) != 0) + break; + if (part->children != NULL) { + depth++; + if (depth < array_count(&mpctx->stack)) + level = array_idx_modifiable(&mpctx->stack, depth); + else { + i_assert(depth == array_count(&mpctx->stack)); + level = array_append_space(&mpctx->stack); + } + level->index = 1; + part = part->children; + } else { + while (part->next == NULL) { + if (part->parent == NULL || part->parent == prev_part) + break; + depth--; + level = array_idx_modifiable(&mpctx->stack, depth); + part = part->parent; + } + level->index++; + part = part->next; + } + } + + array_clear(&mpctx->stack); + array_copy(&mpctx->stack.arr, 0, &prev_stack.arr, 0, + array_count(&prev_stack)); + } T_END; + + mpctx->mime_part = prev_part; + mpctx->index = prev_index; + mpctx->depth = prev_depth; + return ret; +} + +static int +seach_arg_mime_substring_match( + struct search_mimepart_context *mpctx ATTR_UNUSED, + const char *key, const char *value) +{ + if (value == NULL) + return 0; + + /* FIXME: Normalization is required */ + return (strstr(value, key) != NULL ? 1 : 0); +} + +static int +seach_arg_mime_envelope_time_match( + struct search_mimepart_context *mpctx ATTR_UNUSED, + enum mail_search_mime_arg_type type, time_t search_time, + const struct message_part_envelope *envelope) +{ + time_t sent_time; + int timezone_offset; + + if (envelope == NULL) + return 0; + + /* NOTE: RFC-3501 specifies that timezone is ignored + in searches. sent_time is returned as UTC, so change it. */ + // FIXME: adjust comment + if (!message_date_parse((const unsigned char *)envelope->date, + strlen(envelope->date), &sent_time, &timezone_offset)) + return 0; + sent_time += timezone_offset * 60; + + switch (type) { + case SEARCH_MIME_SENTBEFORE: + return sent_time < search_time ? 1 : 0; + case SEARCH_MIME_SENTON: + return (sent_time >= search_time && + sent_time < search_time + 3600*24) ? 1 : 0; + case SEARCH_MIME_SENTSINCE: + return sent_time >= search_time ? 1 : 0; + default: + i_unreached(); + } +} + +static int +seach_arg_mime_envelope_address_match( + struct search_mimepart_context *mpctx ATTR_UNUSED, + enum mail_search_mime_arg_type type, const char *key, + const struct message_part_envelope *envelope) +{ + const struct message_address *addrs; + string_t *addrs_enc; + + if (envelope == NULL) + return 0; + + switch (type) { + case SEARCH_MIME_CC: + addrs = envelope->cc; + break; + case SEARCH_MIME_BCC: + addrs = envelope->bcc; + break; + case SEARCH_MIME_FROM: + addrs = envelope->from; + break; + case SEARCH_MIME_SENDER: + addrs = envelope->sender; + break; + case SEARCH_MIME_REPLY_TO: + addrs = envelope->reply_to; + break; + case SEARCH_MIME_TO: + addrs = envelope->to; + break; + default: + i_unreached(); + } + + /* FIXME: do we need to normalize anything? at least case insensitivity. + MIME header encoding will make this a bit difficult, so it should + probably be normalized directly in the struct message_address. */ + + addrs_enc = t_str_new(128); + message_address_write(addrs_enc, addrs); + return (strstr(str_c(addrs_enc), key) != NULL ? 1 : 0); +} + +static int +seach_arg_mime_filename_match(struct search_mimepart_context *mpctx, + struct mail_search_mime_arg *arg) +{ + struct index_search_context *ictx = mpctx->index_ctx; + struct message_part *part = mpctx->mime_part; + char *key; + const char *value; + size_t vlen, alen; + + if (!message_part_data_get_filename(part, &value)) + return 0; + + if (mpctx->buf == NULL) + mpctx->buf = str_new(default_pool, 256); + + if (arg->context == NULL) { + str_truncate(mpctx->buf, 0); + + if (ictx->mail_ctx.normalizer(arg->value.str, + strlen(arg->value.str), mpctx->buf) < 0) + i_panic("search key not utf8: %s", arg->value.str); + key = i_strdup(str_c(mpctx->buf)); + arg->context = (void *)key; + } else { + key = (char *)arg->context; + } + + str_truncate(mpctx->buf, 0); + if (ictx->mail_ctx.normalizer(value, + strlen(value), mpctx->buf) >= 0) + value = str_c(mpctx->buf); + + switch (arg->type) { + case SEARCH_MIME_FILENAME_IS: + return (strcmp(value, key) == 0 ? 1 : 0); + case SEARCH_MIME_FILENAME_CONTAINS: + return (strstr(value, key) != NULL ? 1 : 0); + case SEARCH_MIME_FILENAME_BEGINS: + return (str_begins(value, key) ? 1 : 0); + case SEARCH_MIME_FILENAME_ENDS: + vlen = strlen(value); + alen = strlen(key); + return (str_begins(value + (vlen - alen), key) ? 1 : 0); + default: + break; + } + i_unreached(); +} +static void +search_arg_mime_filename_deinit( + struct search_mimepart_context *mpctx ATTR_UNUSED, + struct mail_search_mime_arg *arg) +{ + char *key = (char *)arg->context; + + i_free(key); +} + +static int +seach_arg_mime_param_match(const struct message_part_param *params, + unsigned int params_count, + const char *name, const char *key) +{ + unsigned int i; + + /* FIXME: Is normalization required? */ + + for (i = 0; i < params_count; i++) { + if (strcasecmp(params[i].name, name) == 0) { + if (key == NULL || *key == '\0') + return 1; + return (strstr(params[i].value, key) != NULL ? 1 : 0); + } + } + return 0; +} + +static int +seach_arg_mime_language_match(struct search_mimepart_context *mpctx, + const char *key) +{ + struct message_part_data *data = mpctx->mime_part->data; + const char *const *lang; + + i_assert(data != NULL); + + lang = data->content_language; + if (lang != NULL) { + while (*lang != NULL) { + /* FIXME: Should use RFC 4647 matching rules */ + if (strcasecmp(*lang, key) == 0) + return 1; + lang++; + } + } + return 0; +} + +/* Returns >0 = matched, 0 = not matched (unused), -1 = unknown */ +static int search_mime_arg_match(struct search_mimepart_context *mpctx, + struct mail_search_mime_arg *arg) +{ + struct message_part *part = mpctx->mime_part; + const struct message_part_data *data = part->data; + + i_assert(data != NULL); + + switch (arg->type) { + case SEARCH_MIME_OR: + case SEARCH_MIME_SUB: + i_unreached(); + + case SEARCH_MIME_SIZE_EQUAL: + return (part->body_size.virtual_size == arg->value.size ? 1 : 0); + case SEARCH_MIME_SIZE_LARGER: + return (part->body_size.virtual_size > arg->value.size ? 1 : 0); + case SEARCH_MIME_SIZE_SMALLER: + return (part->body_size.virtual_size < arg->value.size ? 1 : 0); + + case SEARCH_MIME_DESCRIPTION: + return seach_arg_mime_substring_match(mpctx, + arg->value.str, data->content_description); + case SEARCH_MIME_DISPOSITION_TYPE: + return (data->content_disposition != NULL && + strcasecmp(data->content_disposition, + arg->value.str) == 0 ? 1 : 0); + case SEARCH_MIME_DISPOSITION_PARAM: + return seach_arg_mime_param_match + (data->content_disposition_params, + data->content_disposition_params_count, + arg->field_name, arg->value.str); + case SEARCH_MIME_ENCODING: + return (data->content_transfer_encoding != NULL && + strcasecmp(data->content_transfer_encoding, + arg->value.str) == 0 ? 1 : 0); + case SEARCH_MIME_ID: + return (data->content_id != NULL && + strcasecmp(data->content_id, + arg->value.str) == 0 ? 1 : 0); + case SEARCH_MIME_LANGUAGE: + return seach_arg_mime_language_match(mpctx, arg->value.str); + case SEARCH_MIME_LOCATION: + return (data->content_location != NULL && + strcasecmp(data->content_location, + arg->value.str) == 0 ? 1 : 0); + case SEARCH_MIME_MD5: + return (data->content_md5 != NULL && + strcmp(data->content_md5, + arg->value.str) == 0 ? 1 : 0); + + case SEARCH_MIME_TYPE: + return (data->content_type != NULL && + strcasecmp(data->content_type, + arg->value.str) == 0 ? 1 : 0); + case SEARCH_MIME_SUBTYPE: + return (data->content_subtype != NULL && + strcasecmp(data->content_subtype, + arg->value.str) == 0 ? 1 : 0); + case SEARCH_MIME_PARAM: + return seach_arg_mime_param_match + (data->content_type_params, + data->content_type_params_count, + arg->field_name, arg->value.str); + + case SEARCH_MIME_SENTBEFORE: + case SEARCH_MIME_SENTON: + case SEARCH_MIME_SENTSINCE: + return seach_arg_mime_envelope_time_match + (mpctx, arg->type, arg->value.time, data->envelope); + + case SEARCH_MIME_CC: + case SEARCH_MIME_BCC: + case SEARCH_MIME_FROM: + case SEARCH_MIME_REPLY_TO: + case SEARCH_MIME_SENDER: + case SEARCH_MIME_TO: + return seach_arg_mime_envelope_address_match + (mpctx, arg->type, arg->value.str, data->envelope); + + case SEARCH_MIME_SUBJECT: + if (data->envelope == NULL) + return 0; + return seach_arg_mime_substring_match(mpctx, + arg->value.str, data->envelope->subject); + case SEARCH_MIME_IN_REPLY_TO: + if (data->envelope == NULL) + return 0; + return seach_arg_mime_substring_match(mpctx, + arg->value.str, data->envelope->in_reply_to); + case SEARCH_MIME_MESSAGE_ID: + if (data->envelope == NULL) + return 0; + return seach_arg_mime_substring_match(mpctx, + arg->value.str, data->envelope->message_id); + + case SEARCH_MIME_DEPTH_EQUAL: + return (mpctx->depth == arg->value.number ? 1 : 0); + case SEARCH_MIME_DEPTH_MIN: + return (mpctx->depth >= arg->value.number ? 1 : 0); + case SEARCH_MIME_DEPTH_MAX: + return (mpctx->depth <= arg->value.number ? 1 : 0); + case SEARCH_MIME_INDEX: + return (mpctx->index == arg->value.number ? 1 : 0); + + case SEARCH_MIME_PARENT: + return seach_arg_mime_parent_match(mpctx, arg); + case SEARCH_MIME_CHILD: + return seach_arg_mime_child_match(mpctx, arg); + + case SEARCH_MIME_FILENAME_IS: + case SEARCH_MIME_FILENAME_CONTAINS: + case SEARCH_MIME_FILENAME_BEGINS: + case SEARCH_MIME_FILENAME_ENDS: + return seach_arg_mime_filename_match(mpctx, arg); + + case SEARCH_MIME_HEADER: + case SEARCH_MIME_BODY: + case SEARCH_MIME_TEXT: + break; + } + return -1; +} + +static void search_mime_arg(struct mail_search_mime_arg *arg, + struct search_mimepart_context *mpctx) +{ + switch (search_mime_arg_match(mpctx, arg)) { + case -1: + /* unknown */ + break; + case 0: + ARG_SET_RESULT(arg, 0); + break; + default: + ARG_SET_RESULT(arg, 1); + break; + } +} + +static int seach_arg_mime_parts_match(struct search_mimepart_context *mpctx, + struct mail_search_mime_arg *args, + struct message_part *parts) +{ + struct message_part *part; + struct search_mimepart_stack *level; + int ret; + + level = array_append_space(&mpctx->stack); + level->index = 1; + + part = parts; + while (part != NULL) { + mpctx->mime_part = part; + mail_search_mime_args_reset(args, TRUE); + + mpctx->index = level->index; + mpctx->depth = array_count(&mpctx->stack)-1; + + if ((ret=mail_search_mime_args_foreach + (args, search_mime_arg, mpctx)) != 0) + return ret; + if (part->children != NULL) { + level = array_append_space(&mpctx->stack); + level->index = 1; + part = part->children; + } else { + while (part->next == NULL) { + if (part->parent == NULL) + break; + array_pop_back(&mpctx->stack); + level = array_back_modifiable(&mpctx->stack); + part = part->parent; + } + level->index++; + part = part->next; + } + } + + return 0; +} + +/* Returns >0 = matched, 0 = not matched, -1 = unknown */ +static int search_arg_match_mimepart(struct search_mimepart_context *mpctx, + struct mail_search_arg *arg) +{ + struct index_search_context *ctx = mpctx->index_ctx; + const char *bodystructure, *error; + + if (arg->type != SEARCH_MIMEPART) + return -1; + + if (mpctx->pool == NULL) { + mpctx->pool = pool_alloconly_create + (MEMPOOL_GROWING"search mime parts", 4096); + p_array_init(&mpctx->stack, mpctx->pool, 16); + } + if (mpctx->mime_parts == NULL) { + /* FIXME: could the mail object already have message_part tree with + data? */ + if (mail_get_special(ctx->cur_mail, + MAIL_FETCH_IMAP_BODYSTRUCTURE, &bodystructure) < 0) + return -1; + if (imap_bodystructure_parse_full(bodystructure, mpctx->pool, + &mpctx->mime_parts, &error) < 0) + return -1; + } + + /* FIXME: implement HEADER, BODY and TEXT (not from BODYSTRUCTURE) + Needs to support FTS */ + return seach_arg_mime_parts_match + (mpctx, arg->value.mime_part->args, mpctx->mime_parts); +} + +static void search_mimepart_arg(struct mail_search_arg *arg, + struct search_mimepart_context *mpctx) +{ + switch (search_arg_match_mimepart(mpctx, arg)) { + case -1: + /* unknown */ + break; + case 0: + ARG_SET_RESULT(arg, 0); + break; + default: + ARG_SET_RESULT(arg, 1); + break; + } +} + +int index_search_mime_arg_match(struct mail_search_arg *args, + struct index_search_context *ctx) +{ + struct search_mimepart_context mpctx; + int ret; + + i_zero(&mpctx); + mpctx.index_ctx = ctx; + + ret = mail_search_args_foreach(args, + search_mimepart_arg, &mpctx); + + pool_unref(&mpctx.pool); + str_free(&mpctx.buf); + return ret; +} + +static void +search_mime_arg_deinit(struct mail_search_mime_arg *arg, + struct search_mimepart_context *mpctx ATTR_UNUSED) +{ + switch (arg->type) { + case SEARCH_MIME_FILENAME_IS: + case SEARCH_MIME_FILENAME_CONTAINS: + case SEARCH_MIME_FILENAME_BEGINS: + case SEARCH_MIME_FILENAME_ENDS: + search_arg_mime_filename_deinit(mpctx, arg); + break; + default: + break; + } +} + +void index_search_mime_arg_deinit(struct mail_search_arg *arg, + struct index_search_context *ctx) +{ + struct search_mimepart_context mpctx; + struct mail_search_mime_arg *args; + + i_assert(arg->type == SEARCH_MIMEPART); + args = arg->value.mime_part->args; + + i_zero(&mpctx); + mpctx.index_ctx = ctx; + + mail_search_mime_args_reset(args, TRUE); + (void)mail_search_mime_args_foreach(args, + search_mime_arg_deinit, &mpctx); +} diff --git a/src/lib-storage/index/index-search-private.h b/src/lib-storage/index/index-search-private.h new file mode 100644 index 0000000..f451b56 --- /dev/null +++ b/src/lib-storage/index/index-search-private.h @@ -0,0 +1,45 @@ +#ifndef INDEX_SEARCH_PRIVATE_H +#define INDEX_SEARCH_PRIVATE_H + +#include "mail-storage-private.h" + +#include <sys/time.h> + +struct mail_search_mime_part; +struct imap_message_part; + +struct index_search_context { + struct mail_search_context mail_ctx; + struct mail_index_view *view; + struct mailbox *box; + + uint32_t pvt_uid, pvt_seq; + + enum mail_fetch_field extra_wanted_fields; + struct mailbox_header_lookup_ctx *extra_wanted_headers; + + uint32_t seq1, seq2; + struct mail *cur_mail; + struct index_mail *cur_imail; + struct mail_thread_context *thread_ctx; + + struct timeval search_start_time, last_notify; + struct timeval last_nonblock_timeval; + unsigned long long cost, next_time_check_cost; + + bool failed:1; + bool sorted:1; + bool have_seqsets:1; + bool have_index_args:1; + bool have_mailbox_args:1; + bool have_nonmatch_always:1; +}; + +struct mail *index_search_get_mail(struct index_search_context *ctx); + +int index_search_mime_arg_match(struct mail_search_arg *args, + struct index_search_context *ctx); +void index_search_mime_arg_deinit(struct mail_search_arg *arg, + struct index_search_context *ctx); + +#endif diff --git a/src/lib-storage/index/index-search-result.c b/src/lib-storage/index/index-search-result.c new file mode 100644 index 0000000..6bce84f --- /dev/null +++ b/src/lib-storage/index/index-search-result.c @@ -0,0 +1,194 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "seq-range-array.h" +#include "mail-search.h" +#include "mailbox-search-result-private.h" +#include "index-storage.h" +#include "index-search-result.h" + +static void +search_result_range_remove(struct mail_search_result *result, + const ARRAY_TYPE(seq_range) *changed_uids_arr, + unsigned int *idx, + uint32_t *next_uid, uint32_t last_uid) +{ + const struct seq_range *uids; + unsigned int i, count; + uint32_t uid; + + /* remove full seq_ranges */ + uid = *next_uid; + uids = array_get(changed_uids_arr, &count); + for (i = *idx; uids[i].seq2 < last_uid;) { + i_assert(uids[i].seq1 <= uid); + for (; uid <= uids[i].seq2; uid++) + mailbox_search_result_remove(result, uid); + i++; + i_assert(i < count); + uid = uids[i].seq1; + } + + /* remove the last seq_range */ + i_assert(uids[i].seq1 <= uid && uids[i].seq2 >= last_uid); + for (; uid <= last_uid; uid++) + mailbox_search_result_remove(result, uid); + + if (uid > uids[i].seq2) { + /* finished this range */ + if (++i < count) + uid = uids[i].seq1; + else { + /* this was the last searched message */ + uid = 0; + } + } + + *next_uid = uid; + *idx = i; +} + +static int +search_result_update_search(struct mail_search_result *result, + const ARRAY_TYPE(seq_range) *changed_uids_arr) +{ + struct mailbox_transaction_context *t; + struct mail_search_context *search_ctx; + struct mail *mail; + const struct seq_range *changed_uids; + unsigned int changed_count, changed_idx; + uint32_t next_uid; + int ret; + + changed_uids = array_get(changed_uids_arr, &changed_count); + i_assert(changed_count > 0); + next_uid = changed_uids[0].seq1; + changed_idx = 0; + + mail_search_args_init(result->search_args, result->box, FALSE, NULL); + + t = mailbox_transaction_begin(result->box, 0, __func__); + search_ctx = mailbox_search_init(t, result->search_args, NULL, 0, NULL); + /* tell search that we're updating an existing search result, + so it can do some optimizations based on it */ + search_ctx->update_result = result; + + while (mailbox_search_next(search_ctx, &mail)) { + i_assert(next_uid != 0); + + if (next_uid != mail->uid) { + /* some messages in changed_uids didn't match. + make sure they don't exist in the search result. */ + search_result_range_remove(result, changed_uids_arr, + &changed_idx, &next_uid, + mail->uid-1); + i_assert(next_uid == mail->uid); + } + if (changed_uids[changed_idx].seq2 > next_uid) { + next_uid++; + } else if (++changed_idx < changed_count) { + next_uid = changed_uids[changed_idx].seq1; + } else { + /* this was the last searched message */ + next_uid = 0; + } + /* match - make sure it exists in search result */ + mailbox_search_result_add(result, mail->uid); + } + mail_search_args_deinit(result->search_args); + ret = mailbox_search_deinit(&search_ctx); + + if (next_uid != 0 && ret == 0) { + /* last message(s) didn't match. make sure they don't exist + in the search result. */ + search_result_range_remove(result, changed_uids_arr, + &changed_idx, &next_uid, + changed_uids[changed_count-1].seq2); + } + + if (mailbox_transaction_commit(&t) < 0) + ret = -1; + return ret; +} + +int index_search_result_update_flags(struct mail_search_result *result, + const ARRAY_TYPE(seq_range) *uids) +{ + struct mail_search_arg search_arg; + int ret; + + if (array_count(uids) == 0) + return 0; + + /* add a temporary search parameter to limit the search only to + the changed messages */ + i_zero(&search_arg); + search_arg.type = SEARCH_UIDSET; + search_arg.value.seqset = *uids; + search_arg.next = result->search_args->args; + result->search_args->args = &search_arg; + ret = search_result_update_search(result, uids); + i_assert(result->search_args->args == &search_arg); + result->search_args->args = search_arg.next; + return ret; +} + +int index_search_result_update_appends(struct mail_search_result *result, + unsigned int old_messages_count) +{ + struct mailbox_transaction_context *t; + struct mail_search_context *search_ctx; + struct mail *mail; + struct mail_search_arg search_arg; + uint32_t message_count; + int ret; + + message_count = mail_index_view_get_messages_count(result->box->view); + if (old_messages_count == message_count) { + /* no new messages */ + return 0; + } + + /* add a temporary search parameter to limit the search only to + the new messages */ + i_zero(&search_arg); + search_arg.type = SEARCH_SEQSET; + t_array_init(&search_arg.value.seqset, 1); + seq_range_array_add_range(&search_arg.value.seqset, + old_messages_count + 1, message_count); + search_arg.next = result->search_args->args; + result->search_args->args = &search_arg; + + /* add all messages matching the search to search result */ + t = mailbox_transaction_begin(result->box, 0, __func__); + search_ctx = mailbox_search_init(t, result->search_args, NULL, 0, NULL); + + while (mailbox_search_next(search_ctx, &mail)) + mailbox_search_result_add(result, mail->uid); + + ret = mailbox_search_deinit(&search_ctx); + if (mailbox_transaction_commit(&t) < 0) + ret = -1; + + i_assert(result->search_args->args == &search_arg); + result->search_args->args = search_arg.next; + return ret; +} + +void index_search_results_update_expunges(struct mailbox *box, + const ARRAY_TYPE(seq_range) *expunges) +{ + const struct seq_range *seqs; + uint32_t seq, uid; + + if (array_count(&box->search_results) == 0) + return; + + array_foreach(expunges, seqs) { + for (seq = seqs->seq1; seq <= seqs->seq2; seq++) { + mail_index_lookup_uid(box->view, seq, &uid); + mailbox_search_results_remove(box, uid); + } + } +} diff --git a/src/lib-storage/index/index-search-result.h b/src/lib-storage/index/index-search-result.h new file mode 100644 index 0000000..a22d368 --- /dev/null +++ b/src/lib-storage/index/index-search-result.h @@ -0,0 +1,11 @@ +#ifndef INDEX_SEARCH_RESULT_H +#define INDEX_SEARCH_RESULT_H + +int index_search_result_update_flags(struct mail_search_result *result, + const ARRAY_TYPE(seq_range) *uids); +int index_search_result_update_appends(struct mail_search_result *result, + unsigned int old_messages_count); +void index_search_results_update_expunges(struct mailbox *box, + const ARRAY_TYPE(seq_range) *expunges); + +#endif diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c new file mode 100644 index 0000000..05eaa39 --- /dev/null +++ b/src/lib-storage/index/index-search.c @@ -0,0 +1,1923 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "istream.h" +#include "utc-offset.h" +#include "str.h" +#include "time-util.h" +#include "unichar.h" +#include "imap-match.h" +#include "message-address.h" +#include "message-date.h" +#include "message-search.h" +#include "message-parser.h" +#include "mail-index-modseq.h" +#include "index-storage.h" +#include "index-mail.h" +#include "index-sort.h" +#include "mail-search.h" +#include "mailbox-search-result-private.h" +#include "mailbox-recent-flags.h" +#include "index-search-private.h" + +#include <ctype.h> + +#define SEARCH_NOTIFY_INTERVAL_SECS 10 + +#define SEARCH_COST_DENTRY 3ULL +#define SEARCH_COST_ATTR 1ULL +#define SEARCH_COST_FILES_READ 25ULL +#define SEARCH_COST_KBYTE 15ULL +#define SEARCH_COST_CACHE 1ULL + +#define SEARCH_MIN_NONBLOCK_USECS 200000 +#define SEARCH_MAX_NONBLOCK_USECS 250000 +#define SEARCH_INITIAL_MAX_COST 30000 +#define SEARCH_RECALC_MIN_USECS 50000 + +struct search_header_context { + struct index_search_context *index_ctx; + struct index_mail *imail; + struct mail_search_arg *args; + + struct message_block decoded_block; + bool decoded_block_set; + + struct message_header_line *hdr; + + bool parse_headers:1; + bool custom_header:1; + bool threading:1; +}; + +struct search_body_context { + struct index_search_context *index_ctx; + struct istream *input; + struct message_part *part; +}; + +static void search_parse_msgset_args(unsigned int messages_count, + struct mail_search_arg *args, + uint32_t *seq1_r, uint32_t *seq2_r); + +static void ATTR_NULL(2) +search_none(struct mail_search_arg *arg ATTR_UNUSED, void *ctx ATTR_UNUSED) +{ +} + +static void search_set_failed(struct index_search_context *ctx) +{ + if (ctx->failed) + return; + + /* remember the first failure */ + mail_storage_last_error_push(ctx->box->storage); + ctx->failed = TRUE; +} + +static void search_cur_mail_failed(struct index_search_context *ctx) +{ + switch (mailbox_get_last_mail_error(ctx->cur_mail->box)) { + case MAIL_ERROR_EXPUNGED: + ctx->mail_ctx.seen_lost_data = TRUE; + break; + case MAIL_ERROR_LOOKUP_ABORTED: + /* expected failure */ + break; + default: + search_set_failed(ctx); + break; + } +} + +static void search_init_arg(struct mail_search_arg *arg, + struct index_search_context *ctx) +{ + struct mailbox_metadata metadata; + bool match; + + switch (arg->type) { + case SEARCH_SEQSET: + ctx->have_seqsets = TRUE; + break; + case SEARCH_UIDSET: + case SEARCH_INTHREAD: + case SEARCH_FLAGS: + case SEARCH_KEYWORDS: + case SEARCH_MODSEQ: + if (arg->type == SEARCH_MODSEQ) + mail_index_modseq_enable(ctx->box->index); + ctx->have_index_args = TRUE; + break; + case SEARCH_MAILBOX_GUID: + if (mailbox_get_metadata(ctx->box, MAILBOX_METADATA_GUID, + &metadata) < 0) { + /* result will be unknown */ + break; + } + + match = strcmp(guid_128_to_string(metadata.guid), + arg->value.str) == 0; + if (match != arg->match_not) + arg->match_always = TRUE; + else { + arg->nonmatch_always = TRUE; + ctx->have_nonmatch_always = TRUE; + } + break; + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GLOB: + ctx->have_mailbox_args = TRUE; + break; + case SEARCH_ALL: + if (!arg->match_not) + arg->match_always = TRUE; + else { + arg->nonmatch_always = TRUE; + ctx->have_nonmatch_always = TRUE; + } + break; + default: + break; + } +} + +static void search_seqset_arg(struct mail_search_arg *arg, + struct index_search_context *ctx) +{ + if (arg->type == SEARCH_SEQSET) { + if (seq_range_exists(&arg->value.seqset, ctx->mail_ctx.seq)) + ARG_SET_RESULT(arg, 1); + else + ARG_SET_RESULT(arg, 0); + } +} + +static int search_arg_match_keywords(struct index_search_context *ctx, + struct mail_search_arg *arg) +{ + ARRAY_TYPE(keyword_indexes) keyword_indexes_arr; + const struct mail_keywords *search_kws = arg->initialized.keywords; + const unsigned int *keyword_indexes; + unsigned int i, j, count; + + if (search_kws->count == 0) { + /* invalid keyword - never matches */ + return 0; + } + + t_array_init(&keyword_indexes_arr, 128); + mail_index_lookup_keywords(ctx->view, ctx->mail_ctx.seq, + &keyword_indexes_arr); + keyword_indexes = array_get(&keyword_indexes_arr, &count); + + /* there probably aren't many keywords, so O(n*m) for now */ + for (i = 0; i < search_kws->count; i++) { + for (j = 0; j < count; j++) { + if (search_kws->idx[i] == keyword_indexes[j]) + break; + } + if (j == count) + return 0; + } + return 1; +} + +static bool +index_search_get_pvt(struct index_search_context *ctx, uint32_t uid) +{ + index_transaction_init_pvt(ctx->mail_ctx.transaction); + + if (ctx->pvt_uid == uid) + return ctx->pvt_seq != 0; + ctx->pvt_uid = uid; + return mail_index_lookup_seq(ctx->mail_ctx.transaction->view_pvt, + uid, &ctx->pvt_seq); +} + +/* Returns >0 = matched, 0 = not matched, -1 = unknown */ +static int search_arg_match_index(struct index_search_context *ctx, + struct mail_search_arg *arg, + const struct mail_index_record *rec) +{ + enum mail_flags flags, pvt_flags_mask; + uint64_t modseq; + int ret; + + switch (arg->type) { + case SEARCH_UIDSET: + case SEARCH_INTHREAD: + return seq_range_exists(&arg->value.seqset, rec->uid) ? 1 : 0; + case SEARCH_FLAGS: + /* recent flag shouldn't be set, but indexes from v1.0.x + may contain it. */ + flags = rec->flags & ENUM_NEGATE(MAIL_RECENT); + if ((arg->value.flags & MAIL_RECENT) != 0 && + mailbox_recent_flags_have_uid(ctx->box, rec->uid)) + flags |= MAIL_RECENT; + if (ctx->box->view_pvt == NULL) { + /* no private view (set by view syncing) -> + no private flags */ + } else { + pvt_flags_mask = mailbox_get_private_flags_mask(ctx->box); + flags &= ENUM_NEGATE(pvt_flags_mask); + if (index_search_get_pvt(ctx, rec->uid)) { + rec = mail_index_lookup(ctx->mail_ctx.transaction->view_pvt, + ctx->pvt_seq); + flags |= rec->flags & pvt_flags_mask; + } + } + return (flags & arg->value.flags) == arg->value.flags ? 1 : 0; + case SEARCH_KEYWORDS: + T_BEGIN { + ret = search_arg_match_keywords(ctx, arg); + } T_END; + return ret; + case SEARCH_MODSEQ: { + if (arg->value.flags != 0) { + modseq = mail_index_modseq_lookup_flags(ctx->view, + arg->value.flags, ctx->mail_ctx.seq); + } else if (arg->initialized.keywords != NULL) { + modseq = mail_index_modseq_lookup_keywords(ctx->view, + arg->initialized.keywords, ctx->mail_ctx.seq); + } else { + modseq = mail_index_modseq_lookup(ctx->view, + ctx->mail_ctx.seq); + } + return modseq >= arg->value.modseq->modseq ? 1 : 0; + } + default: + return -1; + } +} + +static void search_index_arg(struct mail_search_arg *arg, + struct index_search_context *ctx) +{ + const struct mail_index_record *rec; + + rec = mail_index_lookup(ctx->view, ctx->mail_ctx.seq); + switch (search_arg_match_index(ctx, arg, rec)) { + case -1: + /* unknown */ + break; + case 0: + ARG_SET_RESULT(arg, 0); + break; + default: + ARG_SET_RESULT(arg, 1); + break; + } +} + +/* Returns >0 = matched, 0 = not matched, -1 = unknown */ +static int search_arg_match_mailbox(struct index_search_context *ctx, + struct mail_search_arg *arg) +{ + struct mailbox *box = ctx->cur_mail->box; + const char *str; + + switch (arg->type) { + case SEARCH_MAILBOX: + /* first try to match the mailbox name itself. this is + important when using "mailbox virtual/foo" parameter foin + doveadm's search query, otherwise we can never fetch + anything with doveadm from virtual mailboxes because the + mailbox parameter is compared to the mail's backend + mailbox. */ + if (strcmp(box->vname, arg->value.str) == 0) + return 1; + if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME, + &str) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + + if (strcasecmp(str, "INBOX") == 0) + return strcasecmp(arg->value.str, "INBOX") == 0 ? 1 : 0; + return strcmp(str, arg->value.str) == 0 ? 1 : 0; + case SEARCH_MAILBOX_GLOB: + if (imap_match(arg->initialized.mailbox_glob, box->vname) == IMAP_MATCH_YES) + return 1; + if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME, + &str) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + return imap_match(arg->initialized.mailbox_glob, str) == IMAP_MATCH_YES ? 1 : 0; + default: + return -1; + } +} + +static void search_mailbox_arg(struct mail_search_arg *arg, + struct index_search_context *ctx) +{ + switch (search_arg_match_mailbox(ctx, arg)) { + case -1: + /* unknown */ + break; + case 0: + ARG_SET_RESULT(arg, 0); + break; + default: + ARG_SET_RESULT(arg, 1); + break; + } +} + +/* Returns >0 = matched, 0 = not matched, -1 = unknown */ +static int search_arg_match_cached(struct index_search_context *ctx, + struct mail_search_arg *arg) +{ + const char *str; + struct tm *tm; + uoff_t virtual_size; + time_t date; + int tz_offset; + bool have_tz_offset; + int ret; + + switch (arg->type) { + /* internal dates */ + case SEARCH_BEFORE: + case SEARCH_ON: + case SEARCH_SINCE: + have_tz_offset = FALSE; tz_offset = 0; date = (time_t)-1; + switch (arg->value.date_type) { + case MAIL_SEARCH_DATE_TYPE_SENT: + if (mail_get_date(ctx->cur_mail, &date, &tz_offset) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + have_tz_offset = TRUE; + break; + case MAIL_SEARCH_DATE_TYPE_RECEIVED: + if (mail_get_received_date(ctx->cur_mail, &date) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + break; + case MAIL_SEARCH_DATE_TYPE_SAVED: + if (mail_get_save_date(ctx->cur_mail, &date) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + break; + } + + if ((arg->value.search_flags & + MAIL_SEARCH_ARG_FLAG_UTC_TIMES) == 0) { + if (!have_tz_offset) { + tm = localtime(&date); + tz_offset = utc_offset(tm, date); + } + date += tz_offset * 60; + } + + switch (arg->type) { + case SEARCH_BEFORE: + return date < arg->value.time ? 1 : 0; + case SEARCH_ON: + return (date >= arg->value.time && + date < arg->value.time + 3600*24) ? 1 : 0; + case SEARCH_SINCE: + return date >= arg->value.time ? 1 : 0; + default: + i_unreached(); + } + + /* save date attribute */ + case SEARCH_SAVEDATESUPPORTED: + ret = mail_get_save_date(ctx->cur_mail, &date); + if (ret < 0) { + search_cur_mail_failed(ctx); + return -1; + } + return ret; + + /* sizes */ + case SEARCH_SMALLER: + case SEARCH_LARGER: + if (mail_get_virtual_size(ctx->cur_mail, &virtual_size) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + + if (arg->type == SEARCH_SMALLER) + return virtual_size < arg->value.size ? 1 : 0; + else + return virtual_size > arg->value.size ? 1 : 0; + + case SEARCH_GUID: + if (mail_get_special(ctx->cur_mail, MAIL_FETCH_GUID, &str) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + return strcmp(str, arg->value.str) == 0 ? 1 : 0; + case SEARCH_REAL_UID: { + struct mail *real_mail; + + if (mail_get_backend_mail(ctx->cur_mail, &real_mail) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + return seq_range_exists(&arg->value.seqset, real_mail->uid) ? 1 : 0; + } + default: + return -1; + } +} + +static void search_cached_arg(struct mail_search_arg *arg, + struct index_search_context *ctx) +{ + switch (search_arg_match_cached(ctx, arg)) { + case -1: + /* unknown */ + break; + case 0: + ARG_SET_RESULT(arg, 0); + break; + default: + ARG_SET_RESULT(arg, 1); + break; + } +} + +static int search_sent(enum mail_search_arg_type type, time_t search_time, + const unsigned char *sent_value, size_t sent_value_len) +{ + time_t sent_time; + int timezone_offset; + + if (sent_value == NULL) + return 0; + + /* NOTE: RFC-3501 specifies that timezone is ignored + in searches. sent_time is returned as UTC, so change it. */ + if (!message_date_parse(sent_value, sent_value_len, + &sent_time, &timezone_offset)) + return 0; + sent_time += timezone_offset * 60; + + switch (type) { + case SEARCH_BEFORE: + return sent_time < search_time ? 1 : 0; + case SEARCH_ON: + return (sent_time >= search_time && + sent_time < search_time + 3600*24) ? 1 : 0; + case SEARCH_SINCE: + return sent_time >= search_time ? 1 : 0; + default: + i_unreached(); + } +} + +static struct message_search_context * +msg_search_arg_context(struct index_search_context *ctx, + struct mail_search_arg *arg) +{ + enum message_search_flags flags = 0; + + if (arg->context == NULL) T_BEGIN { + string_t *dtc = t_str_new(128); + + if (ctx->mail_ctx.normalizer(arg->value.str, + strlen(arg->value.str), dtc) < 0) + i_panic("search key not utf8: %s", arg->value.str); + + if (arg->type == SEARCH_BODY) + flags |= MESSAGE_SEARCH_FLAG_SKIP_HEADERS; + /* we don't get here if arg is "", but dtc can be "" if it + only contains characters that we need to ignore. handle + those searches by returning them as non-matched. */ + if (str_len(dtc) > 0) { + arg->context = + message_search_init(str_c(dtc), + ctx->mail_ctx.normalizer, + flags); + } + } T_END; + return arg->context; +} + +static void compress_lwsp(string_t *dest, const unsigned char *src, + size_t src_len) +{ + size_t i; + bool prev_lwsp = TRUE; + + for (i = 0; i < src_len; i++) { + if (IS_LWSP(src[i])) { + if (!prev_lwsp) { + prev_lwsp = TRUE; + str_append_c(dest, ' '); + } + } else { + prev_lwsp = FALSE; + str_append_c(dest, src[i]); + } + } +} + +static void search_header_arg(struct mail_search_arg *arg, + struct search_header_context *ctx) +{ + struct message_search_context *msg_search_ctx; + struct message_block block; + struct message_header_line hdr; + int ret; + + /* first check that the field name matches to argument. */ + switch (arg->type) { + case SEARCH_BEFORE: + case SEARCH_ON: + case SEARCH_SINCE: + if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_SENT) + return; + + /* date is handled differently than others */ + if (strcasecmp(ctx->hdr->name, "Date") == 0) { + if (ctx->hdr->continues) { + ctx->hdr->use_full_value = TRUE; + return; + } + ret = search_sent(arg->type, arg->value.time, + ctx->hdr->full_value, + ctx->hdr->full_value_len); + ARG_SET_RESULT(arg, ret); + } + return; + + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + ctx->custom_header = TRUE; + + if (strcasecmp(ctx->hdr->name, arg->hdr_field_name) != 0) + return; + break; + default: + return; + } + + if (arg->value.str[0] == '\0') { + /* we're just testing existence of the field. always matches. */ + ARG_SET_RESULT(arg, 1); + return; + } + + if (ctx->hdr->continues) { + ctx->hdr->use_full_value = TRUE; + return; + } + + i_zero(&block); + + /* We're searching only for values, so drop header name and middle + parts. We use header searching so that MIME words will be decoded. */ + hdr = *ctx->hdr; + hdr.name = ""; hdr.name_len = 0; + hdr.middle_len = 0; + block.hdr = &hdr; + + msg_search_ctx = msg_search_arg_context(ctx->index_ctx, arg); + if (msg_search_ctx == NULL) + return; + + if (!ctx->decoded_block_set) { T_BEGIN { + struct message_address *addr; + string_t *str; + + switch (arg->type) { + case SEARCH_HEADER: + /* simple match */ + break; + case SEARCH_HEADER_ADDRESS: + /* we have to match against normalized address */ + addr = message_address_parse(pool_datastack_create(), + ctx->hdr->full_value, + ctx->hdr->full_value_len, + UINT_MAX, + MESSAGE_ADDRESS_PARSE_FLAG_FILL_MISSING); + str = t_str_new(ctx->hdr->value_len); + message_address_write(str, addr); + hdr.value = hdr.full_value = str_data(str); + hdr.value_len = hdr.full_value_len = str_len(str); + break; + case SEARCH_HEADER_COMPRESS_LWSP: + /* convert LWSP to single spaces */ + str = t_str_new(hdr.full_value_len); + compress_lwsp(str, hdr.full_value, hdr.full_value_len); + hdr.value = hdr.full_value = str_data(str); + hdr.value_len = hdr.full_value_len = str_len(str); + break; + default: + i_unreached(); + } + ret = message_search_more_get_decoded(msg_search_ctx, &block, + &ctx->decoded_block) ? 1 : 0; + ctx->decoded_block_set = TRUE; + } T_END; } else { + /* this block was already decoded and saved by an earlier + search arg. use the already-decoded block to avoid + duplicating work. */ + ret = message_search_more_decoded(msg_search_ctx, + &ctx->decoded_block) ? 1 : 0; + } + + /* there may be multiple headers. don't mark this failed yet. */ + if (ret > 0) + ARG_SET_RESULT(arg, 1); +} + +static void search_header_unmatch(struct mail_search_arg *arg, + struct search_header_context *ctx ATTR_UNUSED) +{ + switch (arg->type) { + case SEARCH_BEFORE: + case SEARCH_ON: + case SEARCH_SINCE: + if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_SENT) + break; + + if (arg->match_not) { + /* date header not found, so we match only for + NOT searches */ + ARG_SET_RESULT(arg, 0); + } + break; + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + ARG_SET_RESULT(arg, 0); + break; + default: + break; + } +} + +static void search_header(struct message_header_line *hdr, + struct search_header_context *ctx) +{ + if (ctx->parse_headers) + index_mail_parse_header(NULL, hdr, ctx->imail); + + if (hdr == NULL) { + /* end of headers, mark all unknown SEARCH_HEADERs unmatched */ + (void)mail_search_args_foreach(ctx->args, search_header_unmatch, + ctx); + return; + } + + if (hdr->eoh) + return; + + if (ctx->custom_header || strcasecmp(hdr->name, "Date") == 0) { + ctx->hdr = hdr; + + ctx->decoded_block_set = FALSE; + ctx->custom_header = FALSE; + (void)mail_search_args_foreach(ctx->args, search_header_arg, ctx); + } +} + +static void search_body(struct mail_search_arg *arg, + struct search_body_context *ctx) +{ + struct message_search_context *msg_search_ctx; + const char *error; + int ret; + + switch (arg->type) { + case SEARCH_BODY: + case SEARCH_TEXT: + break; + default: + return; + } + + msg_search_ctx = msg_search_arg_context(ctx->index_ctx, arg); + if (msg_search_ctx == NULL) { + ARG_SET_RESULT(arg, 0); + return; + } + + i_stream_seek(ctx->input, 0); + ret = message_search_msg(msg_search_ctx, ctx->input, ctx->part, &error); + if (ret < 0 && ctx->input->stream_errno == 0) { + /* try again without cached parts */ + index_mail_set_message_parts_corrupted(ctx->index_ctx->cur_mail, error); + + i_stream_seek(ctx->input, 0); + ret = message_search_msg(msg_search_ctx, ctx->input, NULL, &error); + i_assert(ret >= 0 || ctx->input->stream_errno != 0); + } + if (ctx->input->stream_errno != 0) { + mailbox_set_critical(ctx->index_ctx->box, + "read(%s) failed: %s", i_stream_get_name(ctx->input), + i_stream_get_error(ctx->input)); + } + + ARG_SET_RESULT(arg, ret); +} + +static int search_arg_match_text(struct mail_search_arg *args, + struct index_search_context *ctx) +{ + const enum message_header_parser_flags hdr_parser_flags = + MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE; + struct index_mail *imail = INDEX_MAIL(ctx->cur_mail); + struct mail *real_mail; + struct istream *input = NULL; + struct mailbox_header_lookup_ctx *headers_ctx; + struct search_header_context hdr_ctx; + struct search_body_context body_ctx; + const char *const *headers; + bool have_headers, have_body, failed = FALSE; + int ret; + + /* first check what we need to use */ + headers = mail_search_args_analyze(args, &have_headers, &have_body); + if (!have_headers && !have_body) + return -1; + + i_zero(&hdr_ctx); + hdr_ctx.index_ctx = ctx; + /* hdr_ctx.imail is different from imail for mails in + virtual mailboxes */ + if (mail_get_backend_mail(ctx->cur_mail, &real_mail) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + hdr_ctx.imail = INDEX_MAIL(real_mail); + hdr_ctx.custom_header = TRUE; + hdr_ctx.args = args; + + headers_ctx = headers == NULL ? NULL : + mailbox_header_lookup_init(ctx->box, headers); + if (headers != NULL && + (!have_body || + ctx->cur_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)) { + /* try to look up the specified headers from cache */ + i_assert(*headers != NULL); + + if (mail_get_header_stream(ctx->cur_mail, headers_ctx, + &input) < 0) { + search_cur_mail_failed(ctx); + failed = TRUE; + } else { + message_parse_header(input, NULL, hdr_parser_flags, + search_header, &hdr_ctx); + } + input = NULL; + } else if (have_headers) { + /* we need to read the entire header */ + ret = have_body ? + mail_get_stream_because(ctx->cur_mail, NULL, NULL, "search", &input) : + mail_get_hdr_stream_because(ctx->cur_mail, NULL, "search", &input); + if (ret < 0) { + search_cur_mail_failed(ctx); + failed = TRUE; + } else { + /* FIXME: The header parsing here is an optimization to + avoid parsing the header twice: First when checking + whether the search matches, and secondly when + generating wanted fields. However, if we already + know that we want to generate a BODYSTRUCTURE reply, + index_mail_parse_header() must have a non-NULL part + parameter. That's not easily possible at this point + without larger code changes, so for now we'll just + disable this optimization for that case. */ + hdr_ctx.parse_headers = + !hdr_ctx.imail->data.save_bodystructure_header && + index_mail_want_parse_headers(hdr_ctx.imail); + if (hdr_ctx.parse_headers) { + index_mail_parse_header_init(hdr_ctx.imail, + headers_ctx); + } + message_parse_header(input, NULL, hdr_parser_flags, + search_header, &hdr_ctx); + if (input->stream_errno != 0) { + mailbox_set_critical(ctx->box, + "read(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + failed = TRUE; + search_set_failed(ctx); + } + } + } + mailbox_header_lookup_unref(&headers_ctx); + + if (failed) { + /* opening mail failed. maybe because of lookup_abort. + update access_parts for prefetching */ + if (have_body) + imail->data.access_part |= READ_HDR | READ_BODY; + else + imail->data.access_part |= READ_HDR; + return -1; + } + + if (have_headers) { + /* see if the header search succeeded in finishing the search */ + ret = mail_search_args_foreach(args, search_none, NULL); + if (ret >= 0 || !have_body) + return ret; + } + + i_assert(have_body); + + if (ctx->cur_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) { + imail->data.access_part |= READ_HDR | READ_BODY; + return -1; + } + + if (input == NULL) { + /* we didn't search headers. */ + struct message_size hdr_size; + + if (mail_get_stream_because(ctx->cur_mail, &hdr_size, NULL, "search", &input) < 0) { + search_cur_mail_failed(ctx); + return -1; + } + i_stream_seek(input, hdr_size.physical_size); + } + + i_zero(&body_ctx); + body_ctx.index_ctx = ctx; + body_ctx.input = input; + /* Get parts if they already exist in cache. If they don't, + message-search will parse the mail automatically. */ + ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + (void)mail_get_parts(ctx->cur_mail, &body_ctx.part); + ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER; + + return mail_search_args_foreach(args, search_body, &body_ctx); +} + +static bool +search_msgset_fix_limits(unsigned int messages_count, + ARRAY_TYPE(seq_range) *seqset, bool match_not) +{ + struct seq_range *range; + unsigned int count; + + i_assert(messages_count > 0); + + range = array_get_modifiable(seqset, &count); + if (count > 0) { + i_assert(range[0].seq1 != 0); + if (range[count-1].seq2 == (uint32_t)-1) { + /* "*" used, make sure the last message is in the range + (e.g. with count+1:* we still want to include it) */ + seq_range_array_add(seqset, messages_count); + } + /* remove all nonexistent messages */ + seq_range_array_remove_range(seqset, messages_count + 1, + (uint32_t)-1); + } + if (!match_not) + return array_count(seqset) > 0; + else { + /* if all messages are in the range, it can't match */ + range = array_get_modifiable(seqset, &count); + return count == 0 || range[0].seq1 != 1 || + range[count-1].seq2 != messages_count; + } +} + +static void +search_msgset_fix(unsigned int messages_count, + ARRAY_TYPE(seq_range) *seqset, + uint32_t *seq1_r, uint32_t *seq2_r, bool match_not) +{ + const struct seq_range *range; + unsigned int count; + uint32_t min_seq, max_seq; + + if (!search_msgset_fix_limits(messages_count, seqset, match_not)) { + *seq1_r = (uint32_t)-1; + *seq2_r = 0; + return; + } + + range = array_get(seqset, &count); + if (!match_not) { + min_seq = range[0].seq1; + max_seq = range[count-1].seq2; + } else if (count == 0) { + /* matches all messages */ + min_seq = 1; + max_seq = messages_count; + } else { + min_seq = range[0].seq1 > 1 ? 1 : range[0].seq2 + 1; + max_seq = range[count-1].seq2 < messages_count ? + messages_count : range[count-1].seq1 - 1; + if (min_seq > max_seq) { + *seq1_r = (uint32_t)-1; + *seq2_r = 0; + return; + } + } + + if (*seq1_r < min_seq || *seq1_r == 0) + *seq1_r = min_seq; + if (*seq2_r > max_seq) + *seq2_r = max_seq; +} + +static void search_or_parse_msgset_args(unsigned int messages_count, + struct mail_search_arg *args, + uint32_t *seq1_r, uint32_t *seq2_r) +{ + uint32_t seq1, seq2, min_seq1 = 0, max_seq2 = 0; + + for (; args != NULL; args = args->next) { + seq1 = 1; seq2 = messages_count; + + switch (args->type) { + case SEARCH_SUB: + i_assert(!args->match_not); + search_parse_msgset_args(messages_count, + args->value.subargs, + &seq1, &seq2); + break; + case SEARCH_OR: + i_assert(!args->match_not); + search_or_parse_msgset_args(messages_count, + args->value.subargs, + &seq1, &seq2); + break; + case SEARCH_SEQSET: + search_msgset_fix(messages_count, &args->value.seqset, + &seq1, &seq2, args->match_not); + break; + default: + break; + } + + if (min_seq1 == 0) { + min_seq1 = seq1; + max_seq2 = seq2; + } else { + if (seq1 < min_seq1) + min_seq1 = seq1; + if (seq2 > max_seq2) + max_seq2 = seq2; + } + } + i_assert(min_seq1 != 0); + + if (min_seq1 > *seq1_r) + *seq1_r = min_seq1; + if (max_seq2 < *seq2_r) + *seq2_r = max_seq2; +} + +static void search_parse_msgset_args(unsigned int messages_count, + struct mail_search_arg *args, + uint32_t *seq1_r, uint32_t *seq2_r) +{ + for (; args != NULL; args = args->next) { + switch (args->type) { + case SEARCH_SUB: + i_assert(!args->match_not); + search_parse_msgset_args(messages_count, + args->value.subargs, + seq1_r, seq2_r); + break; + case SEARCH_OR: + /* go through our children and use the widest seqset + range */ + i_assert(!args->match_not); + search_or_parse_msgset_args(messages_count, + args->value.subargs, + seq1_r, seq2_r); + break; + case SEARCH_SEQSET: + search_msgset_fix(messages_count, &args->value.seqset, + seq1_r, seq2_r, args->match_not); + break; + default: + break; + } + } +} + +static void search_limit_lowwater(struct index_search_context *ctx, + uint32_t uid_lowwater, uint32_t *first_seq) +{ + uint32_t seq1, seq2; + + if (uid_lowwater == 0) + return; + + (void)mail_index_lookup_seq_range(ctx->view, uid_lowwater, (uint32_t)-1, + &seq1, &seq2); + if (*first_seq < seq1) + *first_seq = seq1; +} + +static bool search_limit_by_hdr(struct index_search_context *ctx, + struct mail_search_arg *args, + uint32_t *seq1, uint32_t *seq2) +{ + const struct mail_index_header *hdr; + enum mail_flags pvt_flags_mask; + uint64_t highest_modseq; + + hdr = mail_index_get_header(ctx->view); + /* we can't trust that private view's header is fully up to date, + so do this optimization only for non-private flags */ + pvt_flags_mask = ctx->box->view_pvt == NULL ? 0 : + mailbox_get_private_flags_mask(ctx->box); + + for (; args != NULL; args = args->next) { + switch (args->type) { + case SEARCH_ALL: + if (args->match_not) { + /* NOT ALL - pointless noop query */ + return FALSE; + } + continue; + case SEARCH_MODSEQ: + /* MODSEQ higher than current HIGHESTMODSEQ? */ + highest_modseq = mail_index_modseq_get_highest(ctx->view); + if (args->value.modseq->modseq > highest_modseq) + return FALSE; + continue; + default: + continue; + case SEARCH_FLAGS: + break; + } + if ((args->value.flags & MAIL_SEEN) != 0 && + (pvt_flags_mask & MAIL_SEEN) == 0) { + /* SEEN with 0 seen? */ + if (!args->match_not && hdr->seen_messages_count == 0) + return FALSE; + + if (hdr->seen_messages_count == hdr->messages_count) { + /* UNSEEN with all seen? */ + if (args->match_not) + return FALSE; + } else if (args->match_not) { + /* UNSEEN with lowwater limiting */ + search_limit_lowwater(ctx, + hdr->first_unseen_uid_lowwater, seq1); + } + } + if ((args->value.flags & MAIL_DELETED) != 0 && + (pvt_flags_mask & MAIL_DELETED) == 0) { + /* DELETED with 0 deleted? */ + if (!args->match_not && + hdr->deleted_messages_count == 0) + return FALSE; + + if (hdr->deleted_messages_count == hdr->messages_count) { + /* UNDELETED with all deleted? */ + if (args->match_not) + return FALSE; + } else if (!args->match_not) { + /* DELETED with lowwater limiting */ + search_limit_lowwater(ctx, + hdr->first_deleted_uid_lowwater, seq1); + } + } + } + + return *seq1 <= *seq2; +} + +static void search_get_seqset(struct index_search_context *ctx, + unsigned int messages_count, + struct mail_search_arg *args) +{ + if (messages_count == 0) { + /* no messages, don't check sequence ranges. although we could + give error message then for FETCH, we shouldn't do it for + UID FETCH. */ + ctx->seq1 = 1; + ctx->seq2 = 0; + return; + } + + ctx->seq1 = 1; + ctx->seq2 = messages_count; + + search_parse_msgset_args(messages_count, args, &ctx->seq1, &ctx->seq2); + if (ctx->seq1 == 0) { + ctx->seq1 = 1; + ctx->seq2 = messages_count; + } + if (ctx->seq1 > ctx->seq2) { + /* no matches */ + return; + } + + /* See if this search query can never match based on data in index's + header. We'll scan only the root level args, which is usually + enough. */ + if (!search_limit_by_hdr(ctx, args, &ctx->seq1, &ctx->seq2)) { + /* no matches */ + ctx->seq1 = 1; + ctx->seq2 = 0; + } +} + +static int search_build_subthread(struct mail_thread_iterate_context *iter, + ARRAY_TYPE(seq_range) *uids) +{ + struct mail_thread_iterate_context *child_iter; + const struct mail_thread_child_node *node; + int ret = 0; + + while ((node = mail_thread_iterate_next(iter, &child_iter)) != NULL) { + if (child_iter != NULL) { + if (search_build_subthread(child_iter, uids) < 0) + ret = -1; + } + seq_range_array_add(uids, node->uid); + } + if (mail_thread_iterate_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int search_build_inthread_result(struct index_search_context *ctx, + struct mail_search_arg *arg) +{ + struct mail_thread_iterate_context *iter, *child_iter; + const struct mail_thread_child_node *node; + const ARRAY_TYPE(seq_range) *search_uids; + ARRAY_TYPE(seq_range) thread_uids; + int ret = 0; + + /* mail_search_args_init() must have been called by now */ + i_assert(arg->initialized.search_args != NULL); + + p_array_init(&arg->value.seqset, ctx->mail_ctx.args->pool, 64); + if (mailbox_search_result_build(ctx->mail_ctx.transaction, + arg->initialized.search_args, + MAILBOX_SEARCH_RESULT_FLAG_UPDATE | + MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC, + &arg->value.search_result) < 0) + return -1; + if (ctx->thread_ctx == NULL) { + /* failed earlier */ + return -1; + } + + search_uids = mailbox_search_result_get(arg->value.search_result); + if (array_count(search_uids) == 0) { + /* search found nothing - no threads can match */ + return 0; + } + + t_array_init(&thread_uids, 128); + iter = mail_thread_iterate_init(ctx->thread_ctx, + arg->value.thread_type, FALSE); + while ((node = mail_thread_iterate_next(iter, &child_iter)) != NULL) { + seq_range_array_add(&thread_uids, node->uid); + if (child_iter != NULL) { + if (search_build_subthread(child_iter, + &thread_uids) < 0) + ret = -1; + } + if (seq_range_array_have_common(&thread_uids, search_uids)) { + /* yes, we want this thread */ + seq_range_array_merge(&arg->value.seqset, &thread_uids); + } + array_clear(&thread_uids); + } + if (mail_thread_iterate_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int search_build_inthreads(struct index_search_context *ctx, + struct mail_search_arg *arg) +{ + int ret = 0; + + for (; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_OR: + case SEARCH_SUB: + if (search_build_inthreads(ctx, arg->value.subargs) < 0) + ret = -1; + break; + case SEARCH_INTHREAD: + if (search_build_inthread_result(ctx, arg) < 0) + ret = -1; + break; + default: + break; + } + } + return ret; +} + +static void +wanted_sort_fields_get(struct mailbox *box, + const enum mail_sort_type *sort_program, + struct mailbox_header_lookup_ctx *wanted_headers, + enum mail_fetch_field *wanted_fields_r, + struct mailbox_header_lookup_ctx **headers_ctx_r) +{ + ARRAY_TYPE(const_string) headers; + const char *header; + unsigned int i; + + *wanted_fields_r = 0; + *headers_ctx_r = NULL; + + t_array_init(&headers, 8); + for (i = 0; sort_program[i] != MAIL_SORT_END; i++) { + header = NULL; + + switch (sort_program[i] & MAIL_SORT_MASK) { + case MAIL_SORT_ARRIVAL: + *wanted_fields_r |= MAIL_FETCH_RECEIVED_DATE; + break; + case MAIL_SORT_CC: + header = "Cc"; + break; + case MAIL_SORT_DATE: + *wanted_fields_r |= MAIL_FETCH_DATE; + break; + case MAIL_SORT_FROM: + header = "From"; + break; + case MAIL_SORT_SIZE: + *wanted_fields_r |= MAIL_FETCH_VIRTUAL_SIZE; + break; + case MAIL_SORT_SUBJECT: + header = "Subject"; + break; + case MAIL_SORT_TO: + header = "To"; + break; + } + if (header != NULL) + array_push_back(&headers, &header); + } + + if (wanted_headers != NULL) { + for (i = 0; wanted_headers->name[i] != NULL; i++) + array_push_back(&headers, &wanted_headers->name[i]); + } + + if (array_count(&headers) > 0) { + array_append_zero(&headers); + *headers_ctx_r = mailbox_header_lookup_init(box, + array_front(&headers)); + } +} + +struct mail_search_context * +index_storage_search_init(struct mailbox_transaction_context *t, + struct mail_search_args *args, + const enum mail_sort_type *sort_program, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct index_search_context *ctx; + struct mailbox_status status; + + ctx = i_new(struct index_search_context, 1); + ctx->mail_ctx.transaction = t; + ctx->mail_ctx.normalizer = t->box->storage->user->default_normalizer; + ctx->box = t->box; + ctx->view = t->view; + ctx->mail_ctx.args = args; + ctx->mail_ctx.sort_program = index_sort_program_init(t, sort_program); + + ctx->mail_ctx.max_mails = t->box->storage->set->mail_prefetch_count + 1; + if (ctx->mail_ctx.max_mails == 0) + ctx->mail_ctx.max_mails = UINT_MAX; + ctx->next_time_check_cost = SEARCH_INITIAL_MAX_COST; + i_gettimeofday(&ctx->last_nonblock_timeval); + + mailbox_get_open_status(t->box, STATUS_MESSAGES, &status); + ctx->mail_ctx.progress_max = status.messages; + + i_array_init(&ctx->mail_ctx.results, 5); + array_create(&ctx->mail_ctx.module_contexts, default_pool, + sizeof(void *), 5); + i_array_init(&ctx->mail_ctx.mails, ctx->mail_ctx.max_mails); + + mail_search_args_reset(ctx->mail_ctx.args->args, TRUE); + if (args->have_inthreads) { + if (mail_thread_init(t->box, NULL, &ctx->thread_ctx) < 0) + search_set_failed(ctx); + if (search_build_inthreads(ctx, args->args) < 0) + search_set_failed(ctx); + } + + if (sort_program != NULL) { + wanted_sort_fields_get(ctx->box, sort_program, wanted_headers, + &ctx->mail_ctx.wanted_fields, + &ctx->mail_ctx.wanted_headers); + } else if (wanted_headers != NULL) { + ctx->mail_ctx.wanted_headers = wanted_headers; + mailbox_header_lookup_ref(wanted_headers); + } + ctx->mail_ctx.wanted_fields |= wanted_fields; + + search_get_seqset(ctx, status.messages, args->args); + (void)mail_search_args_foreach(args->args, search_init_arg, ctx); + + /* Need to reset results for match_always cases */ + mail_search_args_reset(ctx->mail_ctx.args->args, FALSE); + return &ctx->mail_ctx; +} + +static void ATTR_NULL(2) +search_arg_deinit(struct mail_search_arg *arg, + struct index_search_context *ctx) +{ + switch (arg->type) { + case SEARCH_MIMEPART: + index_search_mime_arg_deinit(arg, ctx); + break; + default: + if (arg->context != NULL) { + struct message_search_context *search_ctx = arg->context; + message_search_deinit(&search_ctx); + arg->context = NULL; + } + } +} + +int index_storage_search_deinit(struct mail_search_context *_ctx) +{ + struct index_search_context *ctx = (struct index_search_context *)_ctx; + struct mail *mail; + int ret; + + ret = ctx->failed ? -1 : 0; + + mail_search_args_reset(ctx->mail_ctx.args->args, FALSE); + (void)mail_search_args_foreach(ctx->mail_ctx.args->args, + search_arg_deinit, ctx); + + mailbox_header_lookup_unref(&ctx->mail_ctx.wanted_headers); + if (ctx->mail_ctx.sort_program != NULL) { + if (index_sort_program_deinit(&ctx->mail_ctx.sort_program) < 0) + ret = -1; + } + if (ctx->thread_ctx != NULL) + mail_thread_deinit(&ctx->thread_ctx); + array_free(&ctx->mail_ctx.results); + array_free(&ctx->mail_ctx.module_contexts); + + array_foreach_elem(&ctx->mail_ctx.mails, mail) { + struct index_mail *imail = INDEX_MAIL(mail); + + imail->mail.search_mail = FALSE; + mail_free(&mail); + } + + if (ctx->failed) + mail_storage_last_error_pop(ctx->box->storage); + array_free(&ctx->mail_ctx.mails); + i_free(ctx); + return ret; +} + +static unsigned long long +search_get_cost(struct mailbox_transaction_context *trans) +{ + return trans->stats.open_lookup_count * SEARCH_COST_DENTRY + + trans->stats.stat_lookup_count * SEARCH_COST_DENTRY + + trans->stats.fstat_lookup_count * SEARCH_COST_ATTR + + trans->stats.cache_hit_count * SEARCH_COST_CACHE + + trans->stats.files_read_count * SEARCH_COST_FILES_READ + + (trans->stats.files_read_bytes/1024) * SEARCH_COST_KBYTE; +} + +static int search_match_once(struct index_search_context *ctx) +{ + int ret; + + ret = mail_search_args_foreach(ctx->mail_ctx.args->args, + search_cached_arg, ctx); + if (ret < 0) + ret = search_arg_match_text(ctx->mail_ctx.args->args, ctx); + if (ret < 0) + ret = index_search_mime_arg_match(ctx->mail_ctx.args->args, ctx); + return ret; +} + +static bool search_arg_is_static(struct mail_search_arg *arg) +{ + struct mail_search_arg *subarg; + + switch (arg->type) { + case SEARCH_OR: + case SEARCH_SUB: + /* they're static only if all subargs are static */ + subarg = arg->value.subargs; + for (; subarg != NULL; subarg = subarg->next) { + if (!search_arg_is_static(subarg)) + return FALSE; + } + return TRUE; + case SEARCH_SEQSET: + /* changes between syncs, but we can't really handle this + currently. seqsets should be converted to uidsets first. */ + case SEARCH_FLAGS: + case SEARCH_KEYWORDS: + case SEARCH_MODSEQ: + case SEARCH_INTHREAD: + break; + case SEARCH_ALL: + case SEARCH_UIDSET: + case SEARCH_BEFORE: + case SEARCH_ON: + case SEARCH_SINCE: + case SEARCH_SMALLER: + case SEARCH_LARGER: + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + case SEARCH_BODY: + case SEARCH_TEXT: + case SEARCH_SAVEDATESUPPORTED: + case SEARCH_GUID: + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GUID: + case SEARCH_MAILBOX_GLOB: + case SEARCH_REAL_UID: + case SEARCH_MIMEPART: + case SEARCH_NIL: + return TRUE; + } + return FALSE; +} + +static void search_set_static_matches(struct mail_search_arg *arg) +{ + for (; arg != NULL; arg = arg->next) { + if (search_arg_is_static(arg)) + arg->result = 1; + } +} + +static bool search_has_static_nonmatches(struct mail_search_arg *arg) +{ + for (; arg != NULL; arg = arg->next) { + if (arg->result == 0 && search_arg_is_static(arg)) + return TRUE; + } + return FALSE; +} + +static void search_match_finish(struct index_search_context *ctx, int match) +{ + if (match == 0 && + search_has_static_nonmatches(ctx->mail_ctx.args->args)) { + /* if there are saved search results remember + that this message never matches */ + mailbox_search_results_never(&ctx->mail_ctx, + ctx->cur_mail->uid); + } +} + +static int search_match_next(struct index_search_context *ctx) +{ + static enum mail_lookup_abort cache_lookups[] = { + MAIL_LOOKUP_ABORT_NOT_IN_CACHE, + MAIL_LOOKUP_ABORT_READ_MAIL, + MAIL_LOOKUP_ABORT_NEVER + }; + unsigned int i, n = N_ELEMENTS(cache_lookups); + int ret = -1; + + if (ctx->have_mailbox_args) { + /* check that the mailbox name matches. + this makes sense only with virtual mailboxes. */ + ret = mail_search_args_foreach(ctx->mail_ctx.args->args, + search_mailbox_arg, ctx); + } + + /* avoid doing extra work for as long as possible */ + if (ctx->mail_ctx.max_mails > 1) { + /* we're doing prefetching. if we have to read the mail, + do a prefetch first and the final search later */ + n--; + } + + i_assert(ctx->cur_mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER); + for (i = 0; i < n && ret < 0; i++) { + ctx->cur_mail->lookup_abort = cache_lookups[i]; + T_BEGIN { + ret = search_match_once(ctx); + } T_END; + } + ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER; + search_match_finish(ctx, ret); + return ret; +} + +static void index_storage_search_notify(struct mailbox *box, + struct index_search_context *ctx) +{ + float percentage; + unsigned int msecs, secs; + + if (ctx->last_notify.tv_sec == 0) { + /* set the search time in here, in case a plugin + already spent some time indexing the mailbox */ + ctx->search_start_time = ioloop_timeval; + } else if (box->storage->callbacks.notify_ok != NULL && + !ctx->mail_ctx.progress_hidden) { + percentage = ctx->mail_ctx.progress_cur * 100.0 / + ctx->mail_ctx.progress_max; + msecs = timeval_diff_msecs(&ioloop_timeval, + &ctx->search_start_time); + secs = (msecs / (percentage / 100.0) - msecs) / 1000; + + T_BEGIN { + const char *text; + + text = t_strdup_printf("Searched %d%% of the mailbox, " + "ETA %d:%02d", (int)percentage, + secs/60, secs%60); + box->storage->callbacks. + notify_ok(box, text, + box->storage->callback_context); + } T_END; + } + ctx->last_notify = ioloop_timeval; +} + +static bool search_would_block(struct index_search_context *ctx) +{ + struct timeval now; + unsigned long long guess_cost; + long long usecs; + bool ret; + + if (ctx->cost < ctx->next_time_check_cost) + return FALSE; + + i_gettimeofday(&now); + + usecs = timeval_diff_usecs(&now, &ctx->last_nonblock_timeval); + if (usecs < 0) { + /* clock moved backwards. */ + ctx->last_nonblock_timeval = now; + ctx->next_time_check_cost = SEARCH_INITIAL_MAX_COST; + return TRUE; + } else if (usecs < SEARCH_MIN_NONBLOCK_USECS) { + /* not finished yet. estimate the next time lookup */ + ret = FALSE; + } else { + /* done, or close enough anyway */ + ctx->last_nonblock_timeval = now; + ret = TRUE; + } + guess_cost = ctx->cost * + (SEARCH_MAX_NONBLOCK_USECS / (double)usecs); + if (usecs < SEARCH_RECALC_MIN_USECS) { + /* the estimate may not be very good since we spent + so little time doing this search. don't allow huge changes + to the guess, but allow anyway large enough so that we can + move to right direction. */ + if (guess_cost > ctx->next_time_check_cost*3) + guess_cost = ctx->next_time_check_cost*3; + else if (guess_cost < ctx->next_time_check_cost/3) + guess_cost = ctx->next_time_check_cost/3; + } + if (ret) + ctx->cost = 0; + ctx->next_time_check_cost = guess_cost; + return ret; +} + +int index_storage_search_next_match_mail(struct mail_search_context *_ctx, + struct mail *mail) +{ + struct index_search_context *ctx = + container_of(_ctx, struct index_search_context, mail_ctx); + struct index_mail *imail = INDEX_MAIL(mail); + int match; + + ctx->cur_mail = mail; + /* mail's access_type is SEARCH only while using it to process + the search query. afterwards the mail can still be accessed + for fetching. */ + ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_SEARCH; + T_BEGIN { + match = search_match_next(ctx); + } T_END; + ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_DEFAULT; + ctx->cur_mail = NULL; + + i_assert(imail->data.search_results == NULL); + if (match < 0) { + /* result isn't known yet, do a prefetch and + finish later */ + imail->data.search_results = + buffer_create_dynamic(imail->mail.data_pool, 64); + mail_search_args_result_serialize(_ctx->args, + imail->data.search_results); + } + + mail_search_args_reset(_ctx->args->args, FALSE); + + if (match != 0) { + /* either matched or result is still unknown. + anyway we're far enough now that we probably want + to update the access_parts. the only problem here is + if searching would want fewer access_parts than the + fetching part, but that's probably not a big problem + usually. */ + index_mail_update_access_parts_pre(mail); + return 1; + } + + /* non-match */ + if (_ctx->args->stop_on_nonmatch) + return -1; + return 0; +} + +static int search_more_with_mail(struct index_search_context *ctx, + struct mail *mail) +{ + struct mail_search_context *_ctx = &ctx->mail_ctx; + struct mailbox *box = _ctx->transaction->box; + unsigned long long cost1, cost2; + int ret; + + if (search_would_block(ctx)) { + /* this lookup is useful when a large number of + messages match */ + return 0; + } + + if (ioloop_time - ctx->last_notify.tv_sec >= + SEARCH_NOTIFY_INTERVAL_SECS) + index_storage_search_notify(box, ctx); + + mail_search_args_reset(_ctx->args->args, FALSE); + + cost1 = search_get_cost(mail->transaction); + ret = -1; + while (box->v.search_next_update_seq(_ctx)) { + mail_set_seq(mail, _ctx->seq); + + ret = box->v.search_next_match_mail(_ctx, mail); + if (ret != 0) + break; + + cost2 = search_get_cost(mail->transaction); + ctx->cost += cost2 - cost1; + cost1 = cost2; + + if (search_would_block(ctx)) + break; + ret = -1; + } + cost2 = search_get_cost(mail->transaction); + ctx->cost += cost2 - cost1; + return ret; +} + +struct mail *index_search_get_mail(struct index_search_context *ctx) +{ + struct index_mail *imail; + struct mail *const *mails, *mail; + unsigned int count; + + if (ctx->mail_ctx.unused_mail_idx == ctx->mail_ctx.max_mails) + return NULL; + + mails = array_get(&ctx->mail_ctx.mails, &count); + if (ctx->mail_ctx.unused_mail_idx < count) + return mails[ctx->mail_ctx.unused_mail_idx]; + + mail = mail_alloc(ctx->mail_ctx.transaction, + ctx->mail_ctx.wanted_fields, + ctx->mail_ctx.wanted_headers); + imail = INDEX_MAIL(mail); + imail->mail.search_mail = TRUE; + ctx->mail_ctx.transaction->stats_track = TRUE; + + array_push_back(&ctx->mail_ctx.mails, &mail); + return mail; +} + +static int search_more_with_prefetching(struct index_search_context *ctx, + struct mail **mail_r) +{ + struct mail *mail, *const *mails; + unsigned int count; + int ret = 0; + + while ((mail = index_search_get_mail(ctx)) != NULL) { + T_BEGIN { + ret = search_more_with_mail(ctx, mail); + } T_END; + if (ret <= 0) + break; + + if (ctx->mail_ctx.sort_program != NULL) { + /* don't prefetch when using a sort program, + since the mails' access order will change */ + i_assert(ctx->mail_ctx.unused_mail_idx == 0); + *mail_r = mail; + return 1; + } + if (mail_prefetch(mail) && ctx->mail_ctx.unused_mail_idx == 0) { + /* no prefetching done, return it immediately */ + *mail_r = mail; + return 1; + } + ctx->mail_ctx.unused_mail_idx++; + } + + if (mail != NULL) { + if (ret == 0) { + /* wait */ + return 0; + } + i_assert(ret < 0); + if (ctx->mail_ctx.unused_mail_idx == 0) { + /* finished */ + return -1; + } + } else { + /* prefetch buffer is full. */ + } + + /* return the next message */ + i_assert(ctx->mail_ctx.unused_mail_idx > 0); + + mails = array_get(&ctx->mail_ctx.mails, &count); + *mail_r = mails[0]; + if (--ctx->mail_ctx.unused_mail_idx > 0) { + array_pop_front(&ctx->mail_ctx.mails); + array_push_back(&ctx->mail_ctx.mails, mail_r); + } + index_mail_update_access_parts_post(*mail_r); + return 1; +} + +static bool search_finish_prefetch(struct index_search_context *ctx, + struct index_mail *imail) +{ + int ret; + + i_assert(imail->mail.mail.lookup_abort == MAIL_LOOKUP_ABORT_NEVER); + + ctx->cur_mail = &imail->mail.mail; + ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_SEARCH; + mail_search_args_result_deserialize(ctx->mail_ctx.args, + imail->data.search_results->data, + imail->data.search_results->used); + T_BEGIN { + ret = search_match_once(ctx); + search_match_finish(ctx, ret); + } T_END; + ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_DEFAULT; + ctx->cur_mail = NULL; + return ret > 0; +} + +static int search_more(struct index_search_context *ctx, + struct mail **mail_r) +{ + struct index_mail *imail; + int ret; + + while ((ret = search_more_with_prefetching(ctx, mail_r)) > 0) { + imail = INDEX_MAIL(*mail_r); + if (imail->data.search_results == NULL) + break; + + /* prefetch running - searching wasn't finished yet */ + if (search_finish_prefetch(ctx, imail)) + break; + /* search finished as non-match */ + if (ctx->mail_ctx.args->stop_on_nonmatch) { + ret = -1; + break; + } + } + return ret; +} + +bool index_storage_search_next_nonblock(struct mail_search_context *_ctx, + struct mail **mail_r, bool *tryagain_r) +{ + struct index_search_context *ctx = (struct index_search_context *)_ctx; + struct mail *mail, *const *mailp; + uint32_t seq; + int ret; + + *tryagain_r = FALSE; + + if (_ctx->sort_program == NULL) { + ret = search_more(ctx, &mail); + if (ret == 0) { + *tryagain_r = TRUE; + return FALSE; + } + if (ret < 0) + return FALSE; + *mail_r = mail; + return TRUE; + } + + if (!ctx->sorted) { + while ((ret = search_more(ctx, &mail)) > 0) + index_sort_list_add(_ctx->sort_program, mail); + + if (ret == 0) { + *tryagain_r = TRUE; + return FALSE; + } + /* finished searching the messages. now sort them and start + returning the messages. */ + ctx->sorted = TRUE; + index_sort_list_finish(_ctx->sort_program); + } + + /* everything searched at this point already. just returning + matches from sort list. FIXME: we could do prefetching here also. */ + if (!index_sort_list_next(_ctx->sort_program, &seq)) + return FALSE; + + mailp = array_front(&ctx->mail_ctx.mails); + mail_set_seq(*mailp, seq); + index_mail_update_access_parts_pre(*mailp); + index_mail_update_access_parts_post(*mailp); + *mail_r = *mailp; + return TRUE; +} + +bool index_storage_search_next_update_seq(struct mail_search_context *_ctx) +{ + struct index_search_context *ctx = (struct index_search_context *)_ctx; + uint32_t uid; + int ret; + + if (_ctx->seq == 0) { + /* first time */ + _ctx->seq = ctx->seq1; + } else { + _ctx->seq++; + } + + if (!ctx->have_seqsets && !ctx->have_index_args && + !ctx->have_nonmatch_always && _ctx->update_result == NULL) { + _ctx->progress_cur = _ctx->seq; + return _ctx->seq <= ctx->seq2; + } + + ret = 0; + while (_ctx->seq <= ctx->seq2) { + /* check if the sequence matches */ + ret = mail_search_args_foreach(ctx->mail_ctx.args->args, + search_seqset_arg, ctx); + if (ret != 0 && ctx->have_index_args) { + /* check if flags/keywords match before anything else + is done. mail_set_seq() can be a bit slow. */ + ret = mail_search_args_foreach(ctx->mail_ctx.args->args, + search_index_arg, ctx); + } + if (ret != 0 && _ctx->update_result != NULL) { + /* see if this message never matches */ + mail_index_lookup_uid(ctx->view, _ctx->seq, &uid); + if (seq_range_exists(&_ctx->update_result->never_uids, + uid)) + ret = 0; + } + if (ret != 0) + break; + + /* doesn't, try next one */ + _ctx->seq++; + mail_search_args_reset(ctx->mail_ctx.args->args, FALSE); + } + + if (ret != 0 && _ctx->update_result != NULL) { + mail_index_lookup_uid(ctx->view, _ctx->seq, &uid); + if (seq_range_exists(&_ctx->update_result->uids, uid)) { + /* we already know that the static data + matches. mark it as such. */ + search_set_static_matches(_ctx->args->args); + } + } + ctx->mail_ctx.progress_cur = _ctx->seq; + return ret != 0; +} diff --git a/src/lib-storage/index/index-sort-private.h b/src/lib-storage/index/index-sort-private.h new file mode 100644 index 0000000..3dd0772 --- /dev/null +++ b/src/lib-storage/index/index-sort-private.h @@ -0,0 +1,35 @@ +#ifndef INDEX_SORT_PRIVATE_H +#define INDEX_SORT_PRIVATE_H + +#include "index-sort.h" + +struct mail_search_sort_program { + struct mailbox_transaction_context *t; + enum mail_sort_type sort_program[MAX_SORT_PROGRAM_SIZE]; + struct mail *temp_mail; + unsigned int slow_mails_left; + + void (*sort_list_add)(struct mail_search_sort_program *program, + struct mail *mail); + void (*sort_list_finish)(struct mail_search_sort_program *program); + void *context; + + ARRAY_TYPE(uint32_t) seqs; + unsigned int iter_idx; + + bool failed; +}; + +/* Returns 1 on success, 0 if mail is already expunged, -1 on other errors. */ +int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq, + enum mail_sort_type sort_type, string_t *dest); +int index_sort_node_cmp_type(struct mail_search_sort_program *program, + const enum mail_sort_type *sort_program, + uint32_t seq1, uint32_t seq2); + +void index_sort_list_init_string(struct mail_search_sort_program *program); +void index_sort_list_add_string(struct mail_search_sort_program *program, + struct mail *mail); +void index_sort_list_finish_string(struct mail_search_sort_program *program); + +#endif diff --git a/src/lib-storage/index/index-sort-string.c b/src/lib-storage/index/index-sort-string.c new file mode 100644 index 0000000..c518e78 --- /dev/null +++ b/src/lib-storage/index/index-sort-string.c @@ -0,0 +1,944 @@ +/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */ + +/* The idea is that we use 32bit integers for string sort IDs which specify + the sort order for primary sort condition. The whole 32bit integer space is + used and whenever adding a string, the available space is halved and the new + ID is added in the middle. For example if we add one mail the first time, it + gets ID 2^31. If we then add two mails which are sorted before the first + one, they get IDs 2^31/3 and 2^31/3*2. Once we run out of the available + space between IDs, more space is made by renumbering some IDs. +*/ +#include "lib.h" +#include "array.h" +#include "str.h" +#include "index-storage.h" +#include "index-sort-private.h" + + +struct mail_sort_node { + uint32_t seq:29; + bool wanted:1; + bool no_update:1; + bool sort_id_changed:1; + uint32_t sort_id; +}; +ARRAY_DEFINE_TYPE(mail_sort_node, struct mail_sort_node); + +struct sort_string_context { + struct mail_search_sort_program *program; + const char *primary_sort_name; + + ARRAY_TYPE(mail_sort_node) zero_nodes, nonzero_nodes, sorted_nodes; + const char **sort_strings; + pool_t sort_string_pool; + unsigned int first_missing_sort_id_idx; + + uint32_t ext_id, last_seq, highest_reset_id, prev_seq; + uint32_t lowest_nonexpunged_zero; + + bool regetting:1; + bool have_all_wanted:1; + bool no_writing:1; + bool reverse:1; + bool seqs_nonsorted:1; + bool broken:1; + bool failed:1; +}; + +static struct sort_string_context *static_zero_cmp_context; + +static void index_sort_list_reset_broken(struct sort_string_context *ctx, + const char *reason); +static void index_sort_node_add(struct sort_string_context *ctx, + struct mail_sort_node *node); + +void index_sort_list_init_string(struct mail_search_sort_program *program) +{ + struct sort_string_context *ctx; + const char *name; + + switch (program->sort_program[0] & MAIL_SORT_MASK) { + case MAIL_SORT_CC: + name = "sort-c"; + break; + case MAIL_SORT_FROM: + name = "sort-f"; + break; + case MAIL_SORT_SUBJECT: + name = "sort-s"; + break; + case MAIL_SORT_TO: + name = "sort-t"; + break; + case MAIL_SORT_DISPLAYFROM: + name = "sort-df"; + break; + case MAIL_SORT_DISPLAYTO: + name = "sort-dt"; + break; + default: + i_unreached(); + } + + program->context = ctx = i_new(struct sort_string_context, 1); + ctx->reverse = (program->sort_program[0] & MAIL_SORT_FLAG_REVERSE) != 0; + ctx->program = program; + ctx->primary_sort_name = name; + ctx->ext_id = mail_index_ext_register(program->t->box->index, name, 0, + sizeof(uint32_t), + sizeof(uint32_t)); + i_array_init(&ctx->zero_nodes, 128); + i_array_init(&ctx->nonzero_nodes, 128); +} + +static int sort_node_seq_cmp(const struct mail_sort_node *n1, + const struct mail_sort_node *n2) +{ + if (n1->seq < n2->seq) + return -1; + if (n1->seq > n2->seq) + return 1; + return 0; +} + +static void index_sort_generate_seqs(struct sort_string_context *ctx) +{ + struct mail_sort_node *nodes, *nodes2; + unsigned int i, j, count, count2; + uint32_t seq; + + nodes = array_get_modifiable(&ctx->nonzero_nodes, &count); + nodes2 = array_get_modifiable(&ctx->zero_nodes, &count2); + + if (!array_is_created(&ctx->program->seqs)) + i_array_init(&ctx->program->seqs, count + count2); + else + array_clear(&ctx->program->seqs); + + for (i = j = 0;;) { + if (i < count && j < count2) { + if (nodes[i].seq < nodes2[j].seq) + seq = nodes[i++].seq; + else + seq = nodes2[j++].seq; + } else if (i < count) { + seq = nodes[i++].seq; + } else if (j < count2) { + seq = nodes2[j++].seq; + } else { + break; + } + array_push_back(&ctx->program->seqs, &seq); + } +} + +static void index_sort_reget_sort_ids(struct sort_string_context *ctx) +{ + struct mail_sort_node node; + const uint32_t *seqs; + unsigned int i, count; + + i_assert(!ctx->regetting); + ctx->regetting = TRUE; + + index_sort_generate_seqs(ctx); + array_clear(&ctx->zero_nodes); + array_clear(&ctx->nonzero_nodes); + + i_zero(&node); + node.wanted = TRUE; + seqs = array_get(&ctx->program->seqs, &count); + for (i = 0; i < count; i++) { + node.seq = seqs[i]; + index_sort_node_add(ctx, &node); + } + ctx->regetting = FALSE; +} + +static void index_sort_node_add(struct sort_string_context *ctx, + struct mail_sort_node *node) +{ + struct mail_index_map *map; + const void *data; + uint32_t reset_id; + bool expunged; + + mail_index_lookup_ext_full(ctx->program->t->view, node->seq, + ctx->ext_id, &map, &data, &expunged); + if (expunged) { + /* we don't want to update expunged messages' sort IDs */ + node->no_update = TRUE; + /* we can't trust expunged messages' sort IDs. they might be + valid, but it's also possible that sort IDs were updated + and the expunged messages' sort IDs became invalid. we could + use sort ID if we could know the extension's reset_id at the + time of the expunge so we could compare it to + highest_reset_id, but this isn't currently possible. */ + node->sort_id = 0; + } else { + node->sort_id = ctx->broken || data == NULL ? 0 : + *(const uint32_t *)data; + if (node->sort_id == 0) { + if (ctx->lowest_nonexpunged_zero > node->seq || + ctx->lowest_nonexpunged_zero == 0) + ctx->lowest_nonexpunged_zero = node->seq; + } else if (ctx->lowest_nonexpunged_zero != 0 && + ctx->lowest_nonexpunged_zero <= node->seq) { + uint32_t nonzero_uid, zero_uid; + + mail_index_lookup_uid(ctx->program->t->view, + node->seq, &nonzero_uid); + mail_index_lookup_uid(ctx->program->t->view, + ctx->lowest_nonexpunged_zero, &zero_uid); + index_sort_list_reset_broken(ctx, t_strdup_printf( + "sort_id=0 found in the middle " + "(uid=%u has sort_id, uid=%u doesn't)", + nonzero_uid, zero_uid)); + ctx->broken = TRUE; + node->sort_id = 0; + } + } + + if (node->sort_id != 0) { + /* if reset ID increases, lookup all existing messages' sort + IDs again. if it decreases, ignore the sort ID. */ + if (!mail_index_ext_get_reset_id(ctx->program->t->view, map, + ctx->ext_id, &reset_id)) + reset_id = 0; + if (reset_id != ctx->highest_reset_id) { + if (reset_id < ctx->highest_reset_id) { + i_assert(expunged); + node->sort_id = 0; + } else if (ctx->have_all_wanted) { + /* a bit late to start changing the reset_id. + the node lists aren't ordered by sequence + anymore. */ + node->sort_id = 0; + ctx->no_writing = TRUE; + } else { + ctx->highest_reset_id = reset_id; + index_sort_reget_sort_ids(ctx); + } + } + } + + if (node->sort_id == 0) + array_push_back(&ctx->zero_nodes, node); + else + array_push_back(&ctx->nonzero_nodes, node); + if (ctx->last_seq < node->seq) + ctx->last_seq = node->seq; +} + +void index_sort_list_add_string(struct mail_search_sort_program *program, + struct mail *mail) +{ + struct sort_string_context *ctx = program->context; + struct mail_sort_node node; + + i_zero(&node); + node.seq = mail->seq; + node.wanted = TRUE; + + if (mail->seq < ctx->prev_seq) + ctx->seqs_nonsorted = TRUE; + ctx->prev_seq = mail->seq; + + index_sort_node_add(ctx, &node); +} + +static int sort_node_zero_string_cmp(const struct mail_sort_node *n1, + const struct mail_sort_node *n2) +{ + struct sort_string_context *ctx = static_zero_cmp_context; + int ret; + + ret = strcmp(ctx->sort_strings[n1->seq], ctx->sort_strings[n2->seq]); + if (ret != 0) + return !ctx->reverse ? ret : -ret; + + return index_sort_node_cmp_type(ctx->program, + ctx->program->sort_program + 1, + n1->seq, n2->seq); +} + +static void index_sort_zeroes(struct sort_string_context *ctx) +{ + enum mail_sort_type sort_type = ctx->program->sort_program[0]; + string_t *str; + pool_t pool; + struct mail_sort_node *nodes; + unsigned int i, count; + + /* first get all the messages' sort strings. although this takes more + memory, it makes error handling easier and probably also helps + CPU caching. */ + ctx->sort_strings = i_new(const char *, ctx->last_seq + 1); + ctx->sort_string_pool = pool = + pool_alloconly_create("sort strings", 1024*64); + str = str_new(default_pool, 512); + nodes = array_get_modifiable(&ctx->zero_nodes, &count); + for (i = 0; i < count; i++) { + i_assert(nodes[i].seq <= ctx->last_seq); + + T_BEGIN { + if (index_sort_header_get(ctx->program, nodes[i].seq, + sort_type, str) < 0) { + nodes[i].no_update = TRUE; + ctx->failed = TRUE; + } + ctx->sort_strings[nodes[i].seq] = + str_len(str) == 0 ? "" : + p_strdup(pool, str_c(str)); + } T_END; + } + str_free(&str); + + /* we have all strings, sort nodes based on them */ + static_zero_cmp_context = ctx; + array_sort(&ctx->zero_nodes, sort_node_zero_string_cmp); +} + +static bool +index_sort_get_expunged_string(struct sort_string_context *ctx, uint32_t idx, + string_t *str, const char **result_r) +{ + enum mail_sort_type sort_type = ctx->program->sort_program[0]; + const struct mail_sort_node *nodes; + const char *result = NULL; + unsigned int i, count; + uint32_t sort_id; + + /* Look forwards and backwards to see if there are + identical sort_ids. If we do find them, try to get + their sort string and use it to update the rest. */ + nodes = array_get(&ctx->nonzero_nodes, &count); + sort_id = nodes[idx].sort_id; + /* If previous sort ID is identical and its sort string is set, we can + trust it. If it's expunged, we already verified that there are no + non-expunged messages. */ + if (idx > 0 && nodes[idx-1].sort_id == sort_id && + ctx->sort_strings[nodes[idx].seq] != NULL) { + *result_r = ctx->sort_strings[nodes[idx].seq]; + return TRUE; + } + + /* Go forwards as long as there are identical sort IDs. If we find one + that's not expunged, update string table for all messages with + identical sort IDs. */ + for (i = idx + 1; i < count; i++) { + if (nodes[i].sort_id != sort_id) + break; + + if (ctx->sort_strings[nodes[i].seq] != NULL) { + /* usually we fill all identical sort_ids and this + shouldn't happen, but we can get here if we skipped + over messages when binary searching */ + result = ctx->sort_strings[nodes[i].seq]; + break; + } + if (index_sort_header_get(ctx->program, nodes[i].seq, + sort_type, str) > 0) { + result = str_len(str) == 0 ? "" : + p_strdup(ctx->sort_string_pool, str_c(str)); + break; + } + } + if (result == NULL) { + /* unknown */ + return FALSE; + } + + /* fill all identical sort_ids with the same value */ + for (i = idx; i > 0 && nodes[i-1].sort_id == sort_id; i--) ; + for (i = idx; i < count && nodes[i].sort_id == sort_id; i++) + ctx->sort_strings[nodes[i].seq] = result; + *result_r = result; + return TRUE; +} + +static bool +index_sort_get_string(struct sort_string_context *ctx, + uint32_t idx, struct mail_sort_node *node, + const char **str_r) +{ + uint32_t seq = node->seq; + int ret = 1; + + if (node->no_update) { + /* we've already determined that we can't do this lookup */ + *str_r = ctx->sort_strings[seq]; + return FALSE; + } + + if (ctx->sort_strings[seq] == NULL) T_BEGIN { + string_t *str; + const char *result; + + str = t_str_new(256); + ret = index_sort_header_get(ctx->program, seq, + ctx->program->sort_program[0], str); + if (ret < 0) + ctx->failed = TRUE; + else if (ret == 0) { + if (!index_sort_get_expunged_string(ctx, idx, str, &result)) + ctx->sort_strings[seq] = ""; + else { + /* found the expunged string - return success */ + ctx->sort_strings[seq] = result; + ret = 1; + } + } else { + ctx->sort_strings[seq] = str_len(str) == 0 ? "" : + p_strdup(ctx->sort_string_pool, str_c(str)); + } + } T_END; + + if (ret <= 0) + node->no_update = TRUE; + *str_r = ctx->sort_strings[seq]; + return ret > 0; +} + +static void +index_sort_bsearch(struct sort_string_context *ctx, const char *key, + unsigned int start_idx, unsigned int *idx_r, + const char **prev_str_r) +{ + struct mail_sort_node *nodes; + const char *str, *str2; + unsigned int idx, left_idx, right_idx, prev; + int ret; + + nodes = array_get_modifiable(&ctx->nonzero_nodes, &right_idx); + i_assert(right_idx < INT_MAX); + idx = left_idx = start_idx; + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + if (index_sort_get_string(ctx, idx, &nodes[idx], &str)) + ret = strcmp(key, str); + else { + /* put expunged (and otherwise failed) messages first */ + ret = 1; + for (prev = idx; prev > 0; ) { + prev--; + if (index_sort_get_string(ctx, prev, + &nodes[prev], + &str2)) { + ret = strcmp(key, str2); + if (ret <= 0) { + idx = prev; + str = str2; + } + break; + } + } + } + if (ret > 0) + left_idx = idx+1; + else if (ret < 0) + right_idx = idx; + else { + *idx_r = idx + 1; + *prev_str_r = str; + return; + } + } + + if (left_idx > idx) + idx++; + + *idx_r = idx; + if (idx > start_idx) { + bool success; + + prev = idx; + do { + prev--; + success = index_sort_get_string(ctx, prev, + &nodes[prev], &str2); + } while (!success && prev > 0 && + nodes[prev-1].sort_id == nodes[prev].sort_id); + *prev_str_r = str2; + } +} + +static void index_sort_merge(struct sort_string_context *ctx) +{ + struct mail_sort_node *znodes, *nznodes; + const char *zstr, *nzstr, *prev_str; + unsigned int zpos, nzpos, nz_next_pos, zcount, nzcount; + int ret; + + /* both zero_nodes and nonzero_nodes are sorted. we'll now just have + to merge them together. use sorted_nodes as the result array. */ + i_array_init(&ctx->sorted_nodes, array_count(&ctx->nonzero_nodes) + + array_count(&ctx->zero_nodes)); + + znodes = array_get_modifiable(&ctx->zero_nodes, &zcount); + nznodes = array_get_modifiable(&ctx->nonzero_nodes, &nzcount); + + prev_str = NULL; + for (zpos = nzpos = 0; zpos < zcount && nzpos < nzcount; ) { + zstr = ctx->sort_strings[znodes[zpos].seq]; + if (index_sort_get_string(ctx, nzpos, &nznodes[nzpos], &nzstr)) + ret = strcmp(zstr, nzstr); + else if (prev_str != NULL && strcmp(zstr, prev_str) == 0) { + /* identical to previous message, must keep them + together */ + ret = -1; + } else { + /* we can't be yet sure about the order, but future + nznodes may reveal that the znode must be added + later. if future nznodes don't reveal that, we have + no idea about these nodes' order. so just always + put the expunged message first. */ + ret = 1; + } + + if (ret == 0) { + ret = index_sort_node_cmp_type(ctx->program, + ctx->program->sort_program + 1, + znodes[zpos].seq, nznodes[nzpos].seq); + } + if (ret <= 0) { + array_push_back(&ctx->sorted_nodes, &znodes[zpos]); + prev_str = zstr; + zpos++; + } else { + array_push_back(&ctx->sorted_nodes, &nznodes[nzpos]); + prev_str = nzstr; + nzpos++; + + /* avoid looking up all existing messages' strings by + binary searching the next zero-node position. don't + bother if it looks like more work than linear + scanning. */ + if (zcount - zpos < (nzcount - nzpos)/2) { + index_sort_bsearch(ctx, zstr, nzpos, + &nz_next_pos, &prev_str); + array_append(&ctx->sorted_nodes, + &nznodes[nzpos], + nz_next_pos - nzpos); + nzpos = nz_next_pos; + } + } + } + /* only one of zero_nodes and nonzero_nodes can be non-empty now */ + for (; zpos < zcount; zpos++) + array_push_back(&ctx->sorted_nodes, &znodes[zpos]); + for (; nzpos < nzcount; nzpos++) + array_push_back(&ctx->sorted_nodes, &nznodes[nzpos]); + + /* future index_sort_get_string() calls use ctx->nonzero_nodes, but we + use only ctx->sorted_nodes. make them identical. */ + array_free(&ctx->nonzero_nodes); + ctx->nonzero_nodes = ctx->sorted_nodes; +} + +static int +index_sort_add_ids_range(struct sort_string_context *ctx, + unsigned int left_idx, unsigned int right_idx, + const char **reason_r) +{ + + struct mail_sort_node *nodes; + unsigned int i, count, rightmost_idx, skip; + const char *left_str = NULL, *right_str = NULL, *str = NULL; + uint32_t left_sort_id, right_sort_id, diff, left_str_idx = 0; + bool no_left_str = FALSE, no_right_str = FALSE; + int ret; + + nodes = array_get_modifiable(&ctx->sorted_nodes, &count); + rightmost_idx = count - 1; + + /* get the sort IDs from left and right */ + left_sort_id = nodes[left_idx].sort_id; + right_sort_id = nodes[right_idx].sort_id; + /* check if all of them should have the same sort IDs. we don't want + to hit the renumbering code in that situation. */ + if (left_sort_id == right_sort_id && left_sort_id != 0) { + /* they should all have the same sort ID */ + for (i = left_idx + 1; i < right_idx; i++) { + nodes[i].sort_id = left_sort_id; + nodes[i].sort_id_changed = TRUE; + } + return 0; + } + + if (left_sort_id == 0) { + i_assert(left_idx == 0); + left_sort_id = 1; + } + if (right_sort_id == 0) { + i_assert(right_idx == rightmost_idx); + right_sort_id = (uint32_t)-1; + } + i_assert(left_sort_id <= right_sort_id); + + diff = right_sort_id - left_sort_id; + while (diff / (right_idx-left_idx + 2) == 0) { + /* we most likely don't have enough space. we have to + renumber some of the existing sort IDs. do this by + widening the area we're giving sort IDs. */ + while (left_idx > 0) { + if (nodes[--left_idx].sort_id != left_sort_id) { + left_sort_id = nodes[left_idx].sort_id; + if (left_sort_id == 0) { + i_assert(left_idx == 0); + left_sort_id = 1; + } + break; + } + } + + while (right_idx < rightmost_idx) { + right_idx++; + if (nodes[right_idx].sort_id > right_sort_id) + break; + } + right_sort_id = nodes[right_idx].sort_id; + if (right_sort_id == 0) { + i_assert(right_idx == rightmost_idx); + right_sort_id = (uint32_t)-1; + } + i_assert(left_sort_id <= right_sort_id); + + if (diff == right_sort_id - left_sort_id) { + /* we did nothing, but there's still not enough space. + have to renumber the leftmost/rightmost node(s) */ + i_assert(left_idx == 0 && right_idx == rightmost_idx); + if (left_sort_id > 1) { + left_sort_id = 1; + no_left_str = TRUE; + } else { + i_assert(right_sort_id != (uint32_t)-1); + right_sort_id = (uint32_t)-1; + no_right_str = TRUE; + } + } + diff = right_sort_id - left_sort_id; + } + + if (nodes[left_idx].sort_id != 0 && !no_left_str) { + if (!index_sort_get_string(ctx, left_idx, + &nodes[left_idx], &left_str)) { + /* not equivalent with any message */ + left_str = NULL; + } else { + left_str_idx = left_idx; + } + left_idx++; + } + if (nodes[right_idx].sort_id != 0 && !no_right_str) { + if (!index_sort_get_string(ctx, right_idx, + &nodes[right_idx], &right_str)) { + /* not equivalent with any message */ + right_str = NULL; + } + right_idx--; + } + i_assert(left_idx <= right_idx); + + /* give (new) sort IDs to all nodes in left_idx..right_idx range. + divide the available space so that each message gets an equal sized + share. some messages' sort strings may be equivalent, so give them + the same sort IDs. */ + for (i = left_idx; i <= right_idx; i++) { + if (!index_sort_get_string(ctx, i, &nodes[i], &str)) { + /* it doesn't really matter what we give to this + message, since it's only temporary and we don't + know its correct position anyway. so let's assume + it's equivalent to previous message. */ + nodes[i].sort_id = left_sort_id; + continue; + } + + ret = left_str == NULL ? 1 : strcmp(str, left_str); + if (ret <= 0) { + if (ret < 0) { + /* broken sort_ids */ + uint32_t str_uid, left_str_uid; + + mail_index_lookup_uid(ctx->program->t->view, + nodes[i].seq, &str_uid); + mail_index_lookup_uid(ctx->program->t->view, + nodes[left_str_idx].seq, &left_str_uid); + *reason_r = t_strdup_printf( + "(idx=%u, seq=%u, uid=%u) '%s' < left string (idx=%u, seq=%u, uid=%u) '%s'", + i, nodes[i].seq, str_uid, str, + left_str_idx, nodes[left_str_idx].seq, left_str_uid, left_str); + return -1; + } + nodes[i].sort_id = left_sort_id; + } else if (right_str != NULL && strcmp(str, right_str) == 0) { + /* the rest of the sort IDs should be the same */ + nodes[i].sort_id = right_sort_id; + left_sort_id = right_sort_id; + } else { + /* divide the available space equally. leave the same + sized space also between the first and the last + messages */ + skip = (right_sort_id - left_sort_id) / + (right_idx - i + 2); + if (skip == 0) { + /* broken sort IDs (we previously assigned + left_sort_id=right_sort_id) */ + uint32_t uid; + mail_index_lookup_uid(ctx->program->t->view, + nodes[i].seq, &uid); + *reason_r = t_strdup_printf( + "no sort_id space for uid=%u", uid); + return -1; + } + left_sort_id += skip; + i_assert(left_sort_id < right_sort_id); + + nodes[i].sort_id = left_sort_id; + left_str = str; + left_str_idx = i; + } + nodes[i].sort_id_changed = TRUE; + } + i_assert(str != NULL); + + if (right_str == NULL || strcmp(str, right_str) < 0 || + (strcmp(str, right_str) == 0 && + nodes[i-1].sort_id == right_sort_id)) + return 0; + + *reason_r = t_strdup_printf("Invalid sort_id order ('%s' > '%s')", + str, right_str); + return -1; +} + +static int +index_sort_add_sort_ids(struct sort_string_context *ctx, const char **reason_r) +{ + const struct mail_sort_node *nodes; + unsigned int i, left_idx, right_idx, count; + + nodes = array_get(&ctx->sorted_nodes, &count); + for (i = 0; i < count; i++) { + if (nodes[i].sort_id != 0) + continue; + + /* get the range for all sort_id=0 nodes. include the nodes + left and right of the range as well */ + for (right_idx = i + 1; right_idx < count; right_idx++) { + if (nodes[right_idx].sort_id != 0) + break; + } + if (right_idx == count) + right_idx--; + left_idx = i == 0 ? 0 : i - 1; + if (index_sort_add_ids_range(ctx, left_idx, right_idx, reason_r) < 0) + return -1; + } + return 0; +} + +static void index_sort_write_changed_sort_ids(struct sort_string_context *ctx) +{ + struct mail_index_transaction *itrans = ctx->program->t->itrans; + uint32_t ext_id = ctx->ext_id; + const struct mail_sort_node *nodes; + unsigned int i, count; + uint32_t lowest_failed_seq; + + if (ctx->no_writing) { + /* our reset_id is already stale - don't even bother + trying to write */ + return; + } + + mail_index_ext_reset_inc(itrans, ext_id, + ctx->highest_reset_id, FALSE); + + /* We require that there aren't sort_id=0 gaps in the middle of the + mails. At this point they could exist though, because some of the + mail lookups may have failed. Failures due to expunges don't matter, + because on the next lookup those mails will be lost anyway. + Otherwise, make sure we don't write those gaps out + + First find the lowest non-expunged mail that has no_update set. */ + nodes = array_get_modifiable(&ctx->sorted_nodes, &count); + lowest_failed_seq = (uint32_t)-1; + for (i = 0; i < count; i++) { + uint32_t seq = nodes[i].seq; + + if (nodes[i].no_update && lowest_failed_seq > seq && + !mail_index_is_expunged(ctx->program->t->view, seq)) + lowest_failed_seq = seq; + } + + /* add the missing sort IDs to index, but only for those sequences + that are below lowest_failed_seq */ + nodes = array_get_modifiable(&ctx->sorted_nodes, &count); + for (i = 0; i < count; i++) { + i_assert(nodes[i].sort_id != 0); + if (!nodes[i].sort_id_changed || nodes[i].no_update || + nodes[i].seq >= lowest_failed_seq) + continue; + + mail_index_update_ext(itrans, nodes[i].seq, ext_id, + &nodes[i].sort_id, NULL); + } +} + +static int sort_node_cmp(const struct mail_sort_node *n1, + const struct mail_sort_node *n2) +{ + struct sort_string_context *ctx = static_zero_cmp_context; + + if (n1->sort_id < n2->sort_id) + return !ctx->reverse ? -1 : 1; + if (n1->sort_id > n2->sort_id) + return !ctx->reverse ? 1 : -1; + + return index_sort_node_cmp_type(ctx->program, + ctx->program->sort_program + 1, + n1->seq, n2->seq); +} + +static void index_sort_add_missing(struct sort_string_context *ctx) +{ + struct mail_sort_node node; + const uint32_t *seqs; + unsigned int i, count; + uint32_t seq, next_seq; + + ctx->have_all_wanted = TRUE; + + seqs = array_get(&ctx->program->seqs, &count); + for (i = 0, next_seq = 1; i < count; i++) { + if (seqs[i] == next_seq) + next_seq++; + else { + i_assert(next_seq < seqs[i]); + for (seq = next_seq; seq < seqs[i]; seq++) { + i_zero(&node); + node.seq = seq; + index_sort_node_add(ctx, &node); + } + next_seq = seqs[i] + 1; + } + } + + if (ctx->lowest_nonexpunged_zero == 0) { + /* we're handling only expunged zeros. if it causes us to + renumber some existing sort IDs, don't save them. */ + ctx->no_writing = TRUE; + } +} + +static void index_sort_list_reset_broken(struct sort_string_context *ctx, + const char *reason) +{ + struct mailbox *box = ctx->program->t->box; + struct mail_sort_node *node; + + mailbox_set_critical(box, "Broken %s indexes, resetting: %s", + ctx->primary_sort_name, reason); + + array_clear(&ctx->zero_nodes); + array_append_array(&ctx->zero_nodes, + &ctx->nonzero_nodes); + array_clear(&ctx->nonzero_nodes); + + array_foreach_modifiable(&ctx->zero_nodes, node) + node->sort_id = 0; +} + +void index_sort_list_finish_string(struct mail_search_sort_program *program) +{ + struct sort_string_context *ctx = program->context; + const struct mail_sort_node *nodes; + unsigned int i, count; + const char *reason; + uint32_t seq; + + static_zero_cmp_context = ctx; + if (array_count(&ctx->zero_nodes) == 0) { + /* fast path: we have all sort IDs */ + array_sort(&ctx->nonzero_nodes, sort_node_cmp); + + nodes = array_get(&ctx->nonzero_nodes, &count); + if (!array_is_created(&program->seqs)) + i_array_init(&program->seqs, count); + else + array_clear(&program->seqs); + + for (i = 0; i < count; i++) { + seq = nodes[i].seq; + array_push_back(&program->seqs, &seq); + } + array_free(&ctx->nonzero_nodes); + } else { + if (ctx->seqs_nonsorted) { + /* the nodes need to be sorted by sequence initially */ + array_sort(&ctx->zero_nodes, sort_node_seq_cmp); + array_sort(&ctx->nonzero_nodes, sort_node_seq_cmp); + } + + /* we have to add some sort IDs. we'll do this for all + messages, so first remember what messages we wanted + to know about. */ + index_sort_generate_seqs(ctx); + /* add messages not in seqs list */ + index_sort_add_missing(ctx); + /* sort all messages with sort IDs */ + array_sort(&ctx->nonzero_nodes, sort_node_cmp); + for (;;) { + /* sort all messages without sort IDs */ + index_sort_zeroes(ctx); + + if (ctx->reverse) { + /* sort lists are descending currently, but + merging and sort ID assigning works only + with ascending lists. reverse the lists + temporarily. we can't do this while earlier + because secondary sort conditions must not + be reversed in results (but while assigning + sort IDs it doesn't matter). */ + array_reverse(&ctx->nonzero_nodes); + array_reverse(&ctx->zero_nodes); + } + + /* merge zero and non-zero arrays into sorted_nodes */ + index_sort_merge(ctx); + /* give sort IDs to messages missing them */ + if (index_sort_add_sort_ids(ctx, &reason) == 0) + break; + + /* broken, try again with sort IDs reset */ + index_sort_list_reset_broken(ctx, reason); + } + index_sort_write_changed_sort_ids(ctx); + + if (ctx->reverse) { + /* restore the correct sort order */ + array_reverse(&ctx->sorted_nodes); + } + + nodes = array_get(&ctx->sorted_nodes, &count); + array_clear(&program->seqs); + for (i = 0; i < count; i++) { + if (nodes[i].wanted) { + seq = nodes[i].seq; + array_push_back(&program->seqs, &seq); + } + } + pool_unref(&ctx->sort_string_pool); + i_free(ctx->sort_strings); + array_free(&ctx->sorted_nodes); + /* NOTE: we already freed nonzero_nodes and made it point to + sorted_nodes. */ + } + if (ctx->failed) + program->failed = TRUE; + + array_free(&ctx->zero_nodes); + i_free(ctx); + program->context = NULL; +} diff --git a/src/lib-storage/index/index-sort.c b/src/lib-storage/index/index-sort.c new file mode 100644 index 0000000..924a8d1 --- /dev/null +++ b/src/lib-storage/index/index-sort.c @@ -0,0 +1,738 @@ +/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "unichar.h" +#include "message-address.h" +#include "message-header-decode.h" +#include "imap-base-subject.h" +#include "index-storage.h" +#include "index-sort-private.h" + + +struct mail_sort_node_date { + uint32_t seq; + time_t date; +}; +ARRAY_DEFINE_TYPE(mail_sort_node_date, struct mail_sort_node_date); + +struct mail_sort_node_size { + uint32_t seq; + uoff_t size; +}; +ARRAY_DEFINE_TYPE(mail_sort_node_size, struct mail_sort_node_size); + +struct mail_sort_node_float { + uint32_t seq; + float num; +}; +ARRAY_DEFINE_TYPE(mail_sort_node_float, struct mail_sort_node_float); + +struct sort_cmp_context { + struct mail_search_sort_program *program; + bool reverse; +}; + +static struct sort_cmp_context static_node_cmp_context; + +static void +index_sort_program_set_mail_failed(struct mail_search_sort_program *program, + struct mail *mail) +{ + switch (mailbox_get_last_mail_error(mail->box)) { + case MAIL_ERROR_EXPUNGED: + break; + case MAIL_ERROR_LOOKUP_ABORTED: + /* just change the error message */ + i_assert(program->slow_mails_left == 0); + mail_storage_set_error(program->t->box->storage, MAIL_ERROR_LIMIT, + "Requested sort would have taken too long."); + /* fall through */ + default: + program->failed = TRUE; + break; + } +} + +static time_t +index_sort_program_set_date_failed(struct mail_search_sort_program *program, + struct mail *mail) +{ + index_sort_program_set_mail_failed(program, mail); + + if (mailbox_get_last_mail_error(mail->box) == MAIL_ERROR_LIMIT) { + /* limit reached - sort the rest of the mails at the end of + the list by their UIDs */ + return LONG_MAX; + } else { + /* expunged / some other error - sort in the beginning */ + return 0; + } +} + +static void +index_sort_list_add_arrival(struct mail_search_sort_program *program, + struct mail *mail) +{ + ARRAY_TYPE(mail_sort_node_date) *nodes = program->context; + struct mail_sort_node_date *node; + + node = array_append_space(nodes); + node->seq = mail->seq; + if (mail_get_received_date(mail, &node->date) < 0) + node->date = index_sort_program_set_date_failed(program, mail); +} + +static void +index_sort_list_add_date(struct mail_search_sort_program *program, + struct mail *mail) +{ + ARRAY_TYPE(mail_sort_node_date) *nodes = program->context; + struct mail_sort_node_date *node; + int tz; + + node = array_append_space(nodes); + node->seq = mail->seq; + if (mail_get_date(mail, &node->date, &tz) < 0) + node->date = index_sort_program_set_date_failed(program, mail); + else if (node->date == 0) { + if (mail_get_received_date(mail, &node->date) < 0) + node->date = index_sort_program_set_date_failed(program, mail); + } +} + +static void +index_sort_list_add_size(struct mail_search_sort_program *program, + struct mail *mail) +{ + ARRAY_TYPE(mail_sort_node_size) *nodes = program->context; + struct mail_sort_node_size *node; + + node = array_append_space(nodes); + node->seq = mail->seq; + if (mail_get_virtual_size(mail, &node->size) < 0) { + index_sort_program_set_mail_failed(program, mail); + node->size = 0; + } +} + +static int index_sort_get_pop3_order(struct mail *mail, uoff_t *size_r) +{ + const char *str; + + if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) { + *size_r = (uint32_t)-1; + return -1; + } + + if (str_to_uoff(str, size_r) < 0) + *size_r = (uint32_t)-1; + return 0; +} + +static void +index_sort_list_add_pop3_order(struct mail_search_sort_program *program, + struct mail *mail) +{ + ARRAY_TYPE(mail_sort_node_size) *nodes = program->context; + struct mail_sort_node_size *node; + + node = array_append_space(nodes); + node->seq = mail->seq; + (void)index_sort_get_pop3_order(mail, &node->size); +} + +static int index_sort_get_relevancy(struct mail *mail, float *result_r) +{ + const char *str; + + if (mail_get_special(mail, MAIL_FETCH_SEARCH_RELEVANCY, &str) < 0) { + *result_r = 0; + return -1; + } + *result_r = strtod(str, NULL); + return 0; +} + +static void +index_sort_list_add_relevancy(struct mail_search_sort_program *program, + struct mail *mail) +{ + ARRAY_TYPE(mail_sort_node_float) *nodes = program->context; + struct mail_sort_node_float *node; + + node = array_append_space(nodes); + node->seq = mail->seq; + (void)index_sort_get_relevancy(mail, &node->num); +} + +void index_sort_list_add(struct mail_search_sort_program *program, + struct mail *mail) +{ + enum mail_access_type orig_access_type = mail->access_type; + bool prev_slow = mail->mail_stream_accessed || + mail->mail_metadata_accessed; + + i_assert(mail->transaction == program->t); + /* if lookup_abort isn't NEVER, mail_sort_max_read_count handling + doesn't work right. */ + i_assert(mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER); + + if (program->slow_mails_left == 0) + mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + + mail->access_type = MAIL_ACCESS_TYPE_SORT; + T_BEGIN { + program->sort_list_add(program, mail); + } T_END; + mail->access_type = orig_access_type; + + if (!prev_slow && (mail->mail_stream_accessed || + mail->mail_metadata_accessed)) { + i_assert(program->slow_mails_left > 0); + program->slow_mails_left--; + } + mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER; +} + +static int sort_node_date_cmp(const struct mail_sort_node_date *n1, + const struct mail_sort_node_date *n2) +{ + struct sort_cmp_context *ctx = &static_node_cmp_context; + + if (n1->date < n2->date) + return !ctx->reverse ? -1 : 1; + if (n1->date > n2->date) + return !ctx->reverse ? 1 : -1; + + return index_sort_node_cmp_type(ctx->program, + ctx->program->sort_program + 1, + n1->seq, n2->seq); +} + +static void +index_sort_list_finish_date(struct mail_search_sort_program *program) +{ + ARRAY_TYPE(mail_sort_node_date) *nodes = program->context; + + array_sort(nodes, sort_node_date_cmp); + memcpy(&program->seqs, nodes, sizeof(program->seqs)); + i_free(nodes); + program->context = NULL; +} + +static int sort_node_size_cmp(const struct mail_sort_node_size *n1, + const struct mail_sort_node_size *n2) +{ + struct sort_cmp_context *ctx = &static_node_cmp_context; + + if (n1->size < n2->size) + return !ctx->reverse ? -1 : 1; + if (n1->size > n2->size) + return !ctx->reverse ? 1 : -1; + + return index_sort_node_cmp_type(ctx->program, + ctx->program->sort_program + 1, + n1->seq, n2->seq); +} + +static void +index_sort_list_finish_size(struct mail_search_sort_program *program) +{ + ARRAY_TYPE(mail_sort_node_size) *nodes = program->context; + + array_sort(nodes, sort_node_size_cmp); + memcpy(&program->seqs, nodes, sizeof(program->seqs)); + i_free(nodes); + program->context = NULL; +} + +static int sort_node_float_cmp(const struct mail_sort_node_float *n1, + const struct mail_sort_node_float *n2) +{ + struct sort_cmp_context *ctx = &static_node_cmp_context; + + if (n1->num < n2->num) + return !ctx->reverse ? -1 : 1; + if (n1->num > n2->num) + return !ctx->reverse ? 1 : -1; + + return index_sort_node_cmp_type(ctx->program, + ctx->program->sort_program + 1, + n1->seq, n2->seq); +} + +static void +index_sort_list_finish_float(struct mail_search_sort_program *program) +{ + ARRAY_TYPE(mail_sort_node_float) *nodes = program->context; + + /* NOTE: higher relevancy is returned first, unlike with all + other number based sort keys, so temporarily reverse the search */ + static_node_cmp_context.reverse = !static_node_cmp_context.reverse; + array_sort(nodes, sort_node_float_cmp); + static_node_cmp_context.reverse = !static_node_cmp_context.reverse; + + memcpy(&program->seqs, nodes, sizeof(program->seqs)); + i_free(nodes); + program->context = NULL; +} + +void index_sort_list_finish(struct mail_search_sort_program *program) +{ + i_zero(&static_node_cmp_context); + static_node_cmp_context.program = program; + static_node_cmp_context.reverse = + (program->sort_program[0] & MAIL_SORT_FLAG_REVERSE) != 0; + + struct event_reason *reason = event_reason_begin("mailbox:sort"); + program->sort_list_finish(program); + event_reason_end(&reason); +} + +bool index_sort_list_next(struct mail_search_sort_program *program, + uint32_t *seq_r) +{ + const uint32_t *seqp; + + if (program->iter_idx == array_count(&program->seqs)) + return FALSE; + + seqp = array_idx(&program->seqs, program->iter_idx++); + *seq_r = *seqp; + return TRUE; +} + +static void +get_wanted_fields(struct mailbox *box, const enum mail_sort_type *sort_program, + enum mail_fetch_field *wanted_fields_r, + struct mailbox_header_lookup_ctx **wanted_headers_r) +{ + enum mail_fetch_field fields = 0; + ARRAY_TYPE(const_string) headers; + const char *hdr_name; + + t_array_init(&headers, 4); + for (unsigned int i = 0; sort_program[i] != MAIL_SORT_END; i++) { + enum mail_sort_type sort_type = + sort_program[i] & MAIL_SORT_MASK; + switch (sort_type) { + case MAIL_SORT_ARRIVAL: + fields |= MAIL_FETCH_RECEIVED_DATE; + break; + case MAIL_SORT_CC: + hdr_name = "Cc"; + array_push_back(&headers, &hdr_name); + break; + case MAIL_SORT_DATE: + fields |= MAIL_FETCH_DATE; + break; + case MAIL_SORT_FROM: + case MAIL_SORT_DISPLAYFROM: + hdr_name = "From"; + array_push_back(&headers, &hdr_name); + break; + case MAIL_SORT_SIZE: + fields |= MAIL_FETCH_VIRTUAL_SIZE; + break; + case MAIL_SORT_SUBJECT: + hdr_name = "Subject"; + array_push_back(&headers, &hdr_name); + break; + case MAIL_SORT_TO: + case MAIL_SORT_DISPLAYTO: + hdr_name = "To"; + array_push_back(&headers, &hdr_name); + break; + case MAIL_SORT_RELEVANCY: + fields |= MAIL_FETCH_SEARCH_RELEVANCY; + break; + case MAIL_SORT_POP3_ORDER: + fields |= MAIL_FETCH_POP3_ORDER; + break; + + case MAIL_SORT_MASK: + case MAIL_SORT_FLAG_REVERSE: + case MAIL_SORT_END: + i_unreached(); + } + } + *wanted_fields_r = fields; + if (array_count(&headers) == 0) + *wanted_headers_r = NULL; + else { + array_append_zero(&headers); + *wanted_headers_r = + mailbox_header_lookup_init(box, array_idx(&headers, 0)); + } +} + +struct mail_search_sort_program * +index_sort_program_init(struct mailbox_transaction_context *t, + const enum mail_sort_type *sort_program) +{ + struct mail_search_sort_program *program; + enum mail_fetch_field wanted_fields; + struct mailbox_header_lookup_ctx *wanted_headers; + unsigned int i; + + if (sort_program == NULL || sort_program[0] == MAIL_SORT_END) + return NULL; + + get_wanted_fields(t->box, sort_program, &wanted_fields, &wanted_headers); + + /* we support internal sorting by the primary condition */ + program = i_new(struct mail_search_sort_program, 1); + program->t = t; + program->temp_mail = mail_alloc(t, wanted_fields, wanted_headers); + program->temp_mail->access_type = MAIL_ACCESS_TYPE_SORT; + if (wanted_headers != NULL) + mailbox_header_lookup_unref(&wanted_headers); + + program->slow_mails_left = + program->t->box->storage->set->mail_sort_max_read_count; + if (program->slow_mails_left == 0) + program->slow_mails_left = UINT_MAX; + + for (i = 0; i < MAX_SORT_PROGRAM_SIZE; i++) { + program->sort_program[i] = sort_program[i]; + if (sort_program[i] == MAIL_SORT_END) + break; + } + if (i == MAX_SORT_PROGRAM_SIZE) + i_panic("index_sort_program_init(): Invalid sort program"); + + switch (program->sort_program[0] & MAIL_SORT_MASK) { + case MAIL_SORT_ARRIVAL: + case MAIL_SORT_DATE: { + ARRAY_TYPE(mail_sort_node_date) *nodes; + + nodes = i_malloc(sizeof(*nodes)); + i_array_init(nodes, 128); + + if ((program->sort_program[0] & + MAIL_SORT_MASK) == MAIL_SORT_ARRIVAL) + program->sort_list_add = index_sort_list_add_arrival; + else + program->sort_list_add = index_sort_list_add_date; + program->sort_list_finish = index_sort_list_finish_date; + program->context = nodes; + break; + } + case MAIL_SORT_SIZE: { + ARRAY_TYPE(mail_sort_node_size) *nodes; + + nodes = i_malloc(sizeof(*nodes)); + i_array_init(nodes, 128); + program->sort_list_add = index_sort_list_add_size; + program->sort_list_finish = index_sort_list_finish_size; + program->context = nodes; + break; + } + case MAIL_SORT_CC: + case MAIL_SORT_FROM: + case MAIL_SORT_SUBJECT: + case MAIL_SORT_TO: + case MAIL_SORT_DISPLAYFROM: + case MAIL_SORT_DISPLAYTO: + program->sort_list_add = index_sort_list_add_string; + program->sort_list_finish = index_sort_list_finish_string; + index_sort_list_init_string(program); + break; + case MAIL_SORT_RELEVANCY: { + ARRAY_TYPE(mail_sort_node_float) *nodes; + + nodes = i_malloc(sizeof(*nodes)); + i_array_init(nodes, 128); + program->sort_list_add = index_sort_list_add_relevancy; + program->sort_list_finish = index_sort_list_finish_float; + program->context = nodes; + break; + } + case MAIL_SORT_POP3_ORDER: { + ARRAY_TYPE(mail_sort_node_size) *nodes; + + nodes = i_malloc(sizeof(*nodes)); + i_array_init(nodes, 128); + program->sort_list_add = index_sort_list_add_pop3_order; + program->sort_list_finish = index_sort_list_finish_size; + program->context = nodes; + break; + } + default: + i_unreached(); + } + return program; +} + +int index_sort_program_deinit(struct mail_search_sort_program **_program) +{ + struct mail_search_sort_program *program = *_program; + + *_program = NULL; + + if (program->context != NULL) + index_sort_list_finish(program); + mail_free(&program->temp_mail); + array_free(&program->seqs); + + int ret = program->failed ? -1 : 0; + i_free(program); + return ret; +} + +static int +get_first_addr(struct mail *mail, const char *header, + struct message_address **addr_r) +{ + const char *str; + int ret; + + if ((ret = mail_get_first_header(mail, header, &str)) <= 0) { + *addr_r = NULL; + return ret; + } + + *addr_r = message_address_parse(pool_datastack_create(), + (const unsigned char *)str, + strlen(str), 1, + MESSAGE_ADDRESS_PARSE_FLAG_FILL_MISSING); + return 0; +} + +static int +get_first_mailbox(struct mail *mail, const char *header, const char **mailbox_r) +{ + struct message_address *addr; + + if (get_first_addr(mail, header, &addr) < 0) { + *mailbox_r = ""; + return -1; + } + *mailbox_r = addr != NULL && addr->mailbox != NULL ? addr->mailbox : ""; + return 0; +} + +static int +get_display_name(struct mail *mail, const char *header, const char **name_r) +{ + struct message_address *addr; + + *name_r = ""; + + if (get_first_addr(mail, header, &addr) < 0) + return -1; + if (addr == NULL) + return 0; + + if (addr->name != NULL) { + string_t *str; + size_t len = strlen(addr->name); + + str = t_str_new(len*2); + (void)message_header_decode_utf8( + (const unsigned char *)addr->name, len, str, NULL); + if (str_len(str) > 0) { + *name_r = str_c(str); + return 0; + } + } + if (addr->mailbox != NULL && addr->domain != NULL) + *name_r = t_strconcat(addr->mailbox, "@", addr->domain, NULL); + else if (addr->mailbox != NULL) + *name_r = addr->mailbox; + return 0; +} + +static void +index_sort_set_seq(struct mail_search_sort_program *program, + struct mail *mail, uint32_t seq) +{ + if ((mail->mail_stream_accessed || mail->mail_metadata_accessed) && + program->slow_mails_left > 0) + program->slow_mails_left--; + mail_set_seq(mail, seq); + if (program->slow_mails_left == 0) { + /* too many slow lookups - just return the rest of the results + in whatever order. */ + mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + } +} + +int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq, + enum mail_sort_type sort_type, string_t *dest) +{ + struct mail *mail = program->temp_mail; + const char *str; + int ret; + bool reply_or_fw; + + index_sort_set_seq(program, mail, seq); + str_truncate(dest, 0); + + switch (sort_type & MAIL_SORT_MASK) { + case MAIL_SORT_SUBJECT: + ret = mail_get_first_header(mail, "Subject", &str); + if (ret < 0) + break; + if (ret == 0) { + /* nonexistent header */ + return 1; + } + str = imap_get_base_subject_cased(pool_datastack_create(), + str, &reply_or_fw); + str_append(dest, str); + return 1; + case MAIL_SORT_CC: + ret = get_first_mailbox(mail, "Cc", &str); + break; + case MAIL_SORT_FROM: + ret = get_first_mailbox(mail, "From", &str); + break; + case MAIL_SORT_TO: + ret = get_first_mailbox(mail, "To", &str); + break; + case MAIL_SORT_DISPLAYFROM: + ret = get_display_name(mail, "From", &str); + break; + case MAIL_SORT_DISPLAYTO: + ret = get_display_name(mail, "To", &str); + break; + default: + i_unreached(); + } + if (ret < 0) { + index_sort_program_set_mail_failed(program, mail); + if (!program->failed) + return 0; + return -1; + } + + (void)uni_utf8_to_decomposed_titlecase(str, strlen(str), dest); + return 1; +} + +int index_sort_node_cmp_type(struct mail_search_sort_program *program, + const enum mail_sort_type *sort_program, + uint32_t seq1, uint32_t seq2) +{ + struct mail *mail = program->temp_mail; + enum mail_sort_type sort_type; + time_t time1, time2; + uoff_t size1, size2; + float float1, float2; + int tz, ret = 0; + + sort_type = *sort_program & MAIL_SORT_MASK; + switch (sort_type) { + case MAIL_SORT_CC: + case MAIL_SORT_FROM: + case MAIL_SORT_TO: + case MAIL_SORT_SUBJECT: + case MAIL_SORT_DISPLAYFROM: + case MAIL_SORT_DISPLAYTO: + T_BEGIN { + string_t *str1, *str2; + + str1 = t_str_new(256); + str2 = t_str_new(256); + if (index_sort_header_get(program, seq1, sort_type, str1) < 0) + index_sort_program_set_mail_failed(program, mail); + if (index_sort_header_get(program, seq2, sort_type, str2) < 0) + index_sort_program_set_mail_failed(program, mail); + + ret = strcmp(str_c(str1), str_c(str2)); + } T_END; + break; + case MAIL_SORT_ARRIVAL: + index_sort_set_seq(program, mail, seq1); + if (mail_get_received_date(mail, &time1) < 0) + time1 = index_sort_program_set_date_failed(program, mail); + + index_sort_set_seq(program, mail, seq2); + if (mail_get_received_date(mail, &time2) < 0) + time2 = index_sort_program_set_date_failed(program, mail); + + ret = time1 < time2 ? -1 : + (time1 > time2 ? 1 : 0); + break; + case MAIL_SORT_DATE: + index_sort_set_seq(program, mail, seq1); + if (mail_get_date(mail, &time1, &tz) < 0) + time1 = index_sort_program_set_date_failed(program, mail); + else if (time1 == 0) { + if (mail_get_received_date(mail, &time1) < 0) + time1 = index_sort_program_set_date_failed(program, mail); + } + + index_sort_set_seq(program, mail, seq2); + if (mail_get_date(mail, &time2, &tz) < 0) + time2 = index_sort_program_set_date_failed(program, mail); + else if (time2 == 0) { + if (mail_get_received_date(mail, &time2) < 0) + time2 = index_sort_program_set_date_failed(program, mail); + } + + ret = time1 < time2 ? -1 : + (time1 > time2 ? 1 : 0); + break; + case MAIL_SORT_SIZE: + index_sort_set_seq(program, mail, seq1); + if (mail_get_virtual_size(mail, &size1) < 0) { + index_sort_program_set_mail_failed(program, mail); + size1 = 0; + } + + index_sort_set_seq(program, mail, seq2); + if (mail_get_virtual_size(mail, &size2) < 0) { + index_sort_program_set_mail_failed(program, mail); + size2 = 0; + } + + ret = size1 < size2 ? -1 : + (size1 > size2 ? 1 : 0); + break; + case MAIL_SORT_RELEVANCY: + index_sort_set_seq(program, mail, seq1); + if (index_sort_get_relevancy(mail, &float1) < 0) + index_sort_program_set_mail_failed(program, mail); + index_sort_set_seq(program, mail, seq2); + if (index_sort_get_relevancy(mail, &float2) < 0) + index_sort_program_set_mail_failed(program, mail); + + /* NOTE: higher relevancy is returned first, unlike with all + other number based sort keys */ + ret = float1 < float2 ? 1 : + (float1 > float2 ? -1 : 0); + break; + case MAIL_SORT_POP3_ORDER: + /* 32bit numbers would be enough, but since there is already + existing code for uoff_t in sizes, just use them. */ + index_sort_set_seq(program, mail, seq1); + if (index_sort_get_pop3_order(mail, &size1) < 0) + index_sort_program_set_mail_failed(program, mail); + index_sort_set_seq(program, mail, seq2); + if (index_sort_get_pop3_order(mail, &size2) < 0) + index_sort_program_set_mail_failed(program, mail); + + ret = size1 < size2 ? -1 : + (size1 > size2 ? 1 : 0); + break; + case MAIL_SORT_END: + return seq1 < seq2 ? -1 : + (seq1 > seq2 ? 1 : 0); + case MAIL_SORT_MASK: + case MAIL_SORT_FLAG_REVERSE: + i_unreached(); + } + + if (ret == 0) { + return index_sort_node_cmp_type(program, sort_program+1, + seq1, seq2); + } + + if ((*sort_program & MAIL_SORT_FLAG_REVERSE) != 0) + ret = ret < 0 ? 1 : -1; + return ret; +} diff --git a/src/lib-storage/index/index-sort.h b/src/lib-storage/index/index-sort.h new file mode 100644 index 0000000..b0e2770 --- /dev/null +++ b/src/lib-storage/index/index-sort.h @@ -0,0 +1,18 @@ +#ifndef INDEX_SORT_H +#define INDEX_SORT_H + +struct mail_search_sort_program; + +struct mail_search_sort_program * +index_sort_program_init(struct mailbox_transaction_context *t, + const enum mail_sort_type *sort_program); +int index_sort_program_deinit(struct mail_search_sort_program **program); + +void index_sort_list_add(struct mail_search_sort_program *program, + struct mail *mail); +void index_sort_list_finish(struct mail_search_sort_program *program); + +bool index_sort_list_next(struct mail_search_sort_program *program, + uint32_t *seq_r); + +#endif diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c new file mode 100644 index 0000000..9d751b9 --- /dev/null +++ b/src/lib-storage/index/index-status.c @@ -0,0 +1,344 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-cache.h" +#include "mail-index-modseq.h" +#include "mailbox-recent-flags.h" +#include "index-storage.h" + +static void +get_last_cached_seq(struct mailbox *box, uint32_t *last_cached_seq_r) +{ + const struct mail_index_header *hdr; + struct mail_cache_view *cache_view; + uint32_t seq; + + *last_cached_seq_r = 0; + if (!mail_cache_exists(box->cache)) + return; + + cache_view = mail_cache_view_open(box->cache, box->view); + hdr = mail_index_get_header(box->view); + for (seq = hdr->messages_count; seq > 0; seq--) { + if (mail_cache_field_exists_any(cache_view, seq)) { + *last_cached_seq_r = seq; + break; + } + } + mail_cache_view_close(&cache_view); +} + +int index_storage_get_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + if (items == 0) { + /* caller could have wanted only e.g. mailbox_status.have_* + flags */ + return 0; + } + + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) + return -1; + } + index_storage_get_open_status(box, items, status_r); + return 0; +} + +static unsigned int index_storage_count_pvt_unseen(struct mailbox *box) +{ + const struct mail_index_record *pvt_rec; + uint32_t shared_seq, pvt_seq, shared_count, pvt_count; + uint32_t shared_uid; + unsigned int unseen_count = 0; + + /* we can't trust private index to be up to date. we'll need to go + through the shared index and for each existing mail lookup its + private flags. if a mail doesn't exist in private index then its + flags are 0. */ + shared_count = mail_index_view_get_messages_count(box->view); + pvt_count = mail_index_view_get_messages_count(box->view_pvt); + shared_seq = pvt_seq = 1; + while (shared_seq <= shared_count && pvt_seq <= pvt_count) { + mail_index_lookup_uid(box->view, shared_seq, &shared_uid); + pvt_rec = mail_index_lookup(box->view_pvt, pvt_seq); + + if (shared_uid == pvt_rec->uid) { + if ((pvt_rec->flags & MAIL_SEEN) == 0) + unseen_count++; + shared_seq++; pvt_seq++; + } else if (shared_uid < pvt_rec->uid) { + shared_seq++; + } else { + pvt_seq++; + } + } + unseen_count += (shared_count+1) - shared_seq; + return unseen_count; +} + +static uint32_t index_storage_find_first_pvt_unseen_seq(struct mailbox *box) +{ + const struct mail_index_header *pvt_hdr; + const struct mail_index_record *pvt_rec; + uint32_t pvt_seq, pvt_count, shared_seq, seq2; + + pvt_count = mail_index_view_get_messages_count(box->view_pvt); + mail_index_lookup_first(box->view_pvt, 0, MAIL_SEEN, &pvt_seq); + if (pvt_seq == 0) + pvt_seq = pvt_count+1; + for (; pvt_seq <= pvt_count; pvt_seq++) { + pvt_rec = mail_index_lookup(box->view_pvt, pvt_seq); + if ((pvt_rec->flags & MAIL_SEEN) == 0 && + mail_index_lookup_seq(box->view, pvt_rec->uid, &shared_seq)) + return shared_seq; + } + /* if shared index has any messages that don't exist in private index, + the first of them is the first unseen message */ + pvt_hdr = mail_index_get_header(box->view_pvt); + if (mail_index_lookup_seq_range(box->view, + pvt_hdr->next_uid, (uint32_t)-1, + &shared_seq, &seq2)) + return shared_seq; + return 0; +} + +void index_storage_get_open_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + const struct mail_index_header *hdr; + + /* we can get most of the status items without any trouble */ + hdr = mail_index_get_header(box->view); + status_r->messages = hdr->messages_count; + if ((items & STATUS_RECENT) != 0) { + if ((box->flags & MAILBOX_FLAG_DROP_RECENT) != 0) { + /* recent flags are set and dropped by the previous + sync while index was locked. if we updated the + recent flags here we'd have a race condition. */ + i_assert(box->synced); + } else { + /* make sure recent count is set, in case we haven't + synced yet */ + index_sync_update_recent_count(box); + } + status_r->recent = mailbox_recent_flags_count(box); + i_assert(status_r->recent <= status_r->messages); + } + if ((items & STATUS_UNSEEN) != 0) { + if (box->view_pvt == NULL || + (mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) { + status_r->unseen = hdr->messages_count - + hdr->seen_messages_count; + } else { + status_r->unseen = index_storage_count_pvt_unseen(box); + } + } + status_r->uidvalidity = hdr->uid_validity; + status_r->uidnext = hdr->next_uid; + status_r->first_recent_uid = hdr->first_recent_uid; + if ((items & STATUS_HIGHESTMODSEQ) != 0) { + status_r->nonpermanent_modseqs = + mail_index_is_in_memory(box->index); + status_r->no_modseq_tracking = + !mail_index_have_modseq_tracking(box->index); + status_r->highest_modseq = + mail_index_modseq_get_highest(box->view); + if (status_r->highest_modseq == 0) { + /* modseqs not enabled yet, but we can't return 0 */ + status_r->highest_modseq = 1; + } + } + if ((items & STATUS_HIGHESTPVTMODSEQ) != 0 && box->view_pvt != NULL) { + status_r->highest_pvt_modseq = + mail_index_modseq_get_highest(box->view_pvt); + if (status_r->highest_pvt_modseq == 0) { + /* modseqs not enabled yet, but we can't return 0 */ + status_r->highest_pvt_modseq = 1; + } + } + + if ((items & STATUS_FIRST_UNSEEN_SEQ) != 0) { + if (box->view_pvt == NULL || + (mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) { + mail_index_lookup_first(box->view, 0, MAIL_SEEN, + &status_r->first_unseen_seq); + } else { + status_r->first_unseen_seq = + index_storage_find_first_pvt_unseen_seq(box); + } + } + if ((items & STATUS_LAST_CACHED_SEQ) != 0) + get_last_cached_seq(box, &status_r->last_cached_seq); + + if ((items & STATUS_KEYWORDS) != 0) + status_r->keywords = mail_index_get_keywords(box->index); + if ((items & STATUS_PERMANENT_FLAGS) != 0) { + if (!mailbox_is_readonly(box)) { + status_r->permanent_flags = MAIL_FLAGS_NONRECENT; + status_r->permanent_keywords = TRUE; + status_r->allow_new_keywords = + !box->disallow_new_keywords; + } + status_r->flags = MAIL_FLAGS_NONRECENT; + } +} + +static void +get_metadata_cache_fields(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + const struct mail_cache_field *fields; + enum mail_cache_decision_type dec; + ARRAY_TYPE(mailbox_cache_field) *cache_fields; + struct mailbox_cache_field *cf; + unsigned int i, count; + + fields = mail_cache_register_get_list(box->cache, + pool_datastack_create(), &count); + + cache_fields = t_new(ARRAY_TYPE(mailbox_cache_field), 1); + t_array_init(cache_fields, count); + for (i = 0; i < count; i++) { + dec = fields[i].decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + if (dec != MAIL_CACHE_DECISION_NO) { + cf = array_append_space(cache_fields); + cf->name = fields[i].name; + cf->decision = fields[i].decision; + cf->last_used = fields[i].last_used; + } + } + metadata_r->cache_fields = cache_fields; +} + +static void get_metadata_precache_fields(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + const struct mail_cache_field *fields; + unsigned int i, count; + enum mail_fetch_field cache = 0; + + fields = mail_cache_register_get_list(box->cache, + pool_datastack_create(), &count); + for (i = 0; i < count; i++) { + const char *name = fields[i].name; + + if (str_begins(name, "hdr.") || + strcmp(name, "date.sent") == 0 || + strcmp(name, "imap.envelope") == 0) + cache |= MAIL_FETCH_STREAM_HEADER; + else if (strcmp(name, "mime.parts") == 0 || + strcmp(name, "binary.parts") == 0 || + strcmp(name, "imap.body") == 0 || + strcmp(name, "imap.bodystructure") == 0 || + strcmp(name, "body.snippet") == 0) + cache |= MAIL_FETCH_STREAM_BODY; + else if (strcmp(name, "date.received") == 0) + cache |= MAIL_FETCH_RECEIVED_DATE; + else if (strcmp(name, "date.save") == 0) + cache |= MAIL_FETCH_SAVE_DATE; + else if (strcmp(name, "size.virtual") == 0) + cache |= MAIL_FETCH_VIRTUAL_SIZE; + else if (strcmp(name, "size.physical") == 0) + cache |= MAIL_FETCH_PHYSICAL_SIZE; + else if (strcmp(name, "pop3.uidl") == 0) + cache |= MAIL_FETCH_UIDL_BACKEND; + else if (strcmp(name, "pop3.order") == 0) + cache |= MAIL_FETCH_POP3_ORDER; + else if (strcmp(name, "guid") == 0) + cache |= MAIL_FETCH_GUID; + else if (strcmp(name, "flags") == 0) { + /* just ignore for now at least.. */ + } else + e_debug(box->event, + "Ignoring unknown cache field: %s", name); + } + metadata_r->precache_fields = cache; +} + +static int +index_mailbox_get_first_save_date(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + const struct mail_index_header *hdr; + struct mailbox_transaction_context *t; + struct mail *mail; + uint32_t seq; + int ret = -1; + + hdr = mail_index_get_header(box->view); + if (hdr->messages_count == 0) { + metadata_r->first_save_date = (time_t)-1; + return 0; + } + + t = mailbox_transaction_begin(box, 0, __func__); + mail = mail_alloc(t, 0, NULL); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_set_seq(mail, seq); + if (mail_get_save_date(mail, &metadata_r->first_save_date) >= 0) { + ret = 0; + break; + } + if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXPUNGED) { + /* failed */ + break; + } + } + mail_free(&mail); + (void)mailbox_transaction_commit(&t); + if (seq > hdr->messages_count) { + /* all messages were expunged after all */ + metadata_r->first_save_date = (time_t)-1; + return 0; + } + return ret; +} + +int index_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + /* handle items that don't require opening the mailbox */ + if ((items & MAILBOX_METADATA_BACKEND_NAMESPACE) != 0) { + metadata_r->backend_ns_prefix = ""; + metadata_r->backend_ns_type = + mailbox_list_get_namespace(box->list)->type; + items &= ENUM_NEGATE(MAILBOX_METADATA_BACKEND_NAMESPACE); + } + if (items == 0) + return 0; + + /* handle items that require opening the mailbox */ + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + if (!box->synced && (items & MAILBOX_METADATA_SYNC_ITEMS) != 0) { + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) + return -1; + } + + if ((items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0) { + if (index_mailbox_get_virtual_size(box, metadata_r) < 0) + return -1; + } + if ((items & MAILBOX_METADATA_PHYSICAL_SIZE) != 0) { + if (index_mailbox_get_physical_size(box, metadata_r) < 0) + return -1; + } + if ((items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0) { + if (index_mailbox_get_first_save_date(box, metadata_r) < 0) + return -1; + } + if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0) + get_metadata_cache_fields(box, metadata_r); + if ((items & MAILBOX_METADATA_PRECACHE_FIELDS) != 0) + get_metadata_precache_fields(box, metadata_r); + return 0; +} diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c new file mode 100644 index 0000000..f699dfb --- /dev/null +++ b/src/lib-storage/index/index-storage.c @@ -0,0 +1,1296 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "ioloop.h" +#include "str.h" +#include "str-sanitize.h" +#include "mkdir-parents.h" +#include "dict.h" +#include "fs-api.h" +#include "message-header-parser.h" +#include "mail-index-alloc-cache.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mailbox-log.h" +#include "mailbox-list-private.h" +#include "mail-search-build.h" +#include "index-storage.h" +#include "index-mail.h" +#include "index-attachment.h" +#include "index-thread-private.h" +#include "index-mailbox-size.h" + +#include <time.h> +#include <unistd.h> +#include <sys/stat.h> + +#define LOCK_NOTIFY_INTERVAL 30 + +struct index_storage_module index_storage_module = + MODULE_CONTEXT_INIT(&mail_storage_module_register); + +static void set_cache_decisions(struct mail_cache *cache, + const char *set, const char *fields, + enum mail_cache_decision_type dec) +{ + struct mail_cache_field field; + const char *const *arr; + unsigned int idx; + + if (fields == NULL || *fields == '\0') + return; + + for (arr = t_strsplit_spaces(fields, " ,"); *arr != NULL; arr++) { + const char *name = *arr; + + idx = mail_cache_register_lookup(cache, name); + if (idx != UINT_MAX) { + field = *mail_cache_register_get_field(cache, idx); + } else if (strncasecmp(name, "hdr.", 4) == 0) { + /* Do some sanity checking for the header name. Mainly + to make sure there aren't UTF-8 characters that look + like their ASCII equivalents or are completely + invisible. */ + if (message_header_name_is_valid(name+4)) { + i_zero(&field); + field.name = name; + field.type = MAIL_CACHE_FIELD_HEADER; + } else { + i_error("%s: Header name '%s' has invalid character, ignoring", + set, name); + continue; + } + } else { + i_error("%s: Unknown cache field name '%s', ignoring", + set, *arr); + continue; + } + + field.decision = dec; + mail_cache_register_fields(cache, &field, 1); + } +} + +static void index_cache_register_defaults(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const struct mail_storage_settings *set = box->storage->set; + struct mail_cache *cache = box->cache; + + ibox->cache_fields = i_malloc(sizeof(global_cache_fields)); + memcpy(ibox->cache_fields, global_cache_fields, + sizeof(global_cache_fields)); + mail_cache_register_fields(cache, ibox->cache_fields, + MAIL_INDEX_CACHE_FIELD_COUNT); + + if (strcmp(set->mail_never_cache_fields, "*") == 0) { + /* all caching disabled for now */ + box->mail_cache_disabled = TRUE; + return; + } + + set_cache_decisions(cache, "mail_cache_fields", + set->mail_cache_fields, + MAIL_CACHE_DECISION_TEMP); + set_cache_decisions(cache, "mail_always_cache_fields", + set->mail_always_cache_fields, + MAIL_CACHE_DECISION_YES | + MAIL_CACHE_DECISION_FORCED); + set_cache_decisions(cache, "mail_never_cache_fields", + set->mail_never_cache_fields, + MAIL_CACHE_DECISION_NO | + MAIL_CACHE_DECISION_FORCED); +} + +void index_storage_lock_notify(struct mailbox *box, + enum mailbox_lock_notify_type notify_type, + unsigned int secs_left) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + struct mail_storage *storage = box->storage; + const char *str; + time_t now; + + /* if notify type changes, print the message immediately */ + now = time(NULL); + if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE || + ibox->last_notify_type == notify_type) { + if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE && + notify_type == MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE) { + /* first override notification, show it */ + } else { + if (now < ibox->next_lock_notify || secs_left < 15) + return; + } + } + + ibox->next_lock_notify = now + LOCK_NOTIFY_INTERVAL; + ibox->last_notify_type = notify_type; + + switch (notify_type) { + case MAILBOX_LOCK_NOTIFY_NONE: + break; + case MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT: + if (storage->callbacks.notify_no == NULL) + break; + + str = t_strdup_printf("Mailbox is locked, will abort in " + "%u seconds", secs_left); + storage->callbacks. + notify_no(box, str, storage->callback_context); + break; + case MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE: + if (storage->callbacks.notify_ok == NULL) + break; + + str = t_strdup_printf("Stale mailbox lock file detected, " + "will override in %u seconds", secs_left); + storage->callbacks. + notify_ok(box, str, storage->callback_context); + break; + } +} + +void index_storage_lock_notify_reset(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL; + ibox->last_notify_type = MAILBOX_LOCK_NOTIFY_NONE; +} + +static int +index_mailbox_alloc_index(struct mailbox *box, struct mail_index **index_r) +{ + const char *index_dir, *mailbox_path; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &mailbox_path) < 0) + return -1; + if ((box->flags & MAILBOX_FLAG_NO_INDEX_FILES) != 0 || + mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, + &index_dir) <= 0) + index_dir = NULL; + /* Note that this may cause box->event to live longer than box */ + *index_r = mail_index_alloc_cache_get(box->event, + mailbox_path, index_dir, + box->index_prefix); + return 0; +} + +int index_storage_mailbox_exists(struct mailbox *box, bool auto_boxes, + enum mailbox_existence *existence_r) +{ + if (auto_boxes && mailbox_is_autocreated(box)) { + *existence_r = MAILBOX_EXISTENCE_SELECT; + return 0; + } + + return index_storage_mailbox_exists_full(box, NULL, existence_r); +} + +int index_storage_mailbox_exists_full(struct mailbox *box, const char *subdir, + enum mailbox_existence *existence_r) +{ + struct stat st; + const char *path, *path2, *index_path; + int ret; + + /* see if it's selectable */ + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path); + if (ret < 0) { + if (mailbox_get_last_mail_error(box) != MAIL_ERROR_NOTFOUND) + return -1; + *existence_r = MAILBOX_EXISTENCE_NONE; + return 0; + } + if (ret == 0) { + /* no mailboxes in this storage? */ + *existence_r = MAILBOX_EXISTENCE_NONE; + return 0; + } + + ret = (subdir != NULL || !box->list->set.iter_from_index_dir) ? 0 : + mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &index_path); + if (ret > 0 && strcmp(path, index_path) != 0) { + /* index directory is different - prefer looking it up first + since it might be on a faster storage. since the directory + itself exists also for \NoSelect mailboxes, we'll need to + check the dovecot.index.log existence. */ + index_path = t_strconcat(index_path, "/", box->index_prefix, + ".log", NULL); + if (stat(index_path, &st) == 0) { + *existence_r = MAILBOX_EXISTENCE_SELECT; + return 0; + } + } + + if (subdir != NULL) + path = t_strconcat(path, "/", subdir, NULL); + if (stat(path, &st) == 0) { + *existence_r = MAILBOX_EXISTENCE_SELECT; + return 0; + } + if (!ENOTFOUND(errno) && errno != EACCES) { + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return -1; + } + + /* see if it's non-selectable */ + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_DIR, &path2) <= 0 || + (strcmp(path, path2) != 0 && stat(path2, &st) == 0)) { + *existence_r = MAILBOX_EXISTENCE_NOSELECT; + return 0; + } + *existence_r = MAILBOX_EXISTENCE_NONE; + return 0; +} + +int index_storage_mailbox_alloc_index(struct mailbox *box) +{ + const char *cache_dir; + + if (box->index != NULL) + return 0; + + if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX) < 0) + return -1; + if (index_mailbox_alloc_index(box, &box->index) < 0) + return -1; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX_CACHE, + &cache_dir) > 0) { + if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX_CACHE) < 0) + return -1; + mail_index_set_cache_dir(box->index, cache_dir); + } + mail_index_set_fsync_mode(box->index, + box->storage->set->parsed_fsync_mode, 0); + mail_index_set_lock_method(box->index, + box->storage->set->parsed_lock_method, + mail_storage_get_lock_timeout(box->storage, UINT_MAX)); + + const struct mail_storage_settings *set = box->storage->set; + struct mail_index_optimization_settings optimization_set = { + .index = { + .rewrite_min_log_bytes = set->mail_index_rewrite_min_log_bytes, + .rewrite_max_log_bytes = set->mail_index_rewrite_max_log_bytes, + }, + .log = { + .min_size = set->mail_index_log_rotate_min_size, + .max_size = set->mail_index_log_rotate_max_size, + .min_age_secs = set->mail_index_log_rotate_min_age, + .log2_max_age_secs = set->mail_index_log2_max_age, + }, + .cache = { + .unaccessed_field_drop_secs = set->mail_cache_unaccessed_field_drop, + .record_max_size = set->mail_cache_record_max_size, + .max_size = set->mail_cache_max_size, + .purge_min_size = set->mail_cache_purge_min_size, + .purge_delete_percentage = set->mail_cache_purge_delete_percentage, + .purge_continued_percentage = set->mail_cache_purge_continued_percentage, + .purge_header_continue_count = set->mail_cache_purge_header_continue_count, + }, + }; + mail_index_set_optimization_settings(box->index, &optimization_set); + return 0; +} + +int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + enum mail_index_open_flags index_flags; + int ret; + + i_assert(!box->opened); + + index_flags = ibox->index_flags; + if (move_to_memory) + index_flags &= ENUM_NEGATE(MAIL_INDEX_OPEN_FLAG_CREATE); + + if (index_storage_mailbox_alloc_index(box) < 0) + return -1; + + /* make sure mail_index_set_permissions() has been called */ + (void)mailbox_get_permissions(box); + + ret = mail_index_open(box->index, index_flags); + if (ret <= 0 || move_to_memory) { + if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) { + i_assert(ret <= 0); + mailbox_set_index_error(box); + return -1; + } + + if (mail_index_move_to_memory(box->index) < 0) { + /* try opening once more. it should be created + directly into memory now. */ + if (mail_index_open_or_create(box->index, + index_flags) < 0) + i_panic("in-memory index creation failed"); + } + } + if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) { + if (mail_index_is_in_memory(box->index)) { + mailbox_set_critical(box, + "Couldn't create index file"); + mail_index_close(box->index); + return -1; + } + } + + if ((box->flags & MAILBOX_FLAG_OPEN_DELETED) == 0) { + if (mail_index_is_deleted(box->index)) { + mailbox_set_deleted(box); + mail_index_close(box->index); + return -1; + } + } + if ((box->flags & MAILBOX_FLAG_FSCK) != 0) { + if (mail_index_fsck(box->index) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + + box->cache = mail_index_get_cache(box->index); + index_cache_register_defaults(box); + box->view = mail_index_view_open(box->index); + ibox->keyword_names = mail_index_get_keywords(box->index); + box->vsize_hdr_ext_id = + mail_index_ext_register(box->index, "hdr-vsize", + sizeof(struct mailbox_index_vsize), 0, + sizeof(uint64_t)); + box->pop3_uidl_hdr_ext_id = + mail_index_ext_register(box->index, "hdr-pop3-uidl", + sizeof(struct mailbox_index_pop3_uidl), 0, 0); + box->box_name_hdr_ext_id = + mail_index_ext_register(box->index, "box-name", 0, 0, 0); + + box->box_last_rename_stamp_ext_id = + mail_index_ext_register(box->index, "last-rename-stamp", + sizeof(uint32_t), 0, sizeof(uint32_t)); + box->mail_vsize_ext_id = mail_index_ext_register(box->index, "vsize", 0, + sizeof(uint32_t), + sizeof(uint32_t)); + + box->opened = TRUE; + + if ((box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0) + mail_index_modseq_enable(box->index); + + index_thread_mailbox_opened(box); + hook_mailbox_opened(box); + return 0; +} + +void index_storage_mailbox_alloc(struct mailbox *box, const char *vname, + enum mailbox_flags flags, + const char *index_prefix) +{ + static unsigned int mailbox_generation_sequence = 0; + struct index_mailbox_context *ibox; + + i_assert(vname != NULL); + + box->generation_sequence = ++mailbox_generation_sequence; + box->vname = p_strdup(box->pool, vname); + box->name = p_strdup(box->pool, + mailbox_list_get_storage_name(box->list, vname)); + box->flags = flags; + box->index_prefix = p_strdup(box->pool, index_prefix); + box->event = event_create(box->storage->event); + event_add_category(box->event, &event_category_mailbox); + event_add_str(box->event, "mailbox", box->vname); + event_set_append_log_prefix(box->event, + t_strdup_printf("Mailbox %s: ", str_sanitize(box->vname, 128))); + + p_array_init(&box->search_results, box->pool, 16); + array_create(&box->module_contexts, + box->pool, sizeof(void *), 5); + + ibox = p_new(box->pool, struct index_mailbox_context, 1); + ibox->list_index_sync_ext_id = (uint32_t)-1; + ibox->index_flags = MAIL_INDEX_OPEN_FLAG_CREATE | + mail_storage_settings_to_index_flags(box->storage->set); + if ((box->flags & MAILBOX_FLAG_SAVEONLY) != 0) + ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_SAVEONLY; + if (event_want_debug(box->event)) + ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_DEBUG; + ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL; + MODULE_CONTEXT_SET(box, index_storage_module, ibox); + + box->inbox_user = strcmp(box->name, "INBOX") == 0 && + (box->list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0; + box->inbox_any = strcmp(box->name, "INBOX") == 0 && + (box->list->ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0; +} + +int index_storage_mailbox_enable(struct mailbox *box, + enum mailbox_feature feature) +{ + if ((feature & MAILBOX_FEATURE_CONDSTORE) != 0) { + box->enabled_features |= MAILBOX_FEATURE_CONDSTORE; + if (box->opened) + mail_index_modseq_enable(box->index); + } + return 0; +} + +void index_storage_mailbox_close(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + mailbox_watch_remove_all(box); + i_stream_unref(&box->input); + + if (box->view_pvt != NULL) + mail_index_view_close(&box->view_pvt); + if (box->index_pvt != NULL) + mail_index_close(box->index_pvt); + if (box->view != NULL) { + mail_index_view_close(&box->view); + mail_index_close(box->index); + } + box->cache = NULL; + + ibox->keyword_names = NULL; + i_free_and_null(ibox->cache_fields); + + ibox->sync_last_check = 0; +} + +static void index_storage_mailbox_unref_indexes(struct mailbox *box) +{ + if (box->index_pvt != NULL) + mail_index_alloc_cache_unref(&box->index_pvt); + if (box->index != NULL) + mail_index_alloc_cache_unref(&box->index); +} + +void index_storage_mailbox_free(struct mailbox *box) +{ + index_storage_mailbox_unref_indexes(box); + event_unref(&box->event); +} + +static void +index_storage_mailbox_update_cache(struct mailbox *box, + const struct mailbox_update *update) +{ + const struct mailbox_cache_field *updates = update->cache_updates; + ARRAY(struct mail_cache_field) new_fields; + const struct mail_cache_field *old_fields; + struct mail_cache_field field; + unsigned int i, j, old_count; + + old_fields = mail_cache_register_get_list(box->cache, + pool_datastack_create(), + &old_count); + + /* There shouldn't be many fields, so don't worry about O(n^2). */ + t_array_init(&new_fields, 32); + for (i = 0; updates[i].name != NULL; i++) { + /* see if it's an existing field */ + for (j = 0; j < old_count; j++) { + if (strcmp(updates[i].name, old_fields[j].name) == 0) + break; + } + if (j != old_count) { + field = old_fields[j]; + } else if (str_begins(updates[i].name, "hdr.")) { + /* new header */ + i_zero(&field); + field.name = updates[i].name; + field.type = MAIL_CACHE_FIELD_HEADER; + } else { + /* new unknown field. we can't do anything about + this since we don't know its type */ + continue; + } + field.decision = updates[i].decision; + if (updates[i].last_used != (time_t)-1) + field.last_used = updates[i].last_used; + array_push_back(&new_fields, &field); + } + if (array_count(&new_fields) > 0) { + mail_cache_register_fields(box->cache, + array_front_modifiable(&new_fields), + array_count(&new_fields)); + } +} + +static int +index_storage_mailbox_update_pvt(struct mailbox *box, + const struct mailbox_update *update) +{ + struct mail_index_transaction *trans; + struct mail_index_view *view; + int ret; + + if ((ret = mailbox_open_index_pvt(box)) <= 0) + return ret; + + mail_index_refresh(box->index_pvt); + view = mail_index_view_open(box->index_pvt); + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + if (update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(view) < update->min_highest_pvt_modseq) { + mail_index_modseq_enable(box->index_pvt); + mail_index_update_highest_modseq(trans, + update->min_highest_pvt_modseq); + } + + if ((ret = mail_index_transaction_commit(&trans)) < 0) + mailbox_set_index_error(box); + mail_index_view_close(&view); + return ret; +} + +int index_storage_mailbox_update_common(struct mailbox *box, + const struct mailbox_update *update) +{ + int ret = 0; + + if (update->cache_updates != NULL) + index_storage_mailbox_update_cache(box, update); + + if (update->min_highest_pvt_modseq != 0) { + if (index_storage_mailbox_update_pvt(box, update) < 0) + ret = -1; + } + return ret; +} + +int index_storage_mailbox_update(struct mailbox *box, + const struct mailbox_update *update) +{ + const struct mail_index_header *hdr; + struct mail_index_view *view; + struct mail_index_transaction *trans; + int ret; + + if (mailbox_open(box) < 0) + return -1; + + /* make sure we get the latest index info */ + mail_index_refresh(box->index); + view = mail_index_view_open(box->index); + hdr = mail_index_get_header(view); + + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + if (update->uid_validity != 0 && + hdr->uid_validity != update->uid_validity) { + uint32_t uid_validity = update->uid_validity; + + if (hdr->uid_validity != 0) { + /* UIDVALIDITY change requires index to be reset */ + mail_index_reset(trans); + } + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + if (update->min_next_uid != 0 && + hdr->next_uid < update->min_next_uid) { + uint32_t next_uid = update->min_next_uid; + + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &next_uid, sizeof(next_uid), FALSE); + } + if (update->min_first_recent_uid != 0 && + hdr->first_recent_uid < update->min_first_recent_uid) { + uint32_t first_recent_uid = update->min_first_recent_uid; + + mail_index_update_header(trans, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + if (update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(view) < update->min_highest_modseq) { + mail_index_modseq_enable(box->index); + mail_index_update_highest_modseq(trans, + update->min_highest_modseq); + } + + if ((ret = mail_index_transaction_commit(&trans)) < 0) + mailbox_set_index_error(box); + mail_index_view_close(&view); + return ret < 0 ? -1 : + index_storage_mailbox_update_common(box, update); +} + +int index_storage_mailbox_create(struct mailbox *box, bool directory) +{ + const char *path, *p; + enum mailbox_list_path_type type; + enum mailbox_existence existence; + bool create_parent_dir; + int ret; + + if ((box->list->props & MAILBOX_LIST_PROP_NO_NOSELECT) != 0) { + /* Layout doesn't support creating \NoSelect mailboxes. + Switch to creating a selectable mailbox. */ + directory = FALSE; + } + + type = directory ? MAILBOX_LIST_PATH_TYPE_DIR : + MAILBOX_LIST_PATH_TYPE_MAILBOX; + if ((ret = mailbox_get_path_to(box, type, &path)) < 0) + return -1; + if (ret == 0) { + /* layout=none */ + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "Mailbox creation not supported"); + return -1; + } + create_parent_dir = !directory && + (box->list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) != 0; + if (create_parent_dir) { + /* we only need to make sure that the parent directory exists */ + p = strrchr(path, '/'); + if (p == NULL) + return 1; + path = t_strdup_until(path, p); + } + + if ((ret = mailbox_mkdir(box, path, type)) < 0) + return -1; + if (box->list->set.iter_from_index_dir) { + /* need to also create the directory to index path or + iteration won't find it. */ + int ret2; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0) + i_unreached(); + if ((ret2 = mailbox_mkdir(box, path, type)) < 0) + return -1; + if (ret == 0 && ret2 > 0) { + /* finish partial creation: existed in mail directory, + but not in index directory. */ + ret = 1; + } + } + mailbox_refresh_permissions(box); + if (ret == 0) { + /* directory already exists */ + if (create_parent_dir) + return 1; + if (!directory && *box->list->set.mailbox_dir_name == '\0') { + /* For example: layout=fs, path=~/Maildir/foo + might itself exist, but does it have the + cur|new|tmp subdirs? */ + if (mailbox_exists(box, FALSE, &existence) < 0) + return -1; + if (existence != MAILBOX_EXISTENCE_SELECT) + return 1; + } else if (!box->storage->rebuilding_list_index) { + /* ignore existing location if we are recovering list index */ + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } + } + + if (directory) { + /* we only wanted to create the directory and it's done now */ + return 0; + } + /* the caller should still create the mailbox */ + return 1; +} + +int index_storage_mailbox_delete_dir(struct mailbox *box, bool mailbox_deleted) +{ + guid_128_t dir_sha128; + enum mail_error error; + + if (mailbox_list_delete_dir(box->list, box->name) == 0) + return 0; + + mailbox_list_get_last_error(box->list, &error); + if (error != MAIL_ERROR_NOTFOUND || !mailbox_deleted) { + mail_storage_copy_list_error(box->storage, box->list); + return -1; + } + /* failed directory deletion, but mailbox deletion succeeded. + this was probably maildir++, which internally deleted the + directory as well. add changelog record about that too. */ + mailbox_name_get_sha128(box->vname, dir_sha128); + mailbox_list_add_change(box->list, MAILBOX_LOG_RECORD_DELETE_DIR, + dir_sha128); + return 0; +} + +static int +mailbox_delete_all_attributes(struct mailbox_transaction_context *t, + enum mail_attribute_type type) +{ + struct mailbox_attribute_iter *iter; + const char *key; + int ret = 0; + bool inbox = t->box->inbox_any; + + iter = mailbox_attribute_iter_init(t->box, type, ""); + while ((key = mailbox_attribute_iter_next(iter)) != NULL) { + if (inbox && + str_begins(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) + continue; + + if (mailbox_attribute_unset(t, type, key) < 0) { + if (mailbox_get_last_mail_error(t->box) != MAIL_ERROR_NOTPOSSIBLE) { + ret = -1; + break; + } + } + } + if (mailbox_attribute_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int mailbox_expunge_all_data(struct mailbox *box) +{ + struct mail_search_context *ctx; + struct mailbox_transaction_context *t; + struct mail *mail; + struct mail_search_args *search_args; + + (void)mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ); + + t = mailbox_transaction_begin(box, 0, __func__); + + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + ctx = mailbox_search_init(t, search_args, NULL, 0, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(ctx, &mail)) + mail_expunge(mail); + + if (mailbox_search_deinit(&ctx) < 0) { + mailbox_transaction_rollback(&t); + return -1; + } + + if (mailbox_delete_all_attributes(t, MAIL_ATTRIBUTE_TYPE_PRIVATE) < 0 || + mailbox_delete_all_attributes(t, MAIL_ATTRIBUTE_TYPE_SHARED) < 0) { + mailbox_transaction_rollback(&t); + return -1; + } + if (mailbox_transaction_commit(&t) < 0) + return -1; + /* sync to actually perform the expunges */ + return mailbox_sync(box, 0); +} + +int index_storage_mailbox_delete_pre(struct mailbox *box) +{ + struct mailbox_status status; + + if (!box->opened) { + /* \noselect mailbox, try deleting only the directory */ + if (index_storage_mailbox_delete_dir(box, FALSE) == 0) + return 0; + if (mailbox_is_autocreated(box)) { + /* Return success when trying to delete autocreated + mailbox. The client sees it as existing, so we + shouldn't be returning an error. */ + return 0; + } + return -1; + } + + if ((box->list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) == 0) { + /* specifically support symlinked shared mailboxes. a deletion + will simply remove the symlink, not actually expunge any + mails */ + if (mailbox_list_delete_symlink(box->list, box->name) == 0) + return 0; + } + + /* we can't easily atomically delete all mails and the mailbox. so: + 1) expunge all mails + 2) mark the mailbox deleted (modifications after this will fail) + 3) check if a race condition between 1) and 2) added any mails: + yes) abort and undelete mailbox + no) finish deleting the mailbox + */ + + if (!box->deleting_must_be_empty) { + if (mailbox_expunge_all_data(box) < 0) + return -1; + } + if (mailbox_mark_index_deleted(box, TRUE) < 0) + return -1; + + if (!box->delete_skip_empty_check || box->deleting_must_be_empty) { + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) + return -1; + mailbox_get_open_status(box, STATUS_MESSAGES, &status); + if (status.messages == 0) + ; + else if (box->deleting_must_be_empty) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox isn't empty"); + return -1; + } else { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "New mails were added to mailbox during deletion"); + return -1; + } + } + return 1; +} + +int index_storage_mailbox_delete_post(struct mailbox *box) +{ + struct mailbox_metadata metadata; + int ret_guid; + + ret_guid = mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata); + + /* Make sure the indexes are closed before trying to delete the + directory that contains them. It can still fail with some NFS + implementations if indexes are opened by another session, but + that can't really be helped. */ + mailbox_close(box); + index_storage_mailbox_unref_indexes(box); + mail_index_alloc_cache_destroy_unrefed(); + + if (box->list->v.delete_mailbox(box->list, box->name) < 0) { + mail_storage_copy_list_error(box->storage, box->list); + return -1; + } + + if (ret_guid == 0) { + mailbox_list_add_change(box->list, + MAILBOX_LOG_RECORD_DELETE_MAILBOX, + metadata.guid); + } + if (index_storage_mailbox_delete_dir(box, TRUE) < 0) { + if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXISTS) + return -1; + /* we deleted the mailbox, but couldn't delete the directory + because it has children. that's not an error. */ + } + return 0; +} + +int index_storage_mailbox_delete(struct mailbox *box) +{ + int ret; + + if ((ret = index_storage_mailbox_delete_pre(box)) <= 0) + return ret; + /* mails have been now successfully deleted. some mailbox formats may + at this point do some other deletion that is required for it. + the _post() deletion will close the index and delete the + directory. */ + return index_storage_mailbox_delete_post(box); +} + +int index_storage_mailbox_rename(struct mailbox *src, struct mailbox *dest) +{ + guid_128_t guid; + + if (src->list->v.rename_mailbox(src->list, src->name, + dest->list, dest->name) < 0) { + mail_storage_copy_list_error(src->storage, src->list); + return -1; + } + + if (mailbox_open(dest) == 0) { + struct mail_index_transaction *t = + mail_index_transaction_begin(dest->view, 0); + + uint32_t stamp = ioloop_time; + + mail_index_update_header_ext(t, dest->box_last_rename_stamp_ext_id, + 0, &stamp, sizeof(stamp)); + + /* can't do much if this fails anyways */ + (void)mail_index_transaction_commit(&t); + } + + /* we'll track mailbox names, instead of GUIDs. We may be renaming a + non-selectable mailbox (directory), which doesn't even have a GUID */ + mailbox_name_get_sha128(dest->vname, guid); + mailbox_list_add_change(src->list, MAILBOX_LOG_RECORD_RENAME, guid); + return 0; +} + +int index_mailbox_view_update_last_temp_file_scan(struct mail_index_view *view) +{ + uint32_t last_temp_file_scan = ioloop_time; + struct mail_index_transaction *trans = + mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_header(trans, + offsetof(struct mail_index_header, last_temp_file_scan), + &last_temp_file_scan, sizeof(last_temp_file_scan), TRUE); + return mail_index_transaction_commit(&trans); +} + +int index_mailbox_update_last_temp_file_scan(struct mailbox *box) +{ + if (index_mailbox_view_update_last_temp_file_scan(box->view) < 0) { + mailbox_set_index_error(box); + return -1; + } + return 0; +} + +bool index_storage_is_readonly(struct mailbox *box) +{ + return (box->flags & MAILBOX_FLAG_READONLY) != 0; +} + +bool index_storage_is_inconsistent(struct mailbox *box) +{ + return box->view != NULL && + mail_index_view_is_inconsistent(box->view); +} + +void index_save_context_free(struct mail_save_context *ctx) +{ + i_assert(ctx->dest_mail != NULL); + + index_mail_save_finish(ctx); + if (ctx->data.keywords != NULL) + mailbox_keywords_unref(&ctx->data.keywords); + i_free_and_null(ctx->data.from_envelope); + i_free_and_null(ctx->data.guid); + i_free_and_null(ctx->data.pop3_uidl); + index_attachment_save_free(ctx); + i_zero(&ctx->data); + + ctx->unfinished = FALSE; +} + +static void +mail_copy_cache_field(struct mail_save_context *ctx, struct mail *src_mail, + uint32_t dest_seq, const char *name, buffer_t *buf) +{ + struct mailbox_transaction_context *dest_trans = ctx->transaction; + const struct mail_cache_field *dest_field; + unsigned int src_field_idx, dest_field_idx; + uint32_t t; + bool add = FALSE; + + src_field_idx = mail_cache_register_lookup(src_mail->box->cache, name); + i_assert(src_field_idx != UINT_MAX); + + dest_field_idx = mail_cache_register_lookup(dest_trans->box->cache, name); + if (dest_field_idx == UINT_MAX) { + /* unknown field */ + return; + } + dest_field = mail_cache_register_get_field(dest_trans->box->cache, + dest_field_idx); + if ((dest_field->decision & + ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) == MAIL_CACHE_DECISION_NO) { + /* field not wanted in destination mailbox */ + return; + } + + buffer_set_used_size(buf, 0); + if (strcmp(name, "date.save") == 0) { + /* save date must update when mail is copied */ + t = ioloop_time; + buffer_append(buf, &t, sizeof(t)); + add = TRUE; + } else if (mail_cache_lookup_field(src_mail->transaction->cache_view, buf, + src_mail->seq, src_field_idx) <= 0) { + /* error / not found */ + buffer_set_used_size(buf, 0); + } else { + if (strcmp(name, "size.physical") == 0 || + strcmp(name, "size.virtual") == 0) { + /* FIXME: until mail_cache_lookup() can read unwritten + cached data from buffer, we'll do this optimization + to make quota plugin's work faster */ + struct index_mail *imail = + INDEX_MAIL(ctx->dest_mail); + uoff_t size; + + i_assert(buf->used == sizeof(size)); + memcpy(&size, buf->data, sizeof(size)); + if (strcmp(name, "size.physical") == 0) + imail->data.physical_size = size; + else + imail->data.virtual_size = size; + } + /* NOTE: we'll want to add also nonexistent headers, which + will keep the buf empty */ + add = TRUE; + } + if (add) { + mail_cache_add(dest_trans->cache_trans, dest_seq, + dest_field_idx, buf->data, buf->used); + } +} + +static void +index_copy_vsize_extension(struct mail_save_context *ctx, + struct mail *src_mail, uint32_t dest_seq) +{ + const uint32_t *vsizep; + bool expunged ATTR_UNUSED; + + vsizep = index_mail_get_vsize_extension(src_mail); + if (vsizep == NULL || *vsizep == 0) + return; + uint32_t vsize = *vsizep; + + if (vsize < (uint32_t)-1) { + /* copy the vsize record to the destination index */ + mail_index_update_ext(ctx->transaction->itrans, dest_seq, + ctx->transaction->box->mail_vsize_ext_id, + &vsize, NULL); + } +} + +void index_copy_cache_fields(struct mail_save_context *ctx, + struct mail *src_mail, uint32_t dest_seq) +{ + T_BEGIN { + struct mailbox_metadata src_metadata, dest_metadata; + const struct mailbox_cache_field *field; + buffer_t *buf; + + if (mailbox_get_metadata(src_mail->box, + MAILBOX_METADATA_CACHE_FIELDS, + &src_metadata) < 0) + i_unreached(); + /* the only reason we're doing the destination lookup is to + make sure that the cache file is opened and the cache + decisions are up to date */ + if (mailbox_get_metadata(ctx->transaction->box, + MAILBOX_METADATA_CACHE_FIELDS, + &dest_metadata) < 0) + i_unreached(); + + buf = t_buffer_create(1024); + array_foreach(src_metadata.cache_fields, field) { + mail_copy_cache_field(ctx, src_mail, dest_seq, + field->name, buf); + } + index_copy_vsize_extension(ctx, src_mail, dest_seq); + } T_END; +} + +int index_storage_set_subscribed(struct mailbox *box, bool set) +{ + struct mail_namespace *ns; + struct mailbox_list *list = box->list; + const char *subs_name; + guid_128_t guid; + + if ((list->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) != 0) + subs_name = box->name; + else { + /* subscriptions=no namespace, find another one where we can + add the subscription to */ + ns = mail_namespace_find_subscribable(list->ns->user->namespaces, + box->vname); + if (ns == NULL) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "This namespace has no subscriptions"); + return -1; + } + /* use <orig ns prefix><orig storage name> as the + subscription name */ + subs_name = t_strconcat(list->ns->prefix, box->name, NULL); + /* drop the common prefix (typically there isn't one) */ + i_assert(str_begins(subs_name, ns->prefix)); + subs_name += strlen(ns->prefix); + + list = ns->list; + } + if (mailbox_list_set_subscribed(list, subs_name, set) < 0) { + mail_storage_copy_list_error(box->storage, list); + return -1; + } + + /* subscriptions are about names, not about mailboxes. it's possible + to have a subscription to nonexistent mailbox. renames also don't + change subscriptions. so instead of using actual GUIDs, we'll use + hash of the name. */ + mailbox_name_get_sha128(box->vname, guid); + mailbox_list_add_change(list, set ? MAILBOX_LOG_RECORD_SUBSCRIBE : + MAILBOX_LOG_RECORD_UNSUBSCRIBE, guid); + return 0; +} + +void index_storage_destroy(struct mail_storage *storage) +{ + if (storage->_shared_attr_dict != NULL) { + dict_wait(storage->_shared_attr_dict); + dict_deinit(&storage->_shared_attr_dict); + } + fs_unref(&storage->mailboxes_fs); +} + +static void index_storage_expunging_init(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + if (ibox->vsize_update != NULL) + return; + + ibox->vsize_update = index_mailbox_vsize_update_init(box); + if (!index_mailbox_vsize_want_updates(ibox->vsize_update) || + !index_mailbox_vsize_update_wait_lock(ibox->vsize_update)) + index_mailbox_vsize_update_deinit(&ibox->vsize_update); +} + +void index_storage_expunging_deinit(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + if (ibox->vsize_update != NULL) + index_mailbox_vsize_update_deinit(&ibox->vsize_update); +} + +static bool index_storage_expunging_want_updates(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + bool ret; + + i_assert(ibox->vsize_update == NULL); + + ibox->vsize_update = index_mailbox_vsize_update_init(box); + ret = index_mailbox_vsize_want_updates(ibox->vsize_update); + index_mailbox_vsize_update_deinit(&ibox->vsize_update); + return ret; +} + +int index_storage_expunged_sync_begin(struct mailbox *box, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + enum mail_index_sync_flags flags) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + int ret; + + /* try to avoid locking vsize updates by checking if we see any + expunges */ + if (mail_index_sync_have_any_expunges(box->index)) + index_storage_expunging_init(box); + + ret = mail_index_sync_begin(box->index, ctx_r, view_r, + trans_r, flags); + if (ret <= 0) { + if (ret < 0) + mailbox_set_index_error(box); + index_storage_expunging_deinit(box); + return ret; + } + if (ibox->vsize_update == NULL && + mail_index_sync_has_expunges(*ctx_r) && + index_storage_expunging_want_updates(box)) { + /* race condition - need to abort the sync and retry with + the vsize locked */ + mail_index_sync_rollback(ctx_r); + index_storage_expunging_deinit(box); + return index_storage_expunged_sync_begin(box, ctx_r, view_r, + trans_r, flags); + } + return 1; +} + +int index_storage_save_continue(struct mail_save_context *ctx, + struct istream *input, + struct mail *cache_dest_mail) +{ + struct mail_storage *storage = ctx->transaction->box->storage; + + do { + switch (o_stream_send_istream(ctx->data.output, input)) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + break; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + break; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + i_unreached(); + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + /* handle below */ + break; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(ctx->dest_mail, + "save: write(%s) failed: %s", + o_stream_get_name(ctx->data.output), + o_stream_get_error(ctx->data.output)); + } + return -1; + } + if (cache_dest_mail != NULL) + index_mail_cache_parse_continue(cache_dest_mail); + + /* both tee input readers may consume data from our primary + input stream. we'll have to make sure we don't return with + one of the streams still having data in them. */ + } while (i_stream_read(input) > 0); + + if (input->stream_errno != 0) { + mail_set_critical(ctx->dest_mail, "save: read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); + return -1; + } + return 0; +} + +void index_storage_save_abort_last(struct mail_save_context *ctx, uint32_t seq) +{ + struct index_mail *imail = INDEX_MAIL(ctx->dest_mail); + + /* Close the mail before it's expunged. This allows it to be + reset cleanly. */ + imail->data.no_caching = TRUE; + imail->mail.v.close(&imail->mail.mail); + + mail_index_expunge(ctx->transaction->itrans, seq); + /* currently we can't just drop pending cache updates for this one + specific record, so we'll reset the whole cache transaction. */ + mail_cache_transaction_reset(ctx->transaction->cache_trans); +} + +int index_mailbox_fix_inconsistent_existence(struct mailbox *box, + const char *path) +{ + const char *index_path; + struct stat st; + + /* Could be a race condition or could be because ITERINDEX is used + and the index directory exists, but the storage directory doesn't. + Handle the existence inconsistency by creating this directory if + the index directory exists (don't bother checking if ITERINDEX is + set or not - it doesn't matter since either both dirs should exist + or not). */ + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, + &index_path) < 0) + return -1; + + if (strcmp(index_path, path) == 0) { + /* there's no separate index path - mailbox was just deleted */ + } else if (stat(index_path, &st) == 0) { + /* inconsistency - create also the mail directory */ + return mailbox_mkdir(box, path, MAILBOX_LIST_PATH_TYPE_MAILBOX); + } else if (errno == ENOENT) { + /* race condition - mailbox was just deleted */ + } else { + mailbox_set_critical(box, "stat(%s) failed: %m", index_path); + return -1; + } + mailbox_set_deleted(box); + return -1; +} diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h new file mode 100644 index 0000000..716e725 --- /dev/null +++ b/src/lib-storage/index/index-storage.h @@ -0,0 +1,194 @@ +#ifndef INDEX_STORAGE_H +#define INDEX_STORAGE_H + +#include "file-dotlock.h" +#include "mail-storage-private.h" +#include "mail-index-private.h" +#include "mailbox-watch.h" + +#define MAILBOX_FULL_SYNC_INTERVAL 5 + +enum mailbox_lock_notify_type { + MAILBOX_LOCK_NOTIFY_NONE, + + /* Mailbox is locked, will abort in secs_left */ + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + /* Mailbox lock looks stale, will override in secs_left */ + MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE +}; + +enum index_storage_list_change { + INDEX_STORAGE_LIST_CHANGE_ERROR = -1, + INDEX_STORAGE_LIST_CHANGE_NONE = 0, + INDEX_STORAGE_LIST_CHANGE_INMEMORY, + INDEX_STORAGE_LIST_CHANGE_NORECORD, + INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS, + INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED, + INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED +}; + +struct index_mailbox_context { + union mailbox_module_context module_ctx; + enum mail_index_open_flags index_flags; + + time_t next_lock_notify; /* temporary */ + enum mailbox_lock_notify_type last_notify_type; + + const ARRAY_TYPE(keywords) *keyword_names; + struct mail_cache_field *cache_fields; + + struct mailbox_vsize_update *vsize_update; + + uint32_t recent_flags_prev_first_recent_uid; + uint32_t recent_flags_last_check_nextuid; + + time_t sync_last_check; + uint32_t list_index_sync_ext_id; +}; + +#define INDEX_STORAGE_CONTEXT(obj) \ + MODULE_CONTEXT_REQUIRE(obj, index_storage_module) +extern MODULE_CONTEXT_DEFINE(index_storage_module, + &mail_storage_module_register); + +void index_storage_lock_notify(struct mailbox *box, + enum mailbox_lock_notify_type notify_type, + unsigned int secs_left); +void index_storage_lock_notify_reset(struct mailbox *box); + +int index_storage_mailbox_alloc_index(struct mailbox *box); +void index_storage_mailbox_alloc(struct mailbox *box, const char *vname, + enum mailbox_flags flags, + const char *index_prefix); +int index_storage_mailbox_exists(struct mailbox *box, bool auto_boxes, + enum mailbox_existence *existence_r); +int index_storage_mailbox_exists_full(struct mailbox *box, const char *subdir, + enum mailbox_existence *existence_r) + ATTR_NULL(2); +int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory); +int index_storage_mailbox_enable(struct mailbox *box, + enum mailbox_feature feature); +void index_storage_mailbox_close(struct mailbox *box); +void index_storage_mailbox_free(struct mailbox *box); +int index_storage_mailbox_update(struct mailbox *box, + const struct mailbox_update *update); +int index_storage_mailbox_update_common(struct mailbox *box, + const struct mailbox_update *update); +int index_storage_mailbox_create(struct mailbox *box, bool directory); +int index_storage_mailbox_delete_pre(struct mailbox *box); +int index_storage_mailbox_delete_post(struct mailbox *box); +int index_storage_mailbox_delete(struct mailbox *box); +int index_storage_mailbox_delete_dir(struct mailbox *box, bool mailbox_deleted); +int index_storage_mailbox_rename(struct mailbox *src, struct mailbox *dest); + +int index_mailbox_view_update_last_temp_file_scan(struct mail_index_view *view); +int index_mailbox_update_last_temp_file_scan(struct mailbox *box); +int index_mailbox_fix_inconsistent_existence(struct mailbox *box, + const char *path); + +bool index_storage_is_readonly(struct mailbox *box); +bool index_storage_is_inconsistent(struct mailbox *box); + +enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box); +bool index_mailbox_want_full_sync(struct mailbox *box, + enum mailbox_sync_flags flags); +struct mailbox_sync_context * +index_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags, + bool failed); +bool index_mailbox_sync_next(struct mailbox_sync_context *ctx, + struct mailbox_sync_rec *sync_rec_r); +int index_mailbox_sync_deinit(struct mailbox_sync_context *ctx, + struct mailbox_sync_status *status_r); + +int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags); +enum mailbox_sync_type index_sync_type_convert(enum mail_index_sync_type type); +void index_sync_update_recent_count(struct mailbox *box); +int index_storage_get_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r); +void index_storage_get_open_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r); +int index_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r); +int index_mailbox_get_virtual_size(struct mailbox *box, + struct mailbox_metadata *metadata_r); +int index_mailbox_get_physical_size(struct mailbox *box, + struct mailbox_metadata *metadata_r); + +int index_storage_attribute_set(struct mailbox_transaction_context *t, + enum mail_attribute_type type_flags, + const char *key, + const struct mail_attribute_value *value); +int index_storage_attribute_get(struct mailbox *box, + enum mail_attribute_type type_flags, + const char *key, + struct mail_attribute_value *value_r); +struct mailbox_attribute_iter * +index_storage_attribute_iter_init(struct mailbox *box, + enum mail_attribute_type type_flags, + const char *prefix); +const char * +index_storage_attribute_iter_next(struct mailbox_attribute_iter *iter); +int index_storage_attribute_iter_deinit(struct mailbox_attribute_iter *iter); + +struct mail_search_context * +index_storage_search_init(struct mailbox_transaction_context *t, + struct mail_search_args *args, + const enum mail_sort_type *sort_program, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); +int index_storage_search_deinit(struct mail_search_context *ctx); +bool index_storage_search_next_nonblock(struct mail_search_context *ctx, + struct mail **mail_r, bool *tryagain_r); +bool index_storage_search_next_update_seq(struct mail_search_context *ctx); +int index_storage_search_next_match_mail(struct mail_search_context *ctx, + struct mail *mail); + +struct mailbox_transaction_context * +index_transaction_begin(struct mailbox *box, + enum mailbox_transaction_flags flags, + const char *reason); +void index_transaction_init(struct mailbox_transaction_context *t, + struct mailbox *box, + enum mailbox_transaction_flags flags, + const char *reason); +void index_transaction_init_pvt(struct mailbox_transaction_context *t); +int index_transaction_commit(struct mailbox_transaction_context *t, + struct mail_transaction_commit_changes *changes_r); +void index_transaction_rollback(struct mailbox_transaction_context *t); +void index_save_context_free(struct mail_save_context *ctx); +void index_copy_cache_fields(struct mail_save_context *ctx, + struct mail *src_mail, uint32_t dest_seq); +int index_storage_set_subscribed(struct mailbox *box, bool set); +void index_storage_destroy(struct mail_storage *storage); + +bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1, + const ARRAY_TYPE(keyword_indexes) *k2); + +int index_storage_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, + const char **reason_r); +enum index_storage_list_change +index_storage_list_index_has_changed_full(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, const char **reason_r); +void index_storage_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq); + +int index_storage_expunged_sync_begin(struct mailbox *box, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + enum mail_index_sync_flags flags); +void index_storage_expunging_deinit(struct mailbox *box); + +int index_storage_save_continue(struct mail_save_context *ctx, + struct istream *input, + struct mail *cache_dest_mail); +void index_storage_save_abort_last(struct mail_save_context *ctx, uint32_t seq); + +#endif diff --git a/src/lib-storage/index/index-sync-changes.c b/src/lib-storage/index/index-sync-changes.c new file mode 100644 index 0000000..6d8cd99 --- /dev/null +++ b/src/lib-storage/index/index-sync-changes.c @@ -0,0 +1,200 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "index-storage.h" +#include "index-sync-changes.h" + +struct index_sync_changes_context { + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *sync_trans; + + ARRAY(struct mail_index_sync_rec) syncs; + struct mail_index_sync_rec sync_rec; + bool dirty_flag_updates; +}; + +struct index_sync_changes_context * +index_sync_changes_init(struct mail_index_sync_ctx *index_sync_ctx, + struct mail_index_view *sync_view, + struct mail_index_transaction *sync_trans, + bool dirty_flag_updates) +{ + struct index_sync_changes_context *ctx; + + ctx = i_new(struct index_sync_changes_context, 1); + ctx->index_sync_ctx = index_sync_ctx; + ctx->sync_view = sync_view; + ctx->sync_trans = sync_trans; + ctx->dirty_flag_updates = dirty_flag_updates; + i_array_init(&ctx->syncs, 16); + return ctx; +} + +void index_sync_changes_deinit(struct index_sync_changes_context **_ctx) +{ + struct index_sync_changes_context *ctx = *_ctx; + + *_ctx = NULL; + array_free(&ctx->syncs); + i_free(ctx); +} + +void index_sync_changes_reset(struct index_sync_changes_context *ctx) +{ + array_clear(&ctx->syncs); + i_zero(&ctx->sync_rec); +} + +void index_sync_changes_delete_to(struct index_sync_changes_context *ctx, + uint32_t last_uid) +{ + struct mail_index_sync_rec *syncs; + unsigned int src, dest, count; + + syncs = array_get_modifiable(&ctx->syncs, &count); + + for (src = dest = 0; src < count; src++) { + i_assert(last_uid >= syncs[src].uid1); + if (last_uid <= syncs[src].uid2) { + /* keep it */ + if (src != dest) + syncs[dest] = syncs[src]; + dest++; + } + } + + array_delete(&ctx->syncs, dest, count - dest); +} + +static bool +index_sync_changes_have_expunges(struct index_sync_changes_context *ctx, + unsigned int count, + guid_128_t expunged_guid_128_r) +{ + const struct mail_index_sync_rec *syncs; + unsigned int i; + + syncs = array_front(&ctx->syncs); + for (i = 0; i < count; i++) { + if (syncs[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + memcpy(expunged_guid_128_r, syncs[i].guid_128, + GUID_128_SIZE); + return TRUE; + } + } + return FALSE; +} + +void index_sync_changes_read(struct index_sync_changes_context *ctx, + uint32_t uid, bool *sync_expunge_r, + guid_128_t expunged_guid_128_r) +{ + struct mail_index_sync_rec *sync_rec = &ctx->sync_rec; + uint32_t seq1, seq2; + unsigned int orig_count; + + *sync_expunge_r = FALSE; + + index_sync_changes_delete_to(ctx, uid); + orig_count = array_count(&ctx->syncs); + + while (uid >= sync_rec->uid1) { + if (uid <= sync_rec->uid2) { + array_push_back(&ctx->syncs, sync_rec); + + if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + *sync_expunge_r = TRUE; + memcpy(expunged_guid_128_r, sync_rec->guid_128, + GUID_128_SIZE); + } + } + + if (!mail_index_sync_next(ctx->index_sync_ctx, sync_rec)) { + i_zero(sync_rec); + break; + } + + switch (sync_rec->type) { + case MAIL_INDEX_SYNC_TYPE_EXPUNGE: + break; + case MAIL_INDEX_SYNC_TYPE_FLAGS: + case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + if (!ctx->dirty_flag_updates) + break; + + /* mark the changes as dirty */ + (void)mail_index_lookup_seq_range(ctx->sync_view, + sync_rec->uid1, + sync_rec->uid2, + &seq1, &seq2); + i_zero(sync_rec); + + if (seq1 == 0) + break; + + mail_index_update_flags_range(ctx->sync_trans, + seq1, seq2, MODIFY_ADD, + (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY); + break; + } + } + + if (!*sync_expunge_r && orig_count > 0) { + *sync_expunge_r = + index_sync_changes_have_expunges(ctx, orig_count, + expunged_guid_128_r); + } +} + +bool index_sync_changes_have(struct index_sync_changes_context *ctx) +{ + return array_count(&ctx->syncs) > 0; +} + +uint32_t +index_sync_changes_get_next_uid(struct index_sync_changes_context *ctx) +{ + return ctx->sync_rec.uid1; +} + +void index_sync_changes_apply(struct index_sync_changes_context *ctx, + pool_t pool, uint8_t *flags, + ARRAY_TYPE(keyword_indexes) *keywords, + enum mail_index_sync_type *sync_type_r) +{ + const struct mail_index_sync_rec *syncs; + unsigned int i, count; + enum mail_index_sync_type sync_type = 0; + + syncs = array_get(&ctx->syncs, &count); + for (i = 0; i < count; i++) { + switch (syncs[i].type) { + case MAIL_INDEX_SYNC_TYPE_FLAGS: + mail_index_sync_flags_apply(&syncs[i], flags); + sync_type |= MAIL_INDEX_SYNC_TYPE_FLAGS; + break; + case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + if (!array_is_created(keywords)) { + /* no existing keywords */ + if (syncs[i].type != + MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD) + break; + + /* adding, create the array */ + p_array_init(keywords, pool, + I_MIN(10, count - i)); + } + if (mail_index_sync_keywords_apply(&syncs[i], keywords)) + sync_type |= syncs[i].type; + break; + default: + break; + } + } + + *sync_type_r = sync_type; +} diff --git a/src/lib-storage/index/index-sync-changes.h b/src/lib-storage/index/index-sync-changes.h new file mode 100644 index 0000000..e6f5d47 --- /dev/null +++ b/src/lib-storage/index/index-sync-changes.h @@ -0,0 +1,28 @@ +#ifndef INDEX_SYNC_CHANGES_H +#define INDEX_SYNC_CHANGES_H + +struct index_sync_changes_context * +index_sync_changes_init(struct mail_index_sync_ctx *index_sync_ctx, + struct mail_index_view *sync_view, + struct mail_index_transaction *sync_trans, + bool dirty_flag_updates); +void index_sync_changes_deinit(struct index_sync_changes_context **_ctx); + +void index_sync_changes_reset(struct index_sync_changes_context *ctx); +void index_sync_changes_delete_to(struct index_sync_changes_context *ctx, + uint32_t last_uid); + +void index_sync_changes_read(struct index_sync_changes_context *ctx, + uint32_t uid, bool *sync_expunge_r, + guid_128_t expunged_guid_128); +bool index_sync_changes_have(struct index_sync_changes_context *ctx); +uint32_t +index_sync_changes_get_next_uid(struct index_sync_changes_context *ctx); + +void index_sync_changes_apply(struct index_sync_changes_context *ctx, + pool_t pool, uint8_t *flags, + ARRAY_TYPE(keyword_indexes) *keywords, + enum mail_index_sync_type *sync_type_r) + ATTR_NULL(2); + +#endif diff --git a/src/lib-storage/index/index-sync-private.h b/src/lib-storage/index/index-sync-private.h new file mode 100644 index 0000000..7d670c4 --- /dev/null +++ b/src/lib-storage/index/index-sync-private.h @@ -0,0 +1,38 @@ +#ifndef INDEX_SYNC_PRIVATE_H +#define INDEX_SYNC_PRIVATE_H + +#include "index-storage.h" + +struct index_mailbox_sync_pvt_context; + +struct index_mailbox_sync_context { + struct mailbox_sync_context ctx; + + struct mail_index_view_sync_ctx *sync_ctx; + uint32_t messages_count; + + ARRAY_TYPE(seq_range) flag_updates; + ARRAY_TYPE(seq_range) hidden_updates; + ARRAY_TYPE(seq_range) all_flag_update_uids; + const ARRAY_TYPE(seq_range) *expunges; + unsigned int flag_update_idx, hidden_update_idx, expunge_pos; + + bool failed; +}; + +void index_sync_search_results_uidify(struct index_mailbox_sync_context *ctx); +void index_sync_search_results_update(struct index_mailbox_sync_context *ctx); +void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx); + +/* Returns 1 = ok, 0 = no private indexes, -1 = error */ +int index_mailbox_sync_pvt_init(struct mailbox *box, bool lock, + enum mail_index_view_sync_flags flags, + struct index_mailbox_sync_pvt_context **ctx_r); +int index_mailbox_sync_pvt_newmails(struct index_mailbox_sync_pvt_context *ctx, + struct mailbox_transaction_context *trans); +int index_mailbox_sync_pvt_view(struct index_mailbox_sync_pvt_context *ctx, + ARRAY_TYPE(seq_range) *flag_updates, + ARRAY_TYPE(seq_range) *hidden_updates); +void index_mailbox_sync_pvt_deinit(struct index_mailbox_sync_pvt_context **ctx); + +#endif diff --git a/src/lib-storage/index/index-sync-pvt.c b/src/lib-storage/index/index-sync-pvt.c new file mode 100644 index 0000000..92eb4e8 --- /dev/null +++ b/src/lib-storage/index/index-sync-pvt.c @@ -0,0 +1,345 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mailbox-list-private.h" +#include "index-sync-private.h" + +struct index_mailbox_sync_pvt_context { + struct mailbox *box; + + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view_pvt; + struct mail_index_transaction *trans_pvt; + struct mail_index_view *view_shared; + + enum mail_index_view_sync_flags flags; +}; + +static int sync_pvt_expunges(struct index_mailbox_sync_pvt_context *ctx) +{ + uint32_t seq_shared, seq_pvt, count_shared, count_pvt; + uint32_t uid_shared, uid_pvt; + + count_shared = mail_index_view_get_messages_count(ctx->view_shared); + count_pvt = mail_index_view_get_messages_count(ctx->view_pvt); + seq_shared = seq_pvt = 1; + while (seq_pvt <= count_pvt && seq_shared <= count_shared) { + mail_index_lookup_uid(ctx->view_pvt, seq_pvt, &uid_pvt); + mail_index_lookup_uid(ctx->view_shared, seq_shared, &uid_shared); + if (uid_pvt == uid_shared) { + seq_pvt++; + seq_shared++; + } else if (uid_pvt < uid_shared) { + /* message expunged */ + mail_index_expunge(ctx->trans_pvt, seq_pvt); + seq_pvt++; + } else { + mailbox_set_critical(ctx->box, + "%s: Message UID=%u unexpectedly inserted to mailbox", + ctx->box->index_pvt->filepath, uid_shared); + return -1; + } + } + return 0; +} + +static void +sync_pvt_copy_self_flags(struct index_mailbox_sync_pvt_context *ctx, + ARRAY_TYPE(keyword_indexes) *keywords, + uint32_t seq_old, uint32_t seq_new) +{ + const struct mail_index_record *old_rec; + + old_rec = mail_index_lookup(ctx->view_pvt, seq_old); + mail_index_lookup_keywords(ctx->view_pvt, seq_old, keywords); + if (old_rec->flags != 0) { + mail_index_update_flags(ctx->trans_pvt, seq_new, + MODIFY_ADD, old_rec->flags); + } + if (array_count(keywords) > 0) { + struct mail_keywords *kw; + + kw = mail_index_keywords_create_from_indexes(ctx->box->index_pvt, + keywords); + mail_index_update_keywords(ctx->trans_pvt, seq_new, + MODIFY_ADD, kw); + mail_index_keywords_unref(&kw); + } +} + +static void +sync_pvt_copy_shared_flags(struct index_mailbox_sync_pvt_context *ctx, + uint32_t seq_shared, uint32_t seq_pvt) +{ + const struct mail_index_record *rec; + + rec = mail_index_lookup(ctx->view_shared, seq_shared); + mail_index_update_flags(ctx->trans_pvt, seq_pvt, MODIFY_ADD, + rec->flags & mailbox_get_private_flags_mask(ctx->box)); +} + +static int +index_mailbox_sync_view_refresh(struct index_mailbox_sync_pvt_context *ctx) +{ + /* open a view for the latest version of the index */ + if (mail_index_refresh(ctx->box->index_pvt) < 0 || + mail_index_refresh(ctx->box->index) < 0) { + mailbox_set_index_error(ctx->box); + return -1; + } + if (ctx->view_shared != NULL) + mail_index_view_close(&ctx->view_shared); + ctx->view_shared = mail_index_view_open(ctx->box->index); + return 0; +} + +static int +index_mailbox_sync_open(struct index_mailbox_sync_pvt_context *ctx, bool force) +{ + const struct mail_index_header *hdr_shared, *hdr_pvt; + + if (index_mailbox_sync_view_refresh(ctx) < 0) + return -1; + + hdr_shared = mail_index_get_header(ctx->view_shared); + if (hdr_shared->uid_validity == 0 && !force) { + /* the mailbox hasn't been fully created yet, + no need for a private index yet */ + return 0; + } + hdr_pvt = mail_index_get_header(ctx->box->view_pvt); + if (hdr_pvt->next_uid == hdr_shared->next_uid && + hdr_pvt->messages_count == hdr_shared->messages_count && !force) { + /* no new or expunged mails, don't bother syncing */ + return 0; + } + if (mail_index_sync_begin(ctx->box->index_pvt, &ctx->sync_ctx, + &ctx->view_pvt, &ctx->trans_pvt, 0) < 0) { + mailbox_set_index_error(ctx->box); + return -1; + } + /* refresh once more now that we're locked */ + if (index_mailbox_sync_view_refresh(ctx) < 0) + return -1; + return 1; +} + +int index_mailbox_sync_pvt_init(struct mailbox *box, bool lock, + enum mail_index_view_sync_flags flags, + struct index_mailbox_sync_pvt_context **ctx_r) +{ + struct index_mailbox_sync_pvt_context *ctx; + int ret; + + *ctx_r = NULL; + + if ((ret = mailbox_open_index_pvt(box)) <= 0) + return ret; + + ctx = i_new(struct index_mailbox_sync_pvt_context, 1); + ctx->box = box; + ctx->flags = flags; + if (lock) { + if (index_mailbox_sync_open(ctx, TRUE) < 0) { + index_mailbox_sync_pvt_deinit(&ctx); + return -1; + } + } + + *ctx_r = ctx; + return 1; +} + +static int +index_mailbox_sync_pvt_index(struct index_mailbox_sync_pvt_context *ctx, + const struct mail_save_private_changes *pvt_changes, + unsigned int pvt_changes_count) +{ + const struct mail_index_header *hdr_shared, *hdr_pvt; + ARRAY_TYPE(keyword_indexes) keywords; + uint32_t seq_shared, seq_pvt, seq_old_pvt, seq2, count_shared, uid; + unsigned int pc_idx = 0; + bool reset = FALSE, preserve_old_flags = FALSE, copy_shared_flags; + bool initial_index = FALSE; + int ret; + + if (ctx->sync_ctx == NULL) { + if ((ret = index_mailbox_sync_open(ctx, FALSE)) <= 0) + return ret; + } + hdr_pvt = mail_index_get_header(ctx->view_pvt); + hdr_shared = mail_index_get_header(ctx->view_shared); + + if (hdr_shared->uid_validity == hdr_pvt->uid_validity) { + /* same mailbox. expunge messages from private index that + no longer exist. */ + if (sync_pvt_expunges(ctx) < 0) { + reset = TRUE; + preserve_old_flags = TRUE; + t_array_init(&keywords, 32); + } + } else if (hdr_pvt->uid_validity == 0 && hdr_pvt->next_uid <= 1) { + /* creating the initial index - no logging */ + reset = TRUE; + initial_index = TRUE; + } else { + /* mailbox created/recreated */ + reset = TRUE; + i_info("Mailbox %s UIDVALIDITY changed (%u -> %u), reseting private index", + ctx->box->vname, hdr_pvt->uid_validity, + hdr_shared->uid_validity); + } + /* for public namespaces copy the initial private flags from the shared + index. this allows Sieve scripts to set the initial flags. */ + copy_shared_flags = + ctx->box->list->ns->type == MAIL_NAMESPACE_TYPE_PUBLIC; + + count_shared = mail_index_view_get_messages_count(ctx->view_shared); + if (!reset) { + if (!mail_index_lookup_seq_range(ctx->view_shared, + hdr_pvt->next_uid, + hdr_shared->next_uid, + &seq_shared, &seq2)) { + /* no new messages */ + seq_shared = count_shared+1; + } + } else { + if (!initial_index) + mail_index_reset(ctx->trans_pvt); + mail_index_update_header(ctx->trans_pvt, + offsetof(struct mail_index_header, uid_validity), + &hdr_shared->uid_validity, + sizeof(hdr_shared->uid_validity), TRUE); + seq_shared = 1; + } + + uid = 0; + for (; seq_shared <= count_shared; seq_shared++) { + mail_index_lookup_uid(ctx->view_shared, seq_shared, &uid); + mail_index_append(ctx->trans_pvt, uid, &seq_pvt); + if (preserve_old_flags && + mail_index_lookup_seq(ctx->view_pvt, uid, &seq_old_pvt)) { + /* copy flags from the original private index */ + sync_pvt_copy_self_flags(ctx, &keywords, + seq_old_pvt, seq_pvt); + } else if (copy_shared_flags) { + sync_pvt_copy_shared_flags(ctx, seq_shared, seq_pvt); + } + + /* add private flags for the recently saved/copied messages */ + while (pc_idx < pvt_changes_count && + pvt_changes[pc_idx].mailnum <= uid) { + if (pvt_changes[pc_idx].mailnum == uid) { + mail_index_update_flags(ctx->trans_pvt, seq_pvt, + MODIFY_ADD, pvt_changes[pc_idx].flags); + } + pc_idx++; + } + } + + if (uid < hdr_shared->next_uid) { + mail_index_update_header(ctx->trans_pvt, + offsetof(struct mail_index_header, next_uid), + &hdr_shared->next_uid, + sizeof(hdr_shared->next_uid), FALSE); + } + + if ((ret = mail_index_sync_commit(&ctx->sync_ctx)) < 0) + mailbox_set_index_error(ctx->box); + return ret; +} + +static int +mail_save_private_changes_mailnum_cmp(const struct mail_save_private_changes *c1, + const struct mail_save_private_changes *c2) +{ + if (c1->mailnum < c2->mailnum) + return -1; + if (c1->mailnum > c2->mailnum) + return 1; + return 0; +} + +int index_mailbox_sync_pvt_newmails(struct index_mailbox_sync_pvt_context *ctx, + struct mailbox_transaction_context *trans) +{ + struct mail_save_private_changes *pvt_changes; + struct seq_range_iter iter; + unsigned int i, n, pvt_count; + uint32_t uid; + + if (index_mailbox_sync_view_refresh(ctx) < 0) + return -1; + + /* translate mail numbers to UIDs */ + pvt_changes = array_get_modifiable(&trans->pvt_saves, &pvt_count); + + n = i = 0; + seq_range_array_iter_init(&iter, &trans->changes->saved_uids); + while (seq_range_array_iter_nth(&iter, n, &uid)) { + if (pvt_changes[i].mailnum == n) { + pvt_changes[i].mailnum = uid; + i++; + } + n++; + } + /* sort the changes by UID */ + array_sort(&trans->pvt_saves, mail_save_private_changes_mailnum_cmp); + + /* add new mails to the private index with the private flags */ + return index_mailbox_sync_pvt_index(ctx, pvt_changes, pvt_count); +} + +int index_mailbox_sync_pvt_view(struct index_mailbox_sync_pvt_context *ctx, + ARRAY_TYPE(seq_range) *flag_updates, + ARRAY_TYPE(seq_range) *hidden_updates) +{ + struct mail_index_view_sync_ctx *view_sync_ctx; + struct mail_index_view_sync_rec sync_rec; + uint32_t seq1, seq2; + bool delayed_expunges; + + /* sync private index against shared index by adding/removing mails */ + if (index_mailbox_sync_pvt_index(ctx, NULL, 0) < 0) + return -1; + + /* Indicate to view syncing that this is a secondary index view */ + ctx->flags |= MAIL_INDEX_VIEW_SYNC_FLAG_2ND_INDEX; + + /* sync the private view */ + view_sync_ctx = mail_index_view_sync_begin(ctx->box->view_pvt, ctx->flags); + while (mail_index_view_sync_next(view_sync_ctx, &sync_rec)) { + if (sync_rec.type != MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS) + continue; + + /* *_updates contains ctx->box->view sequences (not view_pvt + sequences) */ + if (mail_index_lookup_seq_range(ctx->box->view, + sync_rec.uid1, sync_rec.uid2, + &seq1, &seq2)) { + if (!sync_rec.hidden) { + seq_range_array_add_range(flag_updates, + seq1, seq2); + } else { + seq_range_array_add_range(hidden_updates, + seq1, seq2); + } + } + } + if (mail_index_view_sync_commit(&view_sync_ctx, &delayed_expunges) < 0) + return -1; + return 0; +} + +void index_mailbox_sync_pvt_deinit(struct index_mailbox_sync_pvt_context **_ctx) +{ + struct index_mailbox_sync_pvt_context *ctx = *_ctx; + + *_ctx = NULL; + + if (ctx->sync_ctx != NULL) + mail_index_sync_rollback(&ctx->sync_ctx); + if (ctx->view_shared != NULL) + mail_index_view_close(&ctx->view_shared); + i_free(ctx); +} diff --git a/src/lib-storage/index/index-sync-search.c b/src/lib-storage/index/index-sync-search.c new file mode 100644 index 0000000..95cb89d --- /dev/null +++ b/src/lib-storage/index/index-sync-search.c @@ -0,0 +1,97 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "seq-range-array.h" +#include "mail-search.h" +#include "mailbox-search-result-private.h" +#include "index-search-result.h" +#include "index-sync-private.h" + +static bool +search_result_want_flag_updates(const struct mail_search_result *result) +{ + if (!result->args_have_flags && !result->args_have_keywords && + !result->args_have_modseq) { + /* search result doesn't care about flag changes */ + return FALSE; + } + return TRUE; +} + +static void index_sync_uidify_array(struct index_mailbox_sync_context *ctx, + const ARRAY_TYPE(seq_range) *changes) +{ + const struct seq_range *range; + uint32_t seq, uid; + + array_foreach(changes, range) { + for (seq = range->seq1; seq <= range->seq2; seq++) { + mail_index_lookup_uid(ctx->ctx.box->view, seq, &uid); + seq_range_array_add(&ctx->all_flag_update_uids, uid); + } + } +} + +static void index_sync_uidify(struct index_mailbox_sync_context *ctx) +{ + unsigned int count; + + count = array_count(&ctx->flag_updates) + + array_count(&ctx->hidden_updates); + i_array_init(&ctx->all_flag_update_uids, count*2); + + index_sync_uidify_array(ctx, &ctx->flag_updates); + index_sync_uidify_array(ctx, &ctx->hidden_updates); +} + +void index_sync_search_results_uidify(struct index_mailbox_sync_context *ctx) +{ + struct mail_search_result *const *results; + unsigned int i, count; + + i_assert(!array_is_created(&ctx->all_flag_update_uids)); + + results = array_get(&ctx->ctx.box->search_results, &count); + for (i = 0; i < count; i++) { + if ((results[i]->flags & MAILBOX_SEARCH_RESULT_FLAG_UPDATE) != 0 && + search_result_want_flag_updates(results[i])) { + index_sync_uidify(ctx); + break; + } + } +} + +static void +search_result_update(struct index_mailbox_sync_context *ctx, + struct mail_search_result *result) +{ + if ((result->flags & MAILBOX_SEARCH_RESULT_FLAG_UPDATE) == 0) { + /* not an updateable search result */ + return; + } + + if (search_result_want_flag_updates(result)) { + (void)index_search_result_update_flags(result, + &ctx->all_flag_update_uids); + } + (void)index_search_result_update_appends(result, ctx->messages_count); +} + +void index_sync_search_results_update(struct index_mailbox_sync_context *ctx) +{ + struct mail_search_result *const *results; + unsigned int i, count; + + results = array_get(&ctx->ctx.box->search_results, &count); + for (i = 0; i < count; i++) + search_result_update(ctx, results[i]); +} + +void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx) +{ + if (ctx->expunges != NULL) { + index_search_results_update_expunges(ctx->ctx.box, + ctx->expunges); + } +} diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c new file mode 100644 index 0000000..73ea99c --- /dev/null +++ b/src/lib-storage/index/index-sync.c @@ -0,0 +1,560 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "seq-range-array.h" +#include "ioloop.h" +#include "array.h" +#include "index-mailbox-size.h" +#include "index-sync-private.h" +#include "mailbox-recent-flags.h" + +struct index_storage_list_index_record { + uint32_t size; + uint32_t mtime; +}; + +enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box) +{ + enum mail_index_sync_flags sync_flags = 0; + + if ((box->flags & MAILBOX_FLAG_DROP_RECENT) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; + if (box->deleting) { + sync_flags |= box->delete_sync_check ? + MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX : + MAIL_INDEX_SYNC_FLAG_DELETING_INDEX; + } + return sync_flags; +} + +bool index_mailbox_want_full_sync(struct mailbox *box, + enum mailbox_sync_flags flags) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 && + ioloop_time < ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL) + return FALSE; + + if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 && + (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) { + /* lib-lda is syncing the mailbox after saving a mail. + it only wants to find the new mail for potentially copying + to other mailboxes. that's mainly an optimization, and since + the mail was most likely already added to index we don't + need to do a full sync to find it. the main benefit here is + to avoid a very costly sync with a large Maildir/new/ */ + return FALSE; + } + + if (box->to_notify != NULL) + timeout_reset(box->to_notify); + ibox->sync_last_check = ioloop_time; + return TRUE; +} + +static void index_view_sync_recs_get(struct index_mailbox_sync_context *ctx) +{ + struct mail_index_view_sync_rec sync_rec; + uint32_t seq1, seq2; + + i_array_init(&ctx->flag_updates, 128); + i_array_init(&ctx->hidden_updates, 32); + while (mail_index_view_sync_next(ctx->sync_ctx, &sync_rec)) { + switch (sync_rec.type) { + case MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ: + case MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS: + if (!mail_index_lookup_seq_range(ctx->ctx.box->view, + sync_rec.uid1, + sync_rec.uid2, + &seq1, &seq2)) + break; + + if (!sync_rec.hidden && + sync_rec.type == MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS) { + seq_range_array_add_range(&ctx->flag_updates, + seq1, seq2); + } else { + seq_range_array_add_range(&ctx->hidden_updates, + seq1, seq2); + } + break; + } + } +} + +static void +index_view_sync_cleanup_updates(struct index_mailbox_sync_context *ctx) +{ + /* remove expunged messages from flag updates */ + if (ctx->expunges != NULL) { + seq_range_array_remove_seq_range(&ctx->flag_updates, + ctx->expunges); + seq_range_array_remove_seq_range(&ctx->hidden_updates, + ctx->expunges); + } + /* remove flag updates from hidden updates */ + seq_range_array_remove_seq_range(&ctx->hidden_updates, + &ctx->flag_updates); +} + +struct mailbox_sync_context * +index_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags, + bool failed) +{ + struct index_mailbox_sync_context *ctx; + struct index_mailbox_sync_pvt_context *pvt_ctx; + enum mail_index_view_sync_flags sync_flags = 0; + + ctx = i_new(struct index_mailbox_sync_context, 1); + ctx->ctx.box = box; + ctx->ctx.flags = flags; + + if (failed) { + ctx->failed = TRUE; + return &ctx->ctx; + } + + if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) != 0) + sync_flags |= MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES; + + if ((flags & MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) != 0) { + sync_flags |= MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT; + ctx->messages_count = 0; + } else { + ctx->messages_count = + mail_index_view_get_messages_count(box->view); + } + + if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0) { + /* we most likely did a fast sync. refresh the index anyway in + case there were some new changes. */ + (void)mail_index_refresh(box->index); + } + ctx->sync_ctx = mail_index_view_sync_begin(box->view, sync_flags); + if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) == 0) { + mail_index_view_sync_get_expunges(ctx->sync_ctx, + &ctx->expunges); + ctx->expunge_pos = array_count(ctx->expunges); + } + index_view_sync_recs_get(ctx); + index_sync_search_results_expunge(ctx); + + /* sync private index if needed. it doesn't use box->view, so it + doesn't matter if it's called at _sync_init() or _sync_deinit(). + however we also need to know if any private flags have changed + since last sync, so we need to call it before _sync_next() calls. */ + if (index_mailbox_sync_pvt_init(box, FALSE, sync_flags, &pvt_ctx) > 0) { + (void)index_mailbox_sync_pvt_view(pvt_ctx, &ctx->flag_updates, + &ctx->hidden_updates); + index_mailbox_sync_pvt_deinit(&pvt_ctx); + + } + index_view_sync_cleanup_updates(ctx); + return &ctx->ctx; +} + +static bool +index_mailbox_sync_next_expunge(struct index_mailbox_sync_context *ctx, + struct mailbox_sync_rec *sync_rec_r) +{ + const struct seq_range *range; + + if (ctx->expunge_pos == 0) + return FALSE; + + /* expunges is a sorted array of sequences. it's easiest for + us to print them from end to beginning. */ + ctx->expunge_pos--; + range = array_idx(ctx->expunges, ctx->expunge_pos); + i_assert(range->seq2 <= ctx->messages_count); + + mailbox_recent_flags_expunge_seqs(ctx->ctx.box, range->seq1, range->seq2); + ctx->messages_count -= range->seq2 - range->seq1 + 1; + + sync_rec_r->seq1 = range->seq1; + sync_rec_r->seq2 = range->seq2; + sync_rec_r->type = MAILBOX_SYNC_TYPE_EXPUNGE; + return TRUE; +} + +bool index_mailbox_sync_next(struct mailbox_sync_context *_ctx, + struct mailbox_sync_rec *sync_rec_r) +{ + struct index_mailbox_sync_context *ctx = + (struct index_mailbox_sync_context *)_ctx; + const struct seq_range *range; + unsigned int count; + + if (ctx->failed) + return FALSE; + + range = array_get(&ctx->flag_updates, &count); + if (ctx->flag_update_idx < count) { + sync_rec_r->type = MAILBOX_SYNC_TYPE_FLAGS; + sync_rec_r->seq1 = range[ctx->flag_update_idx].seq1; + sync_rec_r->seq2 = range[ctx->flag_update_idx].seq2; + ctx->flag_update_idx++; + return TRUE; + } + if ((_ctx->box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0) { + /* hidden flag changes' MODSEQs still need to be returned */ + range = array_get(&ctx->hidden_updates, &count); + if (ctx->hidden_update_idx < count) { + sync_rec_r->type = MAILBOX_SYNC_TYPE_MODSEQ; + sync_rec_r->seq1 = range[ctx->hidden_update_idx].seq1; + sync_rec_r->seq2 = range[ctx->hidden_update_idx].seq2; + ctx->hidden_update_idx++; + return TRUE; + } + } + + return index_mailbox_sync_next_expunge(ctx, sync_rec_r); +} + +static void +index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context *ctx) +{ + struct mailbox *box = ctx->ctx.box; + struct mail_index_view *view = ctx->ctx.box->view; + const struct mail_index_header *hdr; + uint32_t seq, start_uid, uid; + + if (!array_is_created(&box->recent_flags)) + return; + + /* expunges array contained expunges for the messages that were already + visible in this view, but append+expunge would be invisible. + recent_flags may however contain the append UID, so we'll have to + remove it separately */ + hdr = mail_index_get_header(view); + if (ctx->messages_count == 0) + uid = 0; + else if (ctx->messages_count <= hdr->messages_count) + mail_index_lookup_uid(view, ctx->messages_count, &uid); + else { + i_assert(mail_index_view_is_inconsistent(view)); + return; + } + + for (seq = ctx->messages_count + 1; seq <= hdr->messages_count; seq++) { + start_uid = uid; + mail_index_lookup_uid(view, seq, &uid); + if (start_uid + 1 > uid - 1) + continue; + + box->recent_flags_count -= + seq_range_array_remove_range(&box->recent_flags, + start_uid + 1, uid - 1); + } + + if (uid + 1 < hdr->next_uid) { + box->recent_flags_count -= + seq_range_array_remove_range(&box->recent_flags, + uid + 1, + hdr->next_uid - 1); + } +#ifdef DEBUG + if (!mail_index_view_is_inconsistent(view)) { + const struct seq_range *range; + unsigned int i, count; + + range = array_get(&box->recent_flags, &count); + for (i = 0; i < count; i++) { + for (uid = range[i].seq1; uid <= range[i].seq2; uid++) { + if (uid >= hdr->next_uid) + break; + (void)mail_index_lookup_seq(view, uid, &seq); + i_assert(seq != 0); + } + } + } +#endif +} + +void index_sync_update_recent_count(struct mailbox *box) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const struct mail_index_header *hdr; + uint32_t seq1, seq2; + + hdr = mail_index_get_header(box->view); + if (hdr->first_recent_uid < ibox->recent_flags_prev_first_recent_uid) { + mailbox_set_critical(box, + "first_recent_uid unexpectedly shrank: %u -> %u", + ibox->recent_flags_prev_first_recent_uid, + hdr->first_recent_uid); + mailbox_recent_flags_reset(box); + } + + if (hdr->first_recent_uid > box->recent_flags_prev_uid || + hdr->next_uid > ibox->recent_flags_last_check_nextuid) { + ibox->recent_flags_prev_first_recent_uid = hdr->first_recent_uid; + ibox->recent_flags_last_check_nextuid = hdr->next_uid; + if (mail_index_lookup_seq_range(box->view, + hdr->first_recent_uid, + hdr->next_uid, + &seq1, &seq2)) { + mailbox_recent_flags_set_seqs(box, box->view, + seq1, seq2); + } + } +} + +static void index_mailbox_sync_free(struct index_mailbox_sync_context *ctx) +{ + if (array_is_created(&ctx->flag_updates)) + array_free(&ctx->flag_updates); + if (array_is_created(&ctx->hidden_updates)) + array_free(&ctx->hidden_updates); + if (array_is_created(&ctx->all_flag_update_uids)) + array_free(&ctx->all_flag_update_uids); + i_free(ctx); +} + +int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx, + struct mailbox_sync_status *status_r) +{ + struct index_mailbox_sync_context *ctx = + (struct index_mailbox_sync_context *)_ctx; + struct mailbox_sync_rec sync_rec; + bool delayed_expunges = FALSE; + int ret = ctx->failed ? -1 : 0; + + /* finish handling expunges, so we don't break when updating + recent flags */ + while (index_mailbox_sync_next_expunge(ctx, &sync_rec)) ; + + /* convert sequences to uids before syncing view */ + index_sync_search_results_uidify(ctx); + + if (ctx->sync_ctx != NULL) { + if (mail_index_view_sync_commit(&ctx->sync_ctx, + &delayed_expunges) < 0) { + mailbox_set_index_error(_ctx->box); + ret = -1; + } + } + if (ret < 0) { + index_mailbox_sync_free(ctx); + return -1; + } + index_mailbox_expunge_unseen_recent(ctx); + + if ((_ctx->box->flags & MAILBOX_FLAG_DROP_RECENT) == 0 && + _ctx->box->opened) { + /* mailbox syncing didn't necessarily update our recent state */ + index_sync_update_recent_count(_ctx->box); + } + + if (status_r != NULL) + status_r->sync_delayed_expunges = delayed_expunges; + + /* update search results after private index is updated */ + index_sync_search_results_update(ctx); + /* update vsize header if wanted */ + index_mailbox_vsize_update_appends(_ctx->box); + + if (ret == 0 && mail_index_view_is_inconsistent(_ctx->box->view)) { + /* we probably had MAILBOX_SYNC_FLAG_FIX_INCONSISTENT set, + but the view got broken in the middle. FIXME: We could + attempt to fix it automatically. In any case now the view + isn't usable and we can't return success. */ + mailbox_set_index_error(_ctx->box); + ret = -1; + } + + index_mailbox_sync_free(ctx); + return ret; +} + +bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1, + const ARRAY_TYPE(keyword_indexes) *k2) +{ + const unsigned int *idx1, *idx2; + unsigned int i, j, count1, count2; + + if (!array_is_created(k1)) + return !array_is_created(k2) || array_count(k2) == 0; + if (!array_is_created(k2)) + return array_count(k1) == 0; + + /* The arrays may not be sorted, but they usually are. Optimize for + the assumption that they are */ + idx1 = array_get(k1, &count1); + idx2 = array_get(k2, &count2); + + if (count1 != count2) + return FALSE; + + for (i = 0; i < count1; i++) { + if (idx1[i] != idx2[i]) { + /* not found / unsorted array. check. */ + for (j = 0; j < count1; j++) { + if (idx1[i] == idx2[j]) + break; + } + if (j == count1) + return FALSE; + } + } + return TRUE; +} + +enum mailbox_sync_type index_sync_type_convert(enum mail_index_sync_type type) +{ + enum mailbox_sync_type ret = 0; + + if ((type & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) + ret |= MAILBOX_SYNC_TYPE_EXPUNGE; + if ((type & (MAIL_INDEX_SYNC_TYPE_FLAGS | + MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD | + MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE)) != 0) + ret |= MAILBOX_SYNC_TYPE_FLAGS; + return ret; +} + +static uint32_t +index_list_get_ext_id(struct mailbox *box, struct mail_index_view *view) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + + if (ibox->list_index_sync_ext_id == (uint32_t)-1) { + ibox->list_index_sync_ext_id = + mail_index_ext_register(mail_index_view_get_index(view), + "index sync", 0, + sizeof(struct index_storage_list_index_record), + sizeof(uint32_t)); + } + return ibox->list_index_sync_ext_id; +} + +enum index_storage_list_change +index_storage_list_index_has_changed_full(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, const char **reason_r) +{ + const struct index_storage_list_index_record *rec; + const void *data; + const char *dir, *path; + struct stat st; + uint32_t ext_id; + bool expunged; + int ret; + + *reason_r = NULL; + + if (mail_index_is_in_memory(mail_index_view_get_index(list_view))) { + *reason_r = "List index is in memory"; + return INDEX_STORAGE_LIST_CHANGE_INMEMORY; + } + + ext_id = index_list_get_ext_id(box, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + rec = data; + + if (rec == NULL) { + *reason_r = "Storage record is missing"; + return INDEX_STORAGE_LIST_CHANGE_NORECORD; + } else if (expunged) { + *reason_r = "Storage record is expunged"; + return INDEX_STORAGE_LIST_CHANGE_NORECORD; + } else if (rec->size == 0) { + *reason_r = "Storage record size=0"; + return INDEX_STORAGE_LIST_CHANGE_NORECORD; + } else if (rec->mtime == 0) { + *reason_r = "Storage record mtime=0"; + return INDEX_STORAGE_LIST_CHANGE_NORECORD; + } + if (box->storage->set->mailbox_list_index_very_dirty_syncs) + return INDEX_STORAGE_LIST_CHANGE_NONE; + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir); + if (ret < 0) + return INDEX_STORAGE_LIST_CHANGE_ERROR; + i_assert(ret > 0); + + path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL); + if (stat(path, &st) < 0) { + if (errno == ENOENT) { + *reason_r = t_strdup_printf("%s not found", path); + return INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS; + } + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return INDEX_STORAGE_LIST_CHANGE_ERROR; + } + uint32_t new_size = st.st_size & 0xffffffffU; + if (rec->size != new_size) { + *reason_r = t_strdup_printf("Storage size changed %u != %u", + rec->size, new_size); + return INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED; + } + uint32_t new_mtime = st.st_mtime & 0xffffffffU; + if (rec->mtime != new_mtime) { + *reason_r = t_strdup_printf("Storage mtime changed %u != %u", + rec->mtime, new_mtime); + return INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED; + } + return INDEX_STORAGE_LIST_CHANGE_NONE; +} + +int index_storage_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick ATTR_UNUSED, + const char **reason_r) +{ + switch (index_storage_list_index_has_changed_full(box, list_view, seq, + reason_r)) { + case INDEX_STORAGE_LIST_CHANGE_ERROR: + return -1; + case INDEX_STORAGE_LIST_CHANGE_NONE: + return 0; + default: + return 1; + } +} + +void index_storage_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq) +{ + struct mail_index_view *list_view; + const struct index_storage_list_index_record *old_rec; + struct index_storage_list_index_record new_rec; + const void *data; + const char *dir, *path; + struct stat st; + uint32_t ext_id; + bool expunged; + int ret; + + list_view = mail_index_transaction_get_view(trans); + if (mail_index_is_in_memory(mail_index_view_get_index(list_view))) + return; + + /* get the current record */ + ext_id = index_list_get_ext_id(box, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + if (expunged) + return; + old_rec = data; + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir); + if (ret < 0) + return; + i_assert(ret > 0); + + path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL); + if (stat(path, &st) < 0) { + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return; + } + + i_zero(&new_rec); + new_rec.size = st.st_size & 0xffffffffU; + new_rec.mtime = st.st_mtime & 0xffffffffU; + + if (old_rec == NULL || + memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0) + mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL); +} diff --git a/src/lib-storage/index/index-thread-finish.c b/src/lib-storage/index/index-thread-finish.c new file mode 100644 index 0000000..42326b6 --- /dev/null +++ b/src/lib-storage/index/index-thread-finish.c @@ -0,0 +1,682 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "imap-base-subject.h" +#include "mail-storage-private.h" +#include "index-thread-private.h" + + +struct mail_thread_shadow_node { + uint32_t first_child_idx, next_sibling_idx; +}; + +struct mail_thread_root_node { + /* node.idx usually points to indexes from mail hash. However + REFERENCES step (5) may add temporary dummy roots. They use larger + index numbers than exist in the hash. */ + struct mail_thread_child_node node; + + /* Used temporarily by (5)(B) base subject gathering. + root_idx1 is node's index in roots[] array + 1. + parent_root_idx points to root_idx1, or 0 for root. */ + unsigned int root_idx1; + uint32_t parent_root_idx1; + + /* subject contained a Re: or Fwd: */ + bool reply_or_forward:1; + /* a dummy node */ + bool dummy:1; + /* ignore this node - it's a dummy without children */ + bool ignore:1; +}; + +struct thread_finish_context { + unsigned int refcount; + + struct mail *tmp_mail; + struct mail_thread_cache *cache; + + ARRAY(struct mail_thread_root_node) roots; + ARRAY(struct mail_thread_shadow_node) shadow_nodes; + unsigned int next_new_root_idx; + + bool use_sent_date:1; + bool return_seqs:1; +}; + +struct mail_thread_iterate_context { + struct thread_finish_context *ctx; + + ARRAY_TYPE(mail_thread_child_node) children; + unsigned int next_idx; + bool failed; +}; + +struct subject_gather_context { + struct thread_finish_context *ctx; + + pool_t subject_pool; + HASH_TABLE(char *, struct mail_thread_root_node *) subject_hash; +}; + +static void +add_base_subject(struct subject_gather_context *ctx, const char *subject, + struct mail_thread_root_node *node) +{ + struct mail_thread_root_node *hash_node; + char *hash_subject; + bool is_reply_or_forward; + + subject = imap_get_base_subject_cased(pool_datastack_create(), subject, + &is_reply_or_forward); + /* (ii) If the thread subject is empty, skip this message. */ + if (*subject == '\0') + return; + + /* (iii) Look up the message associated with the thread + subject in the subject table. */ + if (!hash_table_lookup_full(ctx->subject_hash, subject, &hash_subject, + &hash_node)) { + /* (iv) If there is no message in the subject table with the + thread subject, add the current message and the thread + subject to the subject table. */ + hash_subject = p_strdup(ctx->subject_pool, subject); + hash_table_insert(ctx->subject_hash, hash_subject, node); + } else { + /* Otherwise, if the message in the subject table is not a + dummy, AND either of the following criteria are true: + + The current message is a dummy, OR + + The message in the subject table is a reply or forward + and the current message is not. + + then replace the message in the subject table with the + current message. */ + if (!hash_node->dummy && + (node->dummy || + (hash_node->reply_or_forward && !is_reply_or_forward))) { + hash_node->parent_root_idx1 = node->root_idx1; + hash_table_update(ctx->subject_hash, hash_subject, node); + } else { + node->parent_root_idx1 = hash_node->root_idx1; + } + } + + node->reply_or_forward = is_reply_or_forward; +} + +static int mail_thread_child_node_cmp(const struct mail_thread_child_node *c1, + const struct mail_thread_child_node *c2) +{ + if (c1->sort_date < c2->sort_date) + return -1; + if (c1->sort_date > c2->sort_date) + return 1; + + if (c1->uid < c2->uid) + return -1; + if (c1->uid > c2->uid) + return 1; + return 0; +} + +static int mail_thread_root_node_cmp(const struct mail_thread_root_node *r1, + const struct mail_thread_root_node *r2) +{ + return mail_thread_child_node_cmp(&r1->node, &r2->node); +} + +static uint32_t +thread_lookup_existing(struct thread_finish_context *ctx, uint32_t idx) +{ + const struct mail_thread_node *node; + + node = array_idx(&ctx->cache->thread_nodes, idx); + i_assert(MAIL_THREAD_NODE_EXISTS(node)); + i_assert(node->uid != 0); + return node->uid; +} + +static void +thread_child_node_fill(struct thread_finish_context *ctx, + struct mail_thread_child_node *child) +{ + int tz; + + child->uid = thread_lookup_existing(ctx, child->idx); + + if (!mail_set_uid(ctx->tmp_mail, child->uid)) { + /* the UID should have existed. we would have rebuild + the thread tree otherwise. */ + i_unreached(); + } + + /* get sent date if we want to use it and if it's valid */ + if (!ctx->use_sent_date) + child->sort_date = 0; + else if (mail_get_date(ctx->tmp_mail, &child->sort_date, &tz) < 0) + child->sort_date = 0; + + if (child->sort_date == 0) { + /* fallback to received date */ + (void)mail_get_received_date(ctx->tmp_mail, &child->sort_date); + } +} + +static void +thread_sort_children(struct thread_finish_context *ctx, uint32_t parent_idx, + ARRAY_TYPE(mail_thread_child_node) *sorted_children) +{ + const struct mail_thread_shadow_node *shadows; + struct mail_thread_child_node child; + unsigned int count; + + i_zero(&child); + array_clear(sorted_children); + + /* add all child indexes to the array */ + shadows = array_get(&ctx->shadow_nodes, &count); + child.idx = shadows[parent_idx].first_child_idx; + i_assert(child.idx != 0); + if (shadows[child.idx].next_sibling_idx == 0) { + /* only child - don't bother setting sort date */ + child.uid = thread_lookup_existing(ctx, child.idx); + + array_push_back(sorted_children, &child); + return; + } + while (child.idx != 0) { + thread_child_node_fill(ctx, &child); + + array_push_back(sorted_children, &child); + child.idx = shadows[child.idx].next_sibling_idx; + } + + /* sort the children */ + array_sort(sorted_children, mail_thread_child_node_cmp); +} + +static void gather_base_subjects(struct thread_finish_context *ctx) +{ + struct subject_gather_context gather_ctx; + struct mail_thread_root_node *roots; + const char *subject; + unsigned int i, count; + ARRAY_TYPE(mail_thread_child_node) sorted_children; + const struct mail_thread_child_node *children; + uint32_t idx, uid; + + i_zero(&gather_ctx); + gather_ctx.ctx = ctx; + + roots = array_get_modifiable(&ctx->roots, &count); + if (count == 0) + return; + gather_ctx.subject_pool = + pool_alloconly_create(MEMPOOL_GROWING"base subjects", + nearest_power(count * 20)); + hash_table_create(&gather_ctx.subject_hash, gather_ctx.subject_pool, + count * 2, str_hash, strcmp); + + i_array_init(&sorted_children, 64); + for (i = 0; i < count; i++) { + roots[i].root_idx1 = i + 1; + if (!roots[i].dummy) + idx = roots[i].node.idx; + else if (!roots[i].ignore) { + /* find the oldest child */ + thread_sort_children(ctx, roots[i].node.idx, + &sorted_children); + children = array_front(&sorted_children); + idx = children[0].idx; + } else { + /* dummy without children */ + continue; + } + + uid = thread_lookup_existing(ctx, idx); + if (!mail_set_uid(ctx->tmp_mail, uid)) { + /* the UID should have existed. we would have rebuild + the thread tree otherwise. */ + i_unreached(); + } + if (mail_get_first_header(ctx->tmp_mail, HDR_SUBJECT, + &subject) > 0) T_BEGIN { + add_base_subject(&gather_ctx, subject, &roots[i]); + } T_END; + } + i_assert(roots[count-1].parent_root_idx1 <= count); + array_free(&sorted_children); + hash_table_destroy(&gather_ctx.subject_hash); + pool_unref(&gather_ctx.subject_pool); +} + +static void thread_add_shadow_child(struct thread_finish_context *ctx, + uint32_t parent_idx, uint32_t child_idx) +{ + struct mail_thread_shadow_node *parent_shadow, *child_shadow; + + parent_shadow = array_idx_get_space(&ctx->shadow_nodes, parent_idx); + child_shadow = array_idx_modifiable(&ctx->shadow_nodes, child_idx); + + child_shadow->next_sibling_idx = parent_shadow->first_child_idx; + parent_shadow->first_child_idx = child_idx; +} + +static void mail_thread_root_thread_merge(struct thread_finish_context *ctx, + struct mail_thread_root_node *cur) +{ + struct mail_thread_root_node *roots, *root, new_root; + struct mail_thread_shadow_node *shadows; + unsigned int count; + uint32_t idx, next_idx; + + i_assert(cur->parent_root_idx1 != 0); + + /* The highest parent is the same as the current message in the + subject table. */ + roots = array_get_modifiable(&ctx->roots, &count); + root = cur; + do { + i_assert(root->parent_root_idx1 <= count); + root = &roots[root->parent_root_idx1 - 1]; + } while (root->parent_root_idx1 != 0); + i_assert(!root->ignore); + + shadows = array_front_modifiable(&ctx->shadow_nodes); + if (cur->dummy) { + /* If both messages are dummies, append the current + message's children to the children of the message in + the subject table (the children of both messages + become siblings), and then delete the current message. */ + i_assert(root->dummy); + + idx = shadows[cur->node.idx].first_child_idx; + while (idx != 0) { + next_idx = shadows[idx].next_sibling_idx; + thread_add_shadow_child(ctx, root->node.idx, idx); + idx = next_idx; + } + + shadows[cur->node.idx].first_child_idx = 0; + cur->ignore = TRUE; + } else if (root->dummy || (cur->reply_or_forward && + !root->reply_or_forward)) { + /* a) If the message in the subject table is a dummy and the + current message is not, make the current message a + child of the message in the subject table (a sibling + of its children). + + b) If the current message is a reply or forward and the + message in the subject table is not, make the current + message a child of the message in the subject table (a + sibling of its children). */ + thread_add_shadow_child(ctx, root->node.idx, cur->node.idx); + cur->ignore = TRUE; + } else { + /* Otherwise, create a new dummy message and make both + the current message and the message in the subject + table children of the dummy. Then replace the message + in the subject table with the dummy message. */ + i_zero(&new_root); + new_root.root_idx1 = array_count(&ctx->roots) + 1; + new_root.node.idx = ctx->next_new_root_idx++; + new_root.dummy = TRUE; + + thread_add_shadow_child(ctx, new_root.node.idx, root->node.idx); + thread_add_shadow_child(ctx, new_root.node.idx, cur->node.idx); + + root->parent_root_idx1 = new_root.root_idx1; + root->ignore = TRUE; + cur->ignore = TRUE; + + /* append last, since it breaks root and cur pointers */ + array_push_back(&ctx->roots, &new_root); + + /* make sure all shadow indexes are accessible directly */ + (void)array_idx_modifiable(&ctx->shadow_nodes, + new_root.node.idx); + } +} + +static bool merge_subject_threads(struct thread_finish_context *ctx) +{ + struct mail_thread_root_node *roots; + unsigned int i, count; + bool changed = FALSE; + + roots = array_get_modifiable(&ctx->roots, &count); + for (i = 0; i < count; i++) { + if (roots[i].parent_root_idx1 != 0 && !roots[i].ignore) { + mail_thread_root_thread_merge(ctx, &roots[i]); + /* more roots may have been added */ + roots = array_front_modifiable(&ctx->roots); + changed = TRUE; + } + } + + return changed; +} + +static void sort_root_nodes(struct thread_finish_context *ctx) +{ + ARRAY_TYPE(mail_thread_child_node) sorted_children; + const struct mail_thread_child_node *children; + const struct mail_thread_shadow_node *shadows; + struct mail_thread_root_node *roots; + unsigned int i, count, child_count; + + i_array_init(&sorted_children, 64); + shadows = array_front(&ctx->shadow_nodes); + roots = array_get_modifiable(&ctx->roots, &count); + for (i = 0; i < count; i++) { + if (roots[i].ignore) + continue; + if (roots[i].dummy) { + /* sort by the first child */ + if (shadows[roots[i].node.idx].first_child_idx == 0) { + /* childless dummy node */ + roots[i].ignore = TRUE; + continue; + } + thread_sort_children(ctx, roots[i].node.idx, + &sorted_children); + children = array_get(&sorted_children, &child_count); + if (child_count == 1) { + /* only one child - deferred step (3). + promote the child to the root. */ + roots[i].node = children[0]; + thread_child_node_fill(ctx, &roots[i].node); + roots[i].dummy = FALSE; + } else { + roots[i].node.uid = children[0].uid; + roots[i].node.sort_date = children[0].sort_date; + } + } else { + thread_child_node_fill(ctx, &roots[i].node); + } + } + array_free(&sorted_children); + array_sort(&ctx->roots, mail_thread_root_node_cmp); +} + +static int mail_thread_root_node_idx_cmp(const void *key, const void *value) +{ + const uint32_t *idx = key; + const struct mail_thread_root_node *root = value; + + return *idx < root->node.idx ? -1 : + *idx > root->node.idx ? 1 : 0; +} + +static void sort_root_nodes_ref2(struct thread_finish_context *ctx, + uint32_t record_count) +{ + const struct mail_thread_node *node; + struct mail_thread_root_node *roots, *root; + struct mail_thread_child_node child; + const struct mail_thread_shadow_node *shadows; + unsigned int root_count; + uint32_t idx, parent_idx; + + roots = array_get_modifiable(&ctx->roots, &root_count); + + /* drop childless dummy nodes */ + shadows = array_front(&ctx->shadow_nodes); + for (idx = 1; idx < root_count; idx++) { + if (roots[idx].dummy && + shadows[roots[idx].node.idx].first_child_idx == 0) + roots[idx].ignore = TRUE; + } + + for (idx = 1; idx < record_count; idx++) { + node = array_idx(&ctx->cache->thread_nodes, idx); + if (!MAIL_THREAD_NODE_EXISTS(node)) + continue; + + child.idx = idx; + thread_child_node_fill(ctx, &child); + + parent_idx = idx; + while (node->parent_idx != 0) { + parent_idx = node->parent_idx; + node = array_idx(&ctx->cache->thread_nodes, + node->parent_idx); + } + root = bsearch(&parent_idx, roots, root_count, sizeof(*roots), + mail_thread_root_node_idx_cmp); + i_assert(root != NULL); + + if (root->node.sort_date < child.sort_date) + root->node.sort_date = child.sort_date; + } + array_sort(&ctx->roots, mail_thread_root_node_cmp); +} + +static void mail_thread_create_shadows(struct thread_finish_context *ctx, + uint32_t record_count) +{ + const struct mail_thread_node *node, *parent; + struct mail_thread_root_node root; + struct mail_thread_child_node child; + uint32_t idx, parent_idx; + + ctx->use_sent_date = FALSE; + + i_zero(&root); + i_zero(&child); + + /* We may see dummy messages without parents or children. We can't + free them since the nodes are in an array, but they may get reused + later so just leave them be. With the current algorithm when this + happens all the struct fields are always zero at that point, so + we don't even have to try to zero them. */ + for (idx = 1; idx < record_count; idx++) { + node = array_idx(&ctx->cache->thread_nodes, idx); + + if (node->parent_idx == 0) { + /* root node - add to roots list */ + root.node.idx = idx; + if (!MAIL_THREAD_NODE_EXISTS(node)) { + root.dummy = TRUE; + root.node.uid = 0; + } else { + root.dummy = FALSE; + root.node.uid = node->uid; + } + array_push_back(&ctx->roots, &root); + continue; + } + i_assert(node->parent_idx < record_count); + + if (!MAIL_THREAD_NODE_EXISTS(node)) { + /* dummy node */ + continue; + } + + /* Find the node's first non-dummy parent and add the + node as its child. If there are no non-dummy + parents, add it as the highest dummy's child. */ + parent_idx = node->parent_idx; + parent = array_idx(&ctx->cache->thread_nodes, parent_idx); + while (!MAIL_THREAD_NODE_EXISTS(parent) && + parent->parent_idx != 0) { + parent_idx = parent->parent_idx; + parent = array_idx(&ctx->cache->thread_nodes, + parent_idx); + } + thread_add_shadow_child(ctx, parent_idx, idx); + } +} + +static void mail_thread_finish(struct thread_finish_context *ctx, + enum mail_thread_type thread_type) +{ + unsigned int record_count = array_count(&ctx->cache->thread_nodes); + + ctx->next_new_root_idx = record_count + 1; + + /* (2) save root nodes and (3) remove dummy messages */ + i_array_init(&ctx->roots, I_MIN(128, record_count)); + i_array_init(&ctx->shadow_nodes, record_count); + /* make sure all shadow indexes are accessible directly. */ + (void)array_idx_get_space(&ctx->shadow_nodes, record_count); + + mail_thread_create_shadows(ctx, record_count); + + /* (4) */ + ctx->use_sent_date = TRUE; + switch (thread_type) { + case MAIL_THREAD_REFERENCES: + sort_root_nodes(ctx); + /* (5) Gather together messages under the root that have + the same base subject text. */ + gather_base_subjects(ctx); + /* (5.C) Merge threads with the same thread subject. */ + if (merge_subject_threads(ctx)) { + /* root ordering may have changed, sort them again. */ + sort_root_nodes(ctx); + } + break; + case MAIL_THREAD_REFS: + sort_root_nodes_ref2(ctx, record_count); + break; + default: + i_unreached(); + } +} + +static void +nodes_change_uids_to_seqs(struct mail_thread_iterate_context *iter, bool root) +{ + struct mail_thread_child_node *children; + struct mailbox *box = iter->ctx->tmp_mail->box; + unsigned int i, count; + uint32_t uid, seq; + + children = array_get_modifiable(&iter->children, &count); + for (i = 0; i < count; i++) { + uid = children[i].uid; + if (uid == 0) { + /* dummy root */ + if (root) + continue; + i_unreached(); + } else { + mailbox_get_seq_range(box, uid, uid, &seq, &seq); + i_assert(seq != 0); + } + children[i].uid = seq; + } +} + +static void +mail_thread_iterate_fill_root(struct mail_thread_iterate_context *iter) +{ + struct mail_thread_root_node *roots; + unsigned int i, count; + + roots = array_get_modifiable(&iter->ctx->roots, &count); + i_array_init(&iter->children, count); + for (i = 0; i < count; i++) { + if (!roots[i].ignore) { + if (roots[i].dummy) + roots[i].node.uid = 0; + array_push_back(&iter->children, &roots[i].node); + } + } +} + +static struct mail_thread_iterate_context * +mail_thread_iterate_children(struct mail_thread_iterate_context *parent_iter, + uint32_t parent_idx) +{ + struct mail_thread_iterate_context *child_iter; + + child_iter = i_new(struct mail_thread_iterate_context, 1); + child_iter->ctx = parent_iter->ctx; + child_iter->ctx->refcount++; + + i_array_init(&child_iter->children, 8); + struct event_reason *reason = event_reason_begin("mailbox:thread"); + thread_sort_children(child_iter->ctx, parent_idx, + &child_iter->children); + if (child_iter->ctx->return_seqs) + nodes_change_uids_to_seqs(child_iter, FALSE); + event_reason_end(&reason); + return child_iter; +} + +struct mail_thread_iterate_context * +mail_thread_iterate_init_full(struct mail_thread_cache *cache, + struct mail *tmp_mail, + enum mail_thread_type thread_type, + bool return_seqs) +{ + struct mail_thread_iterate_context *iter; + struct thread_finish_context *ctx; + + iter = i_new(struct mail_thread_iterate_context, 1); + ctx = iter->ctx = i_new(struct thread_finish_context, 1); + ctx->refcount = 1; + ctx->cache = cache; + ctx->tmp_mail = tmp_mail; + ctx->return_seqs = return_seqs; + + struct event_reason *reason = event_reason_begin("mailbox:thread"); + mail_thread_finish(ctx, thread_type); + + mail_thread_iterate_fill_root(iter); + if (return_seqs) + nodes_change_uids_to_seqs(iter, TRUE); + event_reason_end(&reason); + return iter; +} + +const struct mail_thread_child_node * +mail_thread_iterate_next(struct mail_thread_iterate_context *iter, + struct mail_thread_iterate_context **child_iter_r) +{ + const struct mail_thread_child_node *children, *child; + const struct mail_thread_shadow_node *shadow; + unsigned int count; + + children = array_get(&iter->children, &count); + if (iter->next_idx >= count) + return NULL; + + child = &children[iter->next_idx++]; + shadow = array_idx(&iter->ctx->shadow_nodes, child->idx); + *child_iter_r = shadow->first_child_idx == 0 ? NULL : + mail_thread_iterate_children(iter, child->idx); + if (child->uid == 0 && *child_iter_r == NULL) { + /* this is a dummy node without children, + there's no point in returning it */ + return mail_thread_iterate_next(iter, child_iter_r); + } + return child; +} + +unsigned int mail_thread_iterate_count(struct mail_thread_iterate_context *iter) +{ + return array_count(&iter->children); +} + +int mail_thread_iterate_deinit(struct mail_thread_iterate_context **_iter) +{ + struct mail_thread_iterate_context *iter = *_iter; + + *_iter = NULL; + + if (--iter->ctx->refcount == 0) { + array_free(&iter->ctx->roots); + array_free(&iter->ctx->shadow_nodes); + i_free(iter->ctx); + } + array_free(&iter->children); + i_free(iter); + return 0; +} diff --git a/src/lib-storage/index/index-thread-links.c b/src/lib-storage/index/index-thread-links.c new file mode 100644 index 0000000..9affb83 --- /dev/null +++ b/src/lib-storage/index/index-thread-links.c @@ -0,0 +1,242 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "message-id.h" +#include "mail-storage.h" +#include "index-thread-private.h" + +static uint32_t thread_msg_add(struct mail_thread_cache *cache, + uint32_t uid, uint32_t msgid_idx) +{ + struct mail_thread_node *node; + + i_assert(msgid_idx != 0); + i_assert(msgid_idx < cache->first_invalid_msgid_str_idx); + + node = array_idx_get_space(&cache->thread_nodes, msgid_idx); + if (node->uid == 0) + node->uid = uid; + else { + /* duplicate message-id, keep the original. + if the original ever gets expunged, rebuild. */ + node->expunge_rebuilds = TRUE; + + msgid_idx = cache->next_invalid_msgid_str_idx++; + node = array_idx_get_space(&cache->thread_nodes, msgid_idx); + node->uid = uid; + } + return msgid_idx; +} + +static bool thread_node_has_ancestor(struct mail_thread_cache *cache, + const struct mail_thread_node *node, + const struct mail_thread_node *ancestor) +{ + while (node != ancestor) { + if (node->parent_idx == 0) + return FALSE; + + node = array_idx(&cache->thread_nodes, node->parent_idx); + } + return TRUE; +} + +static void thread_link_reference(struct mail_thread_cache *cache, + uint32_t parent_idx, uint32_t child_idx) +{ + struct mail_thread_node *node, *parent, *child; + uint32_t idx; + + i_assert(parent_idx < cache->first_invalid_msgid_str_idx); + + /* either child_idx or parent_idx may cause thread_nodes array to + grow. in such situation the other pointer may become invalid if + we don't get the pointers in correct order. */ + if (child_idx < parent_idx) { + parent = array_idx_get_space(&cache->thread_nodes, parent_idx); + child = array_idx_modifiable(&cache->thread_nodes, child_idx); + } else { + child = array_idx_get_space(&cache->thread_nodes, child_idx); + parent = array_idx_modifiable(&cache->thread_nodes, parent_idx); + } + + child->parent_link_refcount++; + if (thread_node_has_ancestor(cache, parent, child)) { + if (parent == child) { + /* loops to itself - ignore */ + return; + } + + /* child is an ancestor of parent. Adding child -> parent_node + would introduce a loop. If any messages referencing the path + between parent_node's parent and child_node get expunged, we + have to rebuild the tree because the loop might break. + For example: + + #1: a -> b (a.ref=1, b.ref=1) + #2: b -> a (a.ref=2, b.ref=2) + #3: c -> a -> b (a.ref=3, b.ref=3, c.ref=1) + + Expunging #3 wouldn't break the loop, but expunging #1 + would. */ + node = parent; + do { + idx = node->parent_idx; + i_assert(idx != 0); + node = array_idx_modifiable(&cache->thread_nodes, idx); + node->child_unref_rebuilds = TRUE; + } while (node != child); + return; + } else if (child->parent_idx == parent_idx) { + /* The same link already exists */ + return; + } + + /* Set parent_node as child_node's parent */ + if (child->parent_idx == 0) { + child->parent_idx = parent_idx; + } else { + /* Conflicting parent already exists, keep the original */ + if (MAIL_THREAD_NODE_EXISTS(child)) { + /* If this message gets expunged, + the parent is changed. */ + child->expunge_rebuilds = TRUE; + } else { + /* Message doesn't exist, so it was one of the node's + children that created the original reference. If + that reference gets dropped, the parent is changed. + We could catch this in one of several ways: + + a) Link to parent node gets unreferenced + b) Link to this node gets unreferenced + c) Any of the child nodes gets expunged + + b) is probably the least likely to happen, + so use it */ + child->child_unref_rebuilds = TRUE; + } + } +} + +static uint32_t +thread_link_references(struct mail_thread_cache *cache, uint32_t uid, + const struct mail_index_strmap_rec *msgid_map, + unsigned int *msgid_map_idx) +{ + uint32_t parent_idx; + + if (msgid_map->uid != uid) + return 0; + + parent_idx = msgid_map->str_idx; + msgid_map++; + *msgid_map_idx += 1; + + for (; msgid_map->uid == uid; msgid_map++) { + thread_link_reference(cache, parent_idx, msgid_map->str_idx); + parent_idx = msgid_map->str_idx; + *msgid_map_idx += 1; + } + i_assert(parent_idx < cache->first_invalid_msgid_str_idx); + return parent_idx; +} + +void mail_thread_add(struct mail_thread_cache *cache, + const struct mail_index_strmap_rec *msgid_map, + unsigned int *msgid_map_idx) +{ + struct mail_thread_node *node; + uint32_t idx, parent_idx; + + i_assert(msgid_map->ref_index == MAIL_THREAD_NODE_REF_MSGID); + i_assert(cache->last_uid <= msgid_map->uid); + + cache->last_uid = msgid_map->uid; + + idx = thread_msg_add(cache, msgid_map->uid, msgid_map->str_idx); + parent_idx = thread_link_references(cache, msgid_map->uid, + msgid_map + 1, msgid_map_idx); + + node = array_idx_modifiable(&cache->thread_nodes, idx); + if (node->parent_idx != parent_idx && node->parent_idx != 0) { + /* conflicting parent, remove it. */ + node->parent_idx = 0; + /* If this message gets expunged, we have to revert back to + the original parent. */ + node->expunge_rebuilds = TRUE; + } + if (parent_idx != 0) + thread_link_reference(cache, parent_idx, idx); + *msgid_map_idx += 1; +} + +static bool +mail_thread_unref_link(struct mail_thread_cache *cache, + uint32_t parent_idx, uint32_t child_idx) +{ + struct mail_thread_node *parent, *child; + + parent = array_idx_modifiable(&cache->thread_nodes, parent_idx); + if (parent->child_unref_rebuilds) + return FALSE; + + child = array_idx_modifiable(&cache->thread_nodes, child_idx); + i_assert(child->parent_link_refcount > 0); + + child->parent_link_refcount--; + if (child->parent_link_refcount == 0) { + /* we don't have a root anymore */ + child->parent_idx = 0; + } + return TRUE; +} + +bool mail_thread_remove(struct mail_thread_cache *cache, + const struct mail_index_strmap_rec *msgid_map, + unsigned int *msgid_map_idx) +{ + struct mail_thread_node *node; + uint32_t idx, parent_idx; + unsigned int count = 1; + + idx = msgid_map->str_idx; + i_assert(idx != 0); + + if (msgid_map->uid > cache->last_uid) { + /* this message was never added to the cache, skip */ + while (msgid_map[count].uid == msgid_map->uid) + count++; + *msgid_map_idx += count; + return TRUE; + } + + node = array_idx_modifiable(&cache->thread_nodes, idx); + if (node->expunge_rebuilds) { + /* this catches the duplicate message-id case */ + return FALSE; + } + i_assert(node->uid == msgid_map->uid); + + /* update link refcounts */ + if (msgid_map[count].uid == node->uid) { + parent_idx = msgid_map[count].str_idx; + count++; + while (msgid_map[count].uid == node->uid) { + if (!mail_thread_unref_link(cache, parent_idx, + msgid_map[count].str_idx)) + return FALSE; + parent_idx = msgid_map[count].str_idx; + count++; + } + if (!mail_thread_unref_link(cache, parent_idx, idx)) + return FALSE; + } + /* mark this message as expunged */ + node->uid = 0; + + /* we don't know (and don't want to waste time figuring out) if other + messages point to this removed message, so don't delete the node */ + *msgid_map_idx += count; + return TRUE; +} diff --git a/src/lib-storage/index/index-thread-private.h b/src/lib-storage/index/index-thread-private.h new file mode 100644 index 0000000..ed2837b --- /dev/null +++ b/src/lib-storage/index/index-thread-private.h @@ -0,0 +1,82 @@ +#ifndef INDEX_THREAD_PRIVATE_H +#define INDEX_THREAD_PRIVATE_H + +#include "crc32.h" +#include "mail-thread.h" +#include "mail-index-strmap.h" + +#define MAIL_THREAD_INDEX_SUFFIX ".thread" + +/* After initially building the index, assign first_invalid_msgid_idx to + the next unused index + SKIP_COUNT. When more messages are added and + the next valid msgid conflicts with the first invalid msgid, the invalid + msgids will be moved forward again this many indexes. */ +#define THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT \ + (4096 / sizeof(struct mail_thread_node)) + +#define HDR_MESSAGE_ID "message-id" +#define HDR_IN_REPLY_TO "in-reply-to" +#define HDR_REFERENCES "references" +#define HDR_SUBJECT "subject" + +#define MAIL_THREAD_NODE_REF_MSGID 0 +#define MAIL_THREAD_NODE_REF_INREPLYTO 1 +#define MAIL_THREAD_NODE_REF_REFERENCES1 2 + +struct mail_thread_node { + /* UID of the message, or 0 for dummy nodes */ + uint32_t uid; + /* Index for this node's parent node, 0 = this is root */ + uint32_t parent_idx; + /* Number of messages containing "this message" -> "parent message" + link, i.e. "number of links to parent node". However since parents + can change, not all of these references might be from our current + child nodes. When this refcount reaches 0, it means we must detach + from our parent. */ + unsigned int parent_link_refcount:30; + /* If uid is expunged, rebuild the thread tree. */ + bool expunge_rebuilds:1; + /* If a link between this node and its child gets unreferenced, + rebuild the thread tree. */ + bool child_unref_rebuilds:1; +}; +ARRAY_DEFINE_TYPE(mail_thread_node, struct mail_thread_node); +#define MAIL_THREAD_NODE_EXISTS(node) \ + ((node)->uid != 0) + +struct mail_thread_cache { + uint32_t last_uid; + /* indexes used for invalid Message-IDs. that means no other messages + point to them and they can safely be moved around whenever + necessary. */ + uint32_t first_invalid_msgid_str_idx; + uint32_t next_invalid_msgid_str_idx; + + struct mail_search_result *search_result; + + /* indexed by mail_index_strmap_rec.str_idx */ + ARRAY_TYPE(mail_thread_node) thread_nodes; +}; + +static inline uint32_t crc32_str_nonzero(const char *str) +{ + uint32_t value = crc32_str(str); + return value == 0 ? 1 : value; +} + +void mail_thread_add(struct mail_thread_cache *cache, + const struct mail_index_strmap_rec *msgid_map, + unsigned int *msgid_map_idx); +bool mail_thread_remove(struct mail_thread_cache *cache, + const struct mail_index_strmap_rec *msgid_map, + unsigned int *msgid_map_idx); + +struct mail_thread_iterate_context * +mail_thread_iterate_init_full(struct mail_thread_cache *cache, + struct mail *tmp_mail, + enum mail_thread_type thread_type, + bool return_seqs); + +void index_thread_mailbox_opened(struct mailbox *box); + +#endif diff --git a/src/lib-storage/index/index-thread.c b/src/lib-storage/index/index-thread.c new file mode 100644 index 0000000..2b1ba04 --- /dev/null +++ b/src/lib-storage/index/index-thread.c @@ -0,0 +1,670 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +/* doc/thread-refs.txt describes the incremental algorithm we use here. */ + +#include "lib.h" +#include "array.h" +#include "bsearch-insert-pos.h" +#include "hash2.h" +#include "message-id.h" +#include "mail-search.h" +#include "mail-search-build.h" +#include "mailbox-search-result-private.h" +#include "index-storage.h" +#include "index-thread-private.h" + + +#define MAIL_THREAD_CONTEXT(obj) \ + MODULE_CONTEXT(obj, mail_thread_storage_module) +#define MAIL_THREAD_CONTEXT_REQUIRE(obj) \ + MODULE_CONTEXT_REQUIRE(obj, mail_thread_storage_module) + +struct mail_thread_context { + struct mailbox *box; + struct mailbox_transaction_context *t; + struct mail_index_strmap_view_sync *strmap_sync; + + struct mail *tmp_mail; + struct mail_search_args *search_args; + ARRAY_TYPE(seq_range) added_uids; + + bool failed:1; + bool corrupted:1; +}; + +struct mail_thread_mailbox { + union mailbox_module_context module_ctx; + + unsigned int next_msgid_idx; + struct mail_thread_cache *cache; + + struct mail_index_strmap *strmap; + struct mail_index_strmap_view *strmap_view; + /* sorted by UID, ref_index */ + const ARRAY_TYPE(mail_index_strmap_rec) *msgid_map; + const struct hash2_table *msgid_hash; + + /* set only temporarily while needed */ + struct mail_thread_context *ctx; +}; + +static MODULE_CONTEXT_DEFINE_INIT(mail_thread_storage_module, + &mail_storage_module_register); + +static void mail_thread_clear(struct mail_thread_context *ctx); + +static int +mail_strmap_rec_get_msgid(struct mail_thread_context *ctx, + const struct mail_index_strmap_rec *rec, + const char **msgid_r) +{ + struct mail *mail = ctx->tmp_mail; + const char *msgids = NULL, *msgid; + unsigned int n = 0; + int ret; + + if (!mail_set_uid(mail, rec->uid)) + return 0; + + switch (rec->ref_index) { + case MAIL_THREAD_NODE_REF_MSGID: + /* Message-ID: header */ + ret = mail_get_first_header(mail, HDR_MESSAGE_ID, &msgids); + break; + case MAIL_THREAD_NODE_REF_INREPLYTO: + /* In-Reply-To: header */ + ret = mail_get_first_header(mail, HDR_IN_REPLY_TO, &msgids); + break; + default: + /* References: header */ + ret = mail_get_first_header(mail, HDR_REFERENCES, &msgids); + n = rec->ref_index - MAIL_THREAD_NODE_REF_REFERENCES1; + break; + } + + if (ret < 0) { + if (mail->expunged) { + /* treat it as if it didn't exist. trying to add it + again will result in failure. */ + return 0; + } + return -1; + } + + /* get the nth message-id */ + msgid = message_id_get_next(&msgids); + if (msgid != NULL) { + for (; n > 0; n--) + msgid = message_id_get_next(&msgids); + } + + if (msgid == NULL) { + /* shouldn't have happened, probably corrupted */ + mail_set_critical(mail, + "Corrupted thread index: lost Message ID %u", + rec->ref_index); + ctx->failed = TRUE; + ctx->corrupted = TRUE; + return -1; + } + *msgid_r = msgid; + return 1; +} + +static bool +mail_thread_hash_key_cmp(const char *key, + const struct mail_index_strmap_rec *rec, + void *context) +{ + struct mail_thread_mailbox *tbox = context; + struct mail_thread_context *ctx = tbox->ctx; + const char *msgid; + bool cmp_ret; + int ret; + + /* either a match or a collision, need to look closer */ + T_BEGIN { + ret = mail_strmap_rec_get_msgid(ctx, rec, &msgid); + if (ret <= 0) { + if (ret < 0) + ctx->failed = TRUE; + cmp_ret = FALSE; + } else { + cmp_ret = strcmp(msgid, key) == 0; + } + } T_END; + return cmp_ret; +} + +static int +mail_thread_hash_rec_cmp(const struct mail_index_strmap_rec *rec1, + const struct mail_index_strmap_rec *rec2, + void *context) +{ + struct mail_thread_mailbox *tbox = context; + struct mail_thread_context *ctx = tbox->ctx; + const char *msgid1, *msgid2; + int ret; + + T_BEGIN { + ret = mail_strmap_rec_get_msgid(ctx, rec1, &msgid1); + if (ret > 0) { + msgid1 = t_strdup(msgid1); + ret = mail_strmap_rec_get_msgid(ctx, rec2, &msgid2); + } + ret = ret <= 0 ? -1 : + strcmp(msgid1, msgid2) == 0; + } T_END; + return ret; +} + +static void mail_thread_strmap_remap(const uint32_t *idx_map, + unsigned int old_count, + unsigned int new_count, void *context) +{ + struct mail_thread_mailbox *tbox = context; + struct mail_thread_cache *cache = tbox->cache; + ARRAY_TYPE(mail_thread_node) new_nodes; + const struct mail_thread_node *old_nodes; + struct mail_thread_node *node; + unsigned int i, nodes_count, max, new_first_invalid, invalid_count; + + if (cache->search_result == NULL) + return; + + if (new_count == 0) { + /* strmap was reset, we'll need to rebuild thread */ + mailbox_search_result_free(&cache->search_result); + return; + } + + invalid_count = cache->next_invalid_msgid_str_idx - + cache->first_invalid_msgid_str_idx; + + old_nodes = array_get(&cache->thread_nodes, &nodes_count); + i_array_init(&new_nodes, new_count + invalid_count + 32); + + /* optimization: allocate all nodes initially */ + (void)array_idx_modifiable(&new_nodes, new_count-1); + + /* renumber existing valid nodes. all existing records in old_nodes + should also exist in idx_map since we've removed expunged messages + from the cache before committing the sync. */ + max = I_MIN(I_MIN(old_count, nodes_count), + cache->first_invalid_msgid_str_idx); + for (i = 0; i < max; i++) { + if (idx_map[i] == 0) { + /* expunged record. */ + i_assert(old_nodes[i].uid == 0); + } else { + node = array_idx_modifiable(&new_nodes, idx_map[i]); + *node = old_nodes[i]; + if (node->parent_idx != 0) { + node->parent_idx = idx_map[node->parent_idx]; + i_assert(node->parent_idx != 0); + } + } + } + + /* copy invalid nodes, if any. no other messages point to them, + so this is safe. we still need to update their parent_idx + pointers though. */ + new_first_invalid = new_count + 1 + + THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT; + for (i = 0; i < invalid_count; i++) { + node = array_idx_modifiable(&new_nodes, new_first_invalid + i); + *node = old_nodes[cache->first_invalid_msgid_str_idx + i]; + if (node->parent_idx != 0) { + node->parent_idx = idx_map[node->parent_idx]; + i_assert(node->parent_idx != 0); + } + } + cache->first_invalid_msgid_str_idx = new_first_invalid; + cache->next_invalid_msgid_str_idx = new_first_invalid + invalid_count; + + /* replace the old nodes with the renumbered ones */ + array_free(&cache->thread_nodes); + cache->thread_nodes = new_nodes; +} + +static int thread_get_mail_header(struct mail *mail, const char *name, + const char **value_r) +{ + if (mail_get_first_header(mail, name, value_r) < 0) { + if (!mail->expunged) + return -1; + + /* Message is expunged. Instead of failing the entire THREAD + command, just treat the header as nonexistent. */ + *value_r = NULL; + } + return 0; +} + +static int +mail_thread_map_add_mail(struct mail_thread_context *ctx, struct mail *mail) +{ + const char *message_id, *in_reply_to, *references, *msgid; + uint32_t ref_index; + + if (thread_get_mail_header(mail, HDR_MESSAGE_ID, &message_id) < 0 || + thread_get_mail_header(mail, HDR_REFERENCES, &references) < 0) + return -1; + + /* add Message-ID: */ + msgid = message_id_get_next(&message_id); + if (msgid != NULL) { + mail_index_strmap_view_sync_add(ctx->strmap_sync, mail->uid, + MAIL_THREAD_NODE_REF_MSGID, + msgid); + } else { + mail_index_strmap_view_sync_add_unique(ctx->strmap_sync, + mail->uid, MAIL_THREAD_NODE_REF_MSGID); + } + + /* add References: if there are any valid ones */ + msgid = message_id_get_next(&references); + if (msgid != NULL) { + ref_index = MAIL_THREAD_NODE_REF_REFERENCES1; + do { + mail_index_strmap_view_sync_add(ctx->strmap_sync, + mail->uid, + ref_index, msgid); + ref_index++; + msgid = message_id_get_next(&references); + } while (msgid != NULL); + } else { + /* no References:, use In-Reply-To: */ + if (thread_get_mail_header(mail, HDR_IN_REPLY_TO, + &in_reply_to) < 0) + return -1; + + msgid = message_id_get_next(&in_reply_to); + if (msgid != NULL) { + mail_index_strmap_view_sync_add(ctx->strmap_sync, + mail->uid, MAIL_THREAD_NODE_REF_INREPLYTO, + msgid); + } + } + if (ctx->failed) { + /* message-id lookup failed in hash compare */ + return -1; + } + return 0; +} + +static int mail_thread_index_map_build(struct mail_thread_context *ctx) +{ + static const char *wanted_headers[] = { + HDR_MESSAGE_ID, HDR_IN_REPLY_TO, HDR_REFERENCES, HDR_SUBJECT, + NULL + }; + struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(ctx->box); + struct mailbox_header_lookup_ctx *headers_ctx; + struct mail_search_args *search_args; + struct mail_search_context *search_ctx; + struct mail *mail; + uint32_t last_uid, seq1, seq2; + int ret = 0; + + if (tbox->strmap_view == NULL) { + /* first time we're threading this mailbox */ + tbox->strmap_view = + mail_index_strmap_view_open(tbox->strmap, + ctx->box->view, + mail_thread_hash_key_cmp, + mail_thread_hash_rec_cmp, + mail_thread_strmap_remap, + tbox, &tbox->msgid_map, + &tbox->msgid_hash); + } + + headers_ctx = mailbox_header_lookup_init(ctx->box, wanted_headers); + ctx->tmp_mail = mail_alloc(ctx->t, MAIL_FETCH_DATE | + MAIL_FETCH_RECEIVED_DATE, headers_ctx); + + /* add all missing UIDs */ + ctx->strmap_sync = mail_index_strmap_view_sync_init(tbox->strmap_view, + &last_uid); + mailbox_get_seq_range(ctx->box, last_uid + 1, (uint32_t)-1, + &seq1, &seq2); + if (seq1 == 0) { + /* nothing is missing */ + mail_index_strmap_view_sync_commit(&ctx->strmap_sync); + mailbox_header_lookup_unref(&headers_ctx); + return 0; + } + + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, seq1, seq2); + search_ctx = mailbox_search_init(ctx->t, search_args, NULL, + MAIL_FETCH_DATE | + MAIL_FETCH_RECEIVED_DATE, + headers_ctx); + mailbox_header_lookup_unref(&headers_ctx); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + if (mail_thread_map_add_mail(ctx, mail) < 0) { + ret = -1; + break; + } + } + if (mailbox_search_deinit(&search_ctx) < 0) + ret = -1; + + if (ret < 0) + mail_index_strmap_view_sync_rollback(&ctx->strmap_sync); + else + mail_index_strmap_view_sync_commit(&ctx->strmap_sync); + return ret; +} + +static int msgid_map_cmp(const uint32_t *uid, + const struct mail_index_strmap_rec *rec) +{ + return *uid < rec->uid ? -1 : + (*uid > rec->uid ? 1 : 0); +} + +static bool mail_thread_cache_update_removes(struct mail_thread_mailbox *tbox, + ARRAY_TYPE(seq_range) *added_uids) +{ + struct mail_thread_cache *cache = tbox->cache; + ARRAY_TYPE(seq_range) removed_uids; + const struct seq_range *uids; + const struct mail_index_strmap_rec *msgid_map; + unsigned int i, j, idx, map_count, uid_count; + uint32_t uid; + + t_array_init(&removed_uids, 64); + mailbox_search_result_sync(cache->search_result, + &removed_uids, added_uids); + + /* first check that we're not inserting any messages in the middle */ + uids = array_get(added_uids, &uid_count); + if (uid_count > 0 && uids[0].seq1 <= cache->last_uid) + return FALSE; + + /* next remove messages so we'll see early if we have to rebuild. + we expect to find all removed UIDs from msgid_map that are <= max + UID in msgid_map */ + msgid_map = array_get(tbox->msgid_map, &map_count); + uids = array_get(&removed_uids, &uid_count); + for (i = j = 0; i < uid_count; i++) { + /* find and remove from the map */ + bsearch_insert_pos(&uids[i].seq1, &msgid_map[j], + map_count - j, sizeof(*msgid_map), + msgid_map_cmp, &idx); + j += idx; + if (j == map_count) { + /* all removals after this are about messages we never + even added to the cache */ + i_assert(uids[i].seq1 > cache->last_uid); + break; + } + while (j > 0 && msgid_map[j-1].uid == msgid_map[j].uid) + j--; + + /* remove the messages from cache */ + for (uid = uids[i].seq1; uid <= uids[i].seq2; uid++) { + if (j == map_count) { + i_assert(uid > cache->last_uid); + break; + } + i_assert(msgid_map[j].uid == uid); + if (!mail_thread_remove(cache, msgid_map + j, &j)) + return FALSE; + } + } + return TRUE; +} + +static void mail_thread_cache_update_adds(struct mail_thread_mailbox *tbox, + ARRAY_TYPE(seq_range) *added_uids) +{ + struct mail_thread_cache *cache = tbox->cache; + const struct seq_range *uids; + const struct mail_index_strmap_rec *msgid_map; + unsigned int i, j, map_count, uid_count; + uint32_t uid; + + /* everything removed successfully, add the new messages. all of them + should already be in msgid_map. */ + uids = array_get(added_uids, &uid_count); + if (uid_count == 0) + return; + + (void)array_bsearch_insert_pos(tbox->msgid_map, &uids[0].seq1, + msgid_map_cmp, &j); + msgid_map = array_get(tbox->msgid_map, &map_count); + i_assert(j < map_count); + while (j > 0 && msgid_map[j-1].uid == msgid_map[j].uid) + j--; + + for (i = 0; i < uid_count; i++) { + for (uid = uids[i].seq1; uid <= uids[i].seq2; uid++) { + while (j < map_count && msgid_map[j].uid < uid) + j++; + i_assert(j < map_count && msgid_map[j].uid == uid); + mail_thread_add(cache, msgid_map+j, &j); + } + } +} + +static void +mail_thread_cache_fix_invalid_indexes(struct mail_thread_mailbox *tbox) +{ + struct mail_thread_cache *cache = tbox->cache; + uint32_t highest_idx, new_first_idx, count; + + highest_idx = mail_index_strmap_view_get_highest_idx(tbox->strmap_view); + new_first_idx = highest_idx + 1 + + THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT; + count = cache->next_invalid_msgid_str_idx - + cache->first_invalid_msgid_str_idx; + + if (count == 0) { + /* there are no invalid indexes yet, we can update the first + invalid index position to delay conflicts. */ + cache->first_invalid_msgid_str_idx = + cache->next_invalid_msgid_str_idx = new_first_idx; + } else if (highest_idx >= cache->first_invalid_msgid_str_idx) { + /* conflict - move the invalid indexes forward */ + array_copy(&cache->thread_nodes.arr, new_first_idx, + &cache->thread_nodes.arr, + cache->first_invalid_msgid_str_idx, count); + cache->first_invalid_msgid_str_idx = new_first_idx; + cache->next_invalid_msgid_str_idx = new_first_idx + count; + } +} + +static void mail_thread_cache_sync_remove(struct mail_thread_mailbox *tbox, + struct mail_thread_context *ctx) +{ + struct mail_thread_cache *cache = tbox->cache; + + if (cache->search_result == NULL) + return; + + if (mail_search_args_equal(ctx->search_args, + cache->search_result->search_args)) { + t_array_init(&ctx->added_uids, 64); + if (mail_thread_cache_update_removes(tbox, &ctx->added_uids)) { + /* successfully updated the cache */ + return; + } + } + /* failed to use the cache, rebuild */ + mailbox_search_result_free(&cache->search_result); +} + +static void mail_thread_cache_sync_add(struct mail_thread_mailbox *tbox, + struct mail_thread_context *ctx, + struct mail_search_context *search_ctx) +{ + struct mail_thread_cache *cache = tbox->cache; + struct mail *mail; + const struct mail_index_strmap_rec *msgid_map; + unsigned int i, count; + + mail_thread_cache_fix_invalid_indexes(tbox); + + if (cache->search_result != NULL) { + /* we already checked at sync_remove that we can use this + search result. */ + mail_thread_cache_update_adds(tbox, &ctx->added_uids); + return; + } + + cache->last_uid = 0; + cache->first_invalid_msgid_str_idx = cache->next_invalid_msgid_str_idx = + mail_index_strmap_view_get_highest_idx(tbox->strmap_view) + 1 + + THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT; + array_clear(&cache->thread_nodes); + + cache->search_result = + mailbox_search_result_save(search_ctx, + MAILBOX_SEARCH_RESULT_FLAG_UPDATE | + MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC); + + msgid_map = array_get(tbox->msgid_map, &count); + /* we're relying on the array being zero-terminated (outside used + count - kind of kludgy) */ + i_assert(msgid_map[count].uid == 0); + i = 0; + while (i < count && mailbox_search_next(search_ctx, &mail)) { + while (msgid_map[i].uid < mail->uid) + i++; + i_assert(i < count); + mail_thread_add(cache, msgid_map+i, &i); + } +} + +int mail_thread_init(struct mailbox *box, struct mail_search_args *args, + struct mail_thread_context **ctx_r) +{ + struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(box); + struct mail_thread_context *ctx; + struct mail_search_context *search_ctx; + int ret; + + i_assert(tbox->ctx == NULL); + + struct event_reason *reason = event_reason_begin("mailbox:thread"); + if (args != NULL) + mail_search_args_ref(args); + else { + args = mail_search_build_init(); + mail_search_build_add_all(args); + mail_search_args_init(args, box, FALSE, NULL); + } + + ctx = i_new(struct mail_thread_context, 1); + ctx->box = box; + ctx->search_args = args; + ctx->t = mailbox_transaction_begin(ctx->box, 0, __func__); + /* perform search first, so we don't break if there are INTHREAD keys */ + search_ctx = mailbox_search_init(ctx->t, args, NULL, 0, NULL); + + tbox->ctx = ctx; + + mail_thread_cache_sync_remove(tbox, ctx); + ret = mail_thread_index_map_build(ctx); + if (ret == 0) + mail_thread_cache_sync_add(tbox, ctx, search_ctx); + if (mailbox_search_deinit(&search_ctx) < 0) + ret = -1; + if (ctx->failed) { + ret = -1; + if (ctx->corrupted) + mail_index_strmap_view_set_corrupted(tbox->strmap_view); + } + event_reason_end(&reason); + if (ret < 0) { + mail_thread_deinit(&ctx); + return -1; + } else { + i_zero(&ctx->added_uids); + *ctx_r = ctx; + return 0; + } +} + +static void mail_thread_clear(struct mail_thread_context *ctx) +{ + mail_free(&ctx->tmp_mail); + (void)mailbox_transaction_commit(&ctx->t); +} + +void mail_thread_deinit(struct mail_thread_context **_ctx) +{ + struct mail_thread_context *ctx = *_ctx; + struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(ctx->box); + + *_ctx = NULL; + + mail_thread_clear(ctx); + mail_search_args_unref(&ctx->search_args); + tbox->ctx = NULL; + i_free(ctx); +} + +struct mail_thread_iterate_context * +mail_thread_iterate_init(struct mail_thread_context *ctx, + enum mail_thread_type thread_type, bool write_seqs) +{ + struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(ctx->box); + + return mail_thread_iterate_init_full(tbox->cache, ctx->tmp_mail, + thread_type, write_seqs); +} + +static void mail_thread_mailbox_close(struct mailbox *box) +{ + struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(box); + + i_assert(tbox->ctx == NULL); + + if (tbox->strmap_view != NULL) + mail_index_strmap_view_close(&tbox->strmap_view); + if (tbox->cache->search_result != NULL) + mailbox_search_result_free(&tbox->cache->search_result); + tbox->module_ctx.super.close(box); +} + +static void mail_thread_mailbox_free(struct mailbox *box) +{ + struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(box); + + mail_index_strmap_deinit(&tbox->strmap); + tbox->module_ctx.super.free(box); + + array_free(&tbox->cache->thread_nodes); + i_free(tbox->cache); + i_free(tbox); +} + +void index_thread_mailbox_opened(struct mailbox *box) +{ + struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT(box); + + if (tbox != NULL) { + /* mailbox was already opened+closed once. */ + return; + } + + tbox = i_new(struct mail_thread_mailbox, 1); + tbox->module_ctx.super = box->v; + box->v.close = mail_thread_mailbox_close; + box->v.free = mail_thread_mailbox_free; + + tbox->strmap = mail_index_strmap_init(box->index, + MAIL_THREAD_INDEX_SUFFIX); + tbox->next_msgid_idx = 1; + + tbox->cache = i_new(struct mail_thread_cache, 1); + i_array_init(&tbox->cache->thread_nodes, 128); + + MODULE_CONTEXT_SET(box, mail_thread_storage_module, tbox); +} diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c new file mode 100644 index 0000000..e3138b6 --- /dev/null +++ b/src/lib-storage/index/index-transaction.c @@ -0,0 +1,230 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "dict.h" +#include "index-storage.h" +#include "index-sync-private.h" +#include "index-pop3-uidl.h" +#include "index-mail.h" + +static void index_transaction_free(struct mailbox_transaction_context *t) +{ + if (t->view_pvt != NULL) + mail_index_view_close(&t->view_pvt); + mail_cache_view_close(&t->cache_view); + mail_index_view_close(&t->view); + if (array_is_created(&t->pvt_saves)) + array_free(&t->pvt_saves); + array_free(&t->module_contexts); + i_free(t->reason); + i_free(t); +} + +static int +index_transaction_index_commit(struct mail_index_transaction *index_trans, + struct mail_index_transaction_commit_result *result_r) +{ + struct mailbox_transaction_context *t = + MAIL_STORAGE_CONTEXT_REQUIRE(index_trans); + struct index_mailbox_sync_pvt_context *pvt_sync_ctx = NULL; + const char *error; + int ret = 0; + + index_pop3_uidl_update_exists_finish(t); + + if (t->attr_pvt_trans != NULL) { + if (dict_transaction_commit(&t->attr_pvt_trans, &error) < 0) { + mailbox_set_critical(t->box, + "Dict private transaction commit failed: %s", error); + ret = -1; + } + } + if (t->attr_shared_trans != NULL) { + if (dict_transaction_commit(&t->attr_shared_trans, &error) < 0) { + mailbox_set_critical(t->box, + "Dict shared transaction commit failed: %s", error); + ret = -1; + } + } + + if (t->save_ctx != NULL) { + mailbox_save_context_deinit(t->save_ctx); + if (ret < 0) { + t->box->v.transaction_save_rollback(t->save_ctx); + t->save_ctx = NULL; + } else if (t->box->v.transaction_save_commit_pre(t->save_ctx) < 0) { + t->save_ctx = NULL; + ret = -1; + } + } + + if (array_is_created(&t->pvt_saves)) { + if (index_mailbox_sync_pvt_init(t->box, TRUE, 0, &pvt_sync_ctx) < 0) + ret = -1; + } + + i_assert(t->mail_ref_count == 0); + if (ret < 0) + t->super.rollback(index_trans); + else { + if (t->super.commit(index_trans, result_r) < 0) { + mailbox_set_index_error(t->box); + ret = -1; + } else { + t->changes->changes_mask = result_r->changes_mask; + } + } + + if (t->save_ctx == NULL) { + } else if (ret >= 0) { + i_assert(t->save_ctx->dest_mail == NULL); + t->box->v.transaction_save_commit_post(t->save_ctx, result_r); + } else { + t->box->v.transaction_save_rollback(t->save_ctx); + } + + if (pvt_sync_ctx != NULL) { + if (index_mailbox_sync_pvt_newmails(pvt_sync_ctx, t) < 0) { + /* failed to add private flags. a bit too late to + return failure though, so just ignore silently */ + } + index_mailbox_sync_pvt_deinit(&pvt_sync_ctx); + } + + if (ret < 0) + mail_index_set_error_nolog(t->box->index, mailbox_get_last_error(t->box, NULL)); + index_transaction_free(t); + return ret; +} + +static void +index_transaction_index_rollback(struct mail_index_transaction *index_trans) +{ + struct mailbox_transaction_context *t = + MAIL_STORAGE_CONTEXT_REQUIRE(index_trans); + + dict_transaction_rollback(&t->attr_pvt_trans); + dict_transaction_rollback(&t->attr_shared_trans); + + if (t->save_ctx != NULL) { + mailbox_save_context_deinit(t->save_ctx); + t->box->v.transaction_save_rollback(t->save_ctx); + } + + i_assert(t->mail_ref_count == 0); + t->super.rollback(index_trans); + index_transaction_free(t); +} + +static enum mail_index_transaction_flags +index_transaction_flags_get(enum mailbox_transaction_flags flags) +{ + enum mail_index_transaction_flags itrans_flags; + + itrans_flags = MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES; + if ((flags & MAILBOX_TRANSACTION_FLAG_HIDE) != 0) + itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_HIDE; + if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) + itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + if ((flags & MAILBOX_TRANSACTION_FLAG_SYNC) != 0) + itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_SYNC; + return itrans_flags; +} + +void index_transaction_init_pvt(struct mailbox_transaction_context *t) +{ + enum mail_index_transaction_flags itrans_flags; + + if (t->box->view_pvt == NULL || t->itrans_pvt != NULL) + return; + + itrans_flags = index_transaction_flags_get(t->flags); + t->itrans_pvt = mail_index_transaction_begin(t->box->view_pvt, + itrans_flags); + t->view_pvt = mail_index_transaction_open_updated_view(t->itrans_pvt); +} + +void index_transaction_init(struct mailbox_transaction_context *t, + struct mailbox *box, + enum mailbox_transaction_flags flags, + const char *reason) +{ + enum mail_index_transaction_flags itrans_flags; + + i_assert(box->opened); + + itrans_flags = index_transaction_flags_get(flags); + if ((flags & MAILBOX_TRANSACTION_FLAG_REFRESH) != 0) + mail_index_refresh(box->index); + + t->flags = flags; + t->box = box; + t->reason = i_strdup(reason); + t->itrans = mail_index_transaction_begin(box->view, itrans_flags); + t->view = mail_index_transaction_open_updated_view(t->itrans); + + array_create(&t->module_contexts, default_pool, + sizeof(void *), 5); + + t->cache_view = mail_cache_view_open(box->cache, t->view); + t->cache_trans = mail_cache_get_transaction(t->cache_view, t->itrans); + + if ((flags & MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC) != 0) + mail_cache_view_update_cache_decisions(t->cache_view, FALSE); + + /* set up after mail_cache_get_transaction(), so that we'll still + have the cache_trans available in _index_commit() */ + t->super = t->itrans->v; + t->itrans->v.commit = index_transaction_index_commit; + t->itrans->v.rollback = index_transaction_index_rollback; + MODULE_CONTEXT_SET(t->itrans, mail_storage_mail_index_module, t); +} + +struct mailbox_transaction_context * +index_transaction_begin(struct mailbox *box, + enum mailbox_transaction_flags flags, + const char *reason) +{ + struct mailbox_transaction_context *t; + + t = i_new(struct mailbox_transaction_context, 1); + index_transaction_init(t, box, flags, reason); + return t; +} + +int index_transaction_commit(struct mailbox_transaction_context *t, + struct mail_transaction_commit_changes *changes_r) +{ + struct mailbox *box = t->box; + struct mail_index_transaction *itrans = t->itrans; + struct mail_index_transaction_commit_result result; + int ret = 0; + + i_zero(changes_r); + changes_r->pool = pool_alloconly_create(MEMPOOL_GROWING + "transaction changes", 512); + p_array_init(&changes_r->saved_uids, changes_r->pool, 32); + t->changes = changes_r; + + if (t->itrans_pvt != NULL) + ret = mail_index_transaction_commit(&t->itrans_pvt); + if (mail_index_transaction_commit_full(&itrans, &result) < 0) + ret = -1; + t = NULL; + + if (ret < 0 && mail_index_is_deleted(box->index)) + mailbox_set_deleted(box); + + changes_r->ignored_modseq_changes = result.ignored_modseq_changes; + return ret; +} + +void index_transaction_rollback(struct mailbox_transaction_context *t) +{ + struct mail_index_transaction *itrans = t->itrans; + + if (t->itrans_pvt != NULL) + mail_index_transaction_rollback(&t->itrans_pvt); + mail_index_transaction_rollback(&itrans); +} diff --git a/src/lib-storage/index/istream-mail.c b/src/lib-storage/index/istream-mail.c new file mode 100644 index 0000000..1477823 --- /dev/null +++ b/src/lib-storage/index/istream-mail.c @@ -0,0 +1,173 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-storage-private.h" +#include "istream-private.h" +#include "index-mail.h" +#include "istream-mail.h" + +struct mail_istream { + struct istream_private istream; + + struct mail *mail; + uoff_t expected_size; + bool files_read_increased:1; + bool input_has_body:1; +}; + +static bool i_stream_mail_try_get_cached_size(struct mail_istream *mstream) +{ + struct mail *mail = mstream->mail; + enum mail_lookup_abort orig_lookup_abort; + + if (mstream->expected_size != UOFF_T_MAX) + return TRUE; + + /* make sure this call doesn't change any existing error message, + just in case there's already something important in it. */ + mail_storage_last_error_push(mail->box->storage); + orig_lookup_abort = mail->lookup_abort; + mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + if (mail_get_physical_size(mail, &mstream->expected_size) < 0) + mstream->expected_size = UOFF_T_MAX; + mail->lookup_abort = orig_lookup_abort; + mail_storage_last_error_pop(mail->box->storage); + return mstream->expected_size != UOFF_T_MAX; +} + +static const char * +i_stream_mail_get_cached_mail_id(struct mail_istream *mstream ATTR_UNUSED) +{ +#if 0 + /* FIXME: This function may get called in the middle of header parsing, + which then goes into parsing cached headers and causes crashes. + So disable this for now. Eventually it would be nice if recursion + was possible by each parser using its own private struct. */ + static const char *headers[] = { + "Message-Id", + "Date", + "Subject" + }; + struct mail *mail = mstream->mail; + enum mail_lookup_abort orig_lookup_abort; + const char *value, *ret = ""; + unsigned int i; + + orig_lookup_abort = mail->lookup_abort; + mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; + for (i = 0; i < N_ELEMENTS(headers); i++) { + if (mail_get_first_header(mail, headers[i], &value) > 0) { + ret = t_strdup_printf("%s=%s", headers[i], value); + break; + } + } + mail->lookup_abort = orig_lookup_abort; + return ret; +#else + return ""; +#endif +} + +static void +i_stream_mail_set_size_corrupted(struct mail_istream *mstream, size_t size) +{ + uoff_t cur_size = mstream->istream.istream.v_offset + size; + const char *str, *mail_id; + char chr; + + if (mstream->expected_size < cur_size) { + /* input stream is larger than cached message size */ + str = "smaller"; + chr = '<'; + mstream->istream.istream.stream_errno = EINVAL; + } else { + /* input stream is smaller than cached message size */ + str = "larger"; + chr = '>'; + mstream->istream.istream.stream_errno = EPIPE; + } + + mail_id = i_stream_mail_get_cached_mail_id(mstream); + if (mail_id[0] != '\0') + mail_id = t_strconcat(", cached ", mail_id, NULL); + io_stream_set_error(&mstream->istream.iostream, + "Cached message size %s than expected " + "(%"PRIuUOFF_T" %c %"PRIuUOFF_T", box=%s, UID=%u%s)", str, + mstream->expected_size, chr, cur_size, + mailbox_get_vname(mstream->mail->box), + mstream->mail->uid, mail_id); + mail_set_cache_corrupted(mstream->mail, MAIL_FETCH_PHYSICAL_SIZE, + t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(&mstream->istream.istream), + mstream->istream.iostream.error)); +} + +static ssize_t +i_stream_mail_read(struct istream_private *stream) +{ + struct mail_istream *mstream = (struct mail_istream *)stream; + size_t size; + ssize_t ret; + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + + ret = i_stream_read_copy_from_parent(&stream->istream); + size = i_stream_get_data_size(&stream->istream); + if (ret > 0) { + mstream->mail->transaction->stats.files_read_bytes += ret; + if (!mstream->files_read_increased) { + mstream->files_read_increased = TRUE; + mstream->mail->transaction->stats.files_read_count++; + } + if (mstream->expected_size < stream->istream.v_offset + size) { + i_stream_mail_set_size_corrupted(mstream, size); + /* istream code expects that the position has not changed + when read error occurs, so move pos back. */ + i_assert(stream->pos >= (size_t)ret); + stream->pos -= ret; + return -1; + } + } else if (ret == -1 && stream->istream.eof) { + if (!mstream->input_has_body) { + /* trying to read past the header, but this stream + doesn't have the body */ + return -1; + } + if (stream->istream.stream_errno != 0) { + if (stream->istream.stream_errno == ENOENT) { + /* update mail's expunged-flag if needed */ + index_mail_refresh_expunged(mstream->mail); + } + return -1; + } + if (i_stream_mail_try_get_cached_size(mstream) && + mstream->expected_size > stream->istream.v_offset + size) { + i_stream_mail_set_size_corrupted(mstream, size); + return -1; + } + } + return ret; +} + +struct istream *i_stream_create_mail(struct mail *mail, struct istream *input, + bool input_has_body) +{ + struct mail_istream *mstream; + + mstream = i_new(struct mail_istream, 1); + mstream->mail = mail; + mstream->input_has_body = input_has_body; + mstream->expected_size = UOFF_T_MAX; + (void)i_stream_mail_try_get_cached_size(mstream); + mstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + mstream->istream.stream_size_passthrough = TRUE; + + mstream->istream.read = i_stream_mail_read; + + mstream->istream.istream.readable_fd = input->readable_fd; + mstream->istream.istream.blocking = input->blocking; + mstream->istream.istream.seekable = input->seekable; + return i_stream_create(&mstream->istream, input, + i_stream_get_fd(input), 0); +} diff --git a/src/lib-storage/index/istream-mail.h b/src/lib-storage/index/istream-mail.h new file mode 100644 index 0000000..24bd567 --- /dev/null +++ b/src/lib-storage/index/istream-mail.h @@ -0,0 +1,7 @@ +#ifndef ISTREAM_MAIL_H +#define ISTREAM_MAIL_H + +struct istream *i_stream_create_mail(struct mail *mail, struct istream *input, + bool input_has_body); + +#endif diff --git a/src/lib-storage/index/maildir/Makefile.am b/src/lib-storage/index/maildir/Makefile.am new file mode 100644 index 0000000..7acf249 --- /dev/null +++ b/src/lib-storage/index/maildir/Makefile.am @@ -0,0 +1,36 @@ +noinst_LTLIBRARIES = libstorage_maildir.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_maildir_la_SOURCES = \ + maildir-copy.c \ + maildir-filename.c \ + maildir-filename-flags.c \ + maildir-keywords.c \ + maildir-mail.c \ + maildir-save.c \ + maildir-settings.c \ + maildir-storage.c \ + maildir-sync.c \ + maildir-sync-index.c \ + maildir-uidlist.c \ + maildir-util.c + +headers = \ + maildir-filename.h \ + maildir-filename-flags.h \ + maildir-keywords.h \ + maildir-storage.h \ + maildir-settings.h \ + maildir-sync.h \ + maildir-uidlist.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/maildir/Makefile.in b/src/lib-storage/index/maildir/Makefile.in new file mode 100644 index 0000000..fdc56d1 --- /dev/null +++ b/src/lib-storage/index/maildir/Makefile.in @@ -0,0 +1,875 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/maildir +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_maildir_la_LIBADD = +am_libstorage_maildir_la_OBJECTS = maildir-copy.lo maildir-filename.lo \ + maildir-filename-flags.lo maildir-keywords.lo maildir-mail.lo \ + maildir-save.lo maildir-settings.lo maildir-storage.lo \ + maildir-sync.lo maildir-sync-index.lo maildir-uidlist.lo \ + maildir-util.lo +libstorage_maildir_la_OBJECTS = $(am_libstorage_maildir_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/maildir-copy.Plo \ + ./$(DEPDIR)/maildir-filename-flags.Plo \ + ./$(DEPDIR)/maildir-filename.Plo \ + ./$(DEPDIR)/maildir-keywords.Plo ./$(DEPDIR)/maildir-mail.Plo \ + ./$(DEPDIR)/maildir-save.Plo ./$(DEPDIR)/maildir-settings.Plo \ + ./$(DEPDIR)/maildir-storage.Plo \ + ./$(DEPDIR)/maildir-sync-index.Plo \ + ./$(DEPDIR)/maildir-sync.Plo ./$(DEPDIR)/maildir-uidlist.Plo \ + ./$(DEPDIR)/maildir-util.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_maildir_la_SOURCES) +DIST_SOURCES = $(libstorage_maildir_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_maildir.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_maildir_la_SOURCES = \ + maildir-copy.c \ + maildir-filename.c \ + maildir-filename-flags.c \ + maildir-keywords.c \ + maildir-mail.c \ + maildir-save.c \ + maildir-settings.c \ + maildir-storage.c \ + maildir-sync.c \ + maildir-sync-index.c \ + maildir-uidlist.c \ + maildir-util.c + +headers = \ + maildir-filename.h \ + maildir-filename-flags.h \ + maildir-keywords.h \ + maildir-storage.h \ + maildir-settings.h \ + maildir-sync.h \ + maildir-uidlist.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/maildir/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/maildir/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_maildir.la: $(libstorage_maildir_la_OBJECTS) $(libstorage_maildir_la_DEPENDENCIES) $(EXTRA_libstorage_maildir_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_maildir_la_OBJECTS) $(libstorage_maildir_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-copy.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-filename-flags.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-filename.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-keywords.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-sync-index.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-uidlist.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-util.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/maildir-copy.Plo + -rm -f ./$(DEPDIR)/maildir-filename-flags.Plo + -rm -f ./$(DEPDIR)/maildir-filename.Plo + -rm -f ./$(DEPDIR)/maildir-keywords.Plo + -rm -f ./$(DEPDIR)/maildir-mail.Plo + -rm -f ./$(DEPDIR)/maildir-save.Plo + -rm -f ./$(DEPDIR)/maildir-settings.Plo + -rm -f ./$(DEPDIR)/maildir-storage.Plo + -rm -f ./$(DEPDIR)/maildir-sync-index.Plo + -rm -f ./$(DEPDIR)/maildir-sync.Plo + -rm -f ./$(DEPDIR)/maildir-uidlist.Plo + -rm -f ./$(DEPDIR)/maildir-util.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/maildir-copy.Plo + -rm -f ./$(DEPDIR)/maildir-filename-flags.Plo + -rm -f ./$(DEPDIR)/maildir-filename.Plo + -rm -f ./$(DEPDIR)/maildir-keywords.Plo + -rm -f ./$(DEPDIR)/maildir-mail.Plo + -rm -f ./$(DEPDIR)/maildir-save.Plo + -rm -f ./$(DEPDIR)/maildir-settings.Plo + -rm -f ./$(DEPDIR)/maildir-storage.Plo + -rm -f ./$(DEPDIR)/maildir-sync-index.Plo + -rm -f ./$(DEPDIR)/maildir-sync.Plo + -rm -f ./$(DEPDIR)/maildir-uidlist.Plo + -rm -f ./$(DEPDIR)/maildir-util.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/maildir/maildir-copy.c b/src/lib-storage/index/maildir/maildir-copy.c new file mode 100644 index 0000000..1645ddb --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-copy.c @@ -0,0 +1,149 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "nfs-workarounds.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-filename.h" +#include "maildir-keywords.h" +#include "maildir-sync.h" +#include "index-mail.h" +#include "mail-copy.h" + +#include <unistd.h> +#include <sys/stat.h> + +struct hardlink_ctx { + const char *dest_path; + bool success:1; +}; + +static int do_hardlink(struct maildir_mailbox *mbox, const char *path, + struct hardlink_ctx *ctx) +{ + int ret; + + if (mbox->storage->storage.set->mail_nfs_storage) + ret = nfs_safe_link(path, ctx->dest_path, FALSE); + else + ret = link(path, ctx->dest_path); + if (ret < 0) { + if (errno == ENOENT) + return 0; + + if (ENOQUOTA(errno)) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + return -1; + } + + /* we could handle the EEXIST condition by changing the + filename, but it practically never happens so just fallback + to standard copying for the rare cases when it does. */ + if (errno == EACCES || ECANTLINK(errno) || errno == EEXIST) + return 1; + + mailbox_set_critical(&mbox->box, "link(%s, %s) failed: %m", + path, ctx->dest_path); + return -1; + } + + ctx->success = TRUE; + return 1; +} + +static int +maildir_copy_hardlink(struct mail_save_context *ctx, struct mail *mail) +{ + struct maildir_mailbox *dest_mbox = MAILDIR_MAILBOX(ctx->transaction->box); + struct maildir_mailbox *src_mbox; + struct maildir_filename *mf; + struct hardlink_ctx do_ctx; + const char *path, *guid, *dest_fname; + uoff_t vsize, size; + enum mail_lookup_abort old_abort; + + if (strcmp(mail->box->storage->name, MAILDIR_STORAGE_NAME) == 0) + src_mbox = MAILDIR_MAILBOX(mail->box); + else if (strcmp(mail->box->storage->name, "raw") == 0) { + /* lda uses raw format */ + src_mbox = NULL; + } else { + /* Can't hard link files from the source storage */ + return 0; + } + + /* hard link to tmp/ with a newly generated filename and later when we + have uidlist locked, move it to new/cur. */ + dest_fname = maildir_filename_generate(); + i_zero(&do_ctx); + do_ctx.dest_path = + t_strdup_printf("%s/tmp/%s", mailbox_get_path(&dest_mbox->box), + dest_fname); + if (src_mbox != NULL) { + /* maildir */ + if (maildir_file_do(src_mbox, mail->uid, + do_hardlink, &do_ctx) < 0) + return -1; + } else { + /* raw / lda */ + if (mail_get_special(mail, MAIL_FETCH_STORAGE_ID, + &path) < 0 || *path == '\0') + return 0; + if (do_hardlink(dest_mbox, path, &do_ctx) < 0) + return -1; + } + + if (!do_ctx.success) { + /* couldn't copy with hardlinking, fallback to copying */ + return 0; + } + + /* hardlinked to tmp/, treat as normal copied mail */ + mf = maildir_save_add(ctx, dest_fname, mail); + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) == 0) { + if (*guid != '\0') + maildir_save_set_dest_basename(ctx, mf, guid); + } + + /* finish copying keywords */ + maildir_save_finish_keywords(ctx); + + /* remember size/vsize if possible */ + old_abort = mail->lookup_abort; + mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + if (mail_get_physical_size(mail, &size) < 0) + size = UOFF_T_MAX; + if (mail_get_virtual_size(mail, &vsize) < 0) + vsize = UOFF_T_MAX; + maildir_save_set_sizes(mf, size, vsize); + mail->lookup_abort = old_abort; + return 1; +} + +int maildir_copy(struct mail_save_context *ctx, struct mail *mail) +{ + struct mailbox_transaction_context *_t = ctx->transaction; + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_t->box); + int ret; + + i_assert((_t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (mbox->storage->set->maildir_copy_with_hardlinks && + mail_storage_copy_can_use_hardlink(mail->box, &mbox->box)) { + T_BEGIN { + ret = maildir_copy_hardlink(ctx, mail); + } T_END; + + if (ret != 0) { + index_save_context_free(ctx); + return ret > 0 ? 0 : -1; + } + + /* non-fatal hardlinking failure, try the slow way */ + } + + return mail_storage_copy(ctx, mail); +} diff --git a/src/lib-storage/index/maildir/maildir-filename-flags.c b/src/lib-storage/index/maildir/maildir-filename-flags.c new file mode 100644 index 0000000..ac68908 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename-flags.c @@ -0,0 +1,185 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "maildir-storage.h" +#include "maildir-keywords.h" +#include "maildir-filename-flags.h" + + +void maildir_filename_flags_get(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags *flags_r, + ARRAY_TYPE(keyword_indexes) *keywords_r) +{ + const char *info; + + array_clear(keywords_r); + *flags_r = 0; + + info = strrchr(fname, MAILDIR_INFO_SEP); + if (info == NULL || info[1] != '2' || info[2] != MAILDIR_FLAGS_SEP) + return; + + for (info += 3; *info != '\0' && *info != MAILDIR_FLAGS_SEP; info++) { + switch (*info) { + case 'R': /* replied */ + *flags_r |= MAIL_ANSWERED; + break; + case 'S': /* seen */ + *flags_r |= MAIL_SEEN; + break; + case 'T': /* trashed */ + *flags_r |= MAIL_DELETED; + break; + case 'D': /* draft */ + *flags_r |= MAIL_DRAFT; + break; + case 'F': /* flagged */ + *flags_r |= MAIL_FLAGGED; + break; + default: + if (*info >= MAILDIR_KEYWORD_FIRST && + *info <= MAILDIR_KEYWORD_LAST) { + int idx; + + idx = maildir_keywords_char_idx(ctx, *info); + if (idx < 0) { + /* unknown keyword. */ + break; + } + + array_push_back(keywords_r, + (unsigned int *)&idx); + break; + } + + /* unknown flag - ignore */ + break; + } + } +} + +static int char_cmp(const void *p1, const void *p2) +{ + const unsigned char *c1 = p1, *c2 = p2; + + return *c1 - *c2; +} + +static void +maildir_filename_append_keywords(struct maildir_keywords_sync_ctx *ctx, + ARRAY_TYPE(keyword_indexes) *keywords, + string_t *fname) +{ + const unsigned int *indexes; + unsigned int i, count; + size_t start = str_len(fname); + char chr; + + indexes = array_get(keywords, &count); + for (i = 0; i < count; i++) { + chr = maildir_keywords_idx_char(ctx, indexes[i]); + if (chr != '\0') + str_append_c(fname, chr); + } + + qsort(str_c_modifiable(fname) + start, str_len(fname) - start, 1, + char_cmp); +} + +static const char * ATTR_NULL(1, 4) +maildir_filename_flags_full_set(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags flags, + ARRAY_TYPE(keyword_indexes) *keywords) +{ + string_t *flags_str; + enum mail_flags flags_left; + const char *info, *oldflags; + int nextflag; + + /* remove the old :info from file name, and get the old flags */ + info = strrchr(fname, MAILDIR_INFO_SEP); + if (info != NULL && strrchr(fname, '/') > info) + info = NULL; + + oldflags = ""; + if (info != NULL) { + fname = t_strdup_until(fname, info); + if (info[1] == '2' && info[2] == MAILDIR_FLAGS_SEP) + oldflags = info+3; + } + + /* insert the new flags between old flags. flags must be sorted by + their ASCII code. unknown flags are kept. */ + flags_str = t_str_new(256); + str_append(flags_str, fname); + str_append(flags_str, MAILDIR_FLAGS_FULL_SEP); + flags_left = flags; + for (;;) { + /* skip all known flags */ + while (*oldflags == 'D' || *oldflags == 'F' || + *oldflags == 'R' || *oldflags == 'S' || + *oldflags == 'T' || + (*oldflags >= MAILDIR_KEYWORD_FIRST && + *oldflags <= MAILDIR_KEYWORD_LAST)) + oldflags++; + + nextflag = *oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP ? + 256 : (unsigned char) *oldflags; + + if ((flags_left & MAIL_DRAFT) != 0 && nextflag > 'D') { + str_append_c(flags_str, 'D'); + flags_left &= ENUM_NEGATE(MAIL_DRAFT); + } + if ((flags_left & MAIL_FLAGGED) != 0 && nextflag > 'F') { + str_append_c(flags_str, 'F'); + flags_left &= ENUM_NEGATE(MAIL_FLAGGED); + } + if ((flags_left & MAIL_ANSWERED) != 0 && nextflag > 'R') { + str_append_c(flags_str, 'R'); + flags_left &= ENUM_NEGATE(MAIL_ANSWERED); + } + if ((flags_left & MAIL_SEEN) != 0 && nextflag > 'S') { + str_append_c(flags_str, 'S'); + flags_left &= ENUM_NEGATE(MAIL_SEEN); + } + if ((flags_left & MAIL_DELETED) != 0 && nextflag > 'T') { + str_append_c(flags_str, 'T'); + flags_left &= ENUM_NEGATE(MAIL_DELETED); + } + + if (keywords != NULL && array_is_created(keywords) && + nextflag > MAILDIR_KEYWORD_FIRST) { + maildir_filename_append_keywords(ctx, keywords, + flags_str); + keywords = NULL; + } + + if (*oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP) + break; + + str_append_c(flags_str, *oldflags); + oldflags++; + } + + if (*oldflags == MAILDIR_FLAGS_SEP) { + /* another flagset, we don't know about these, just keep them */ + while (*oldflags != '\0') + str_append_c(flags_str, *oldflags++); + } + + return str_c(flags_str); +} + +const char *maildir_filename_flags_set(const char *fname, enum mail_flags flags) +{ + return maildir_filename_flags_full_set(NULL, fname, flags, NULL); +} + +const char *maildir_filename_flags_kw_set(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags flags, + ARRAY_TYPE(keyword_indexes) *keywords) +{ + return maildir_filename_flags_full_set(ctx, fname, flags, keywords); +} diff --git a/src/lib-storage/index/maildir/maildir-filename-flags.h b/src/lib-storage/index/maildir/maildir-filename-flags.h new file mode 100644 index 0000000..2676e5c --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename-flags.h @@ -0,0 +1,13 @@ +#ifndef MAILDIR_FILENAME_FLAGS_H +#define MAILDIR_FILENAME_FLAGS_H + +void maildir_filename_flags_get(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags *flags_r, + ARRAY_TYPE(keyword_indexes) *keywords_r); + +const char *maildir_filename_flags_set(const char *fname, enum mail_flags flags); +const char *maildir_filename_flags_kw_set(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags flags, + ARRAY_TYPE(keyword_indexes) *keywords); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-filename.c b/src/lib-storage/index/maildir/maildir-filename.c new file mode 100644 index 0000000..9deba82 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename.c @@ -0,0 +1,143 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "time-util.h" +#include "hostpid.h" +#include "maildir-storage.h" +#include "maildir-filename.h" + +const char *maildir_filename_generate(void) +{ + static struct timeval last_tv = { 0, 0 }; + struct timeval tv; + + /* use secs + usecs to guarantee uniqueness within this process. */ + if (timeval_cmp(&ioloop_timeval, &last_tv) > 0) + tv = ioloop_timeval; + else { + tv = last_tv; + if (++tv.tv_usec == 1000000) { + tv.tv_sec++; + tv.tv_usec = 0; + } + } + last_tv = tv; + + return t_strdup_printf("%s.M%sP%s.%s", + dec2str(tv.tv_sec), dec2str(tv.tv_usec), + my_pid, my_hostname); +} + +bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r) +{ + uoff_t size = 0; + + for (; *fname != '\0'; fname++) { + i_assert(*fname != '/'); + if (*fname == ',' && fname[1] == type && fname[2] == '=') { + fname += 3; + break; + } + } + + if (*fname == '\0') + return FALSE; + + while (*fname >= '0' && *fname <= '9') { + size = size * 10 + (*fname - '0'); + fname++; + } + + if (*fname != MAILDIR_INFO_SEP && + *fname != MAILDIR_EXTRA_SEP && + *fname != '\0') + return FALSE; + + *size_r = size; + return TRUE; +} + +/* a char* hash function from ASU -- from glib */ +unsigned int ATTR_NO_SANITIZE_INTEGER +maildir_filename_base_hash(const char *s) +{ + unsigned int g, h = 0; + + while (*s != MAILDIR_INFO_SEP && *s != '\0') { + i_assert(*s != '/'); + h = (h << 4) + *s; + if ((g = h & 0xf0000000UL) != 0) { + h = h ^ (g >> 24); + h = h ^ g; + } + + s++; + } + + return h; +} + +int maildir_filename_base_cmp(const char *fname1, const char *fname2) +{ + while (*fname1 == *fname2 && *fname1 != MAILDIR_INFO_SEP && + *fname1 != '\0') { + i_assert(*fname1 != '/'); + fname1++; fname2++; + } + + if ((*fname1 == '\0' || *fname1 == MAILDIR_INFO_SEP) && + (*fname2 == '\0' || *fname2 == MAILDIR_INFO_SEP)) + return 0; + return *fname1 - *fname2; +} + +static bool maildir_fname_get_usecs(const char *fname, int *usecs_r) +{ + int usecs = 0; + + /* Assume we already read the timestamp. Next up is + ".<uniqueness>.<host>". Find usecs inside the uniqueness. */ + if (*fname != '.') + return FALSE; + + fname++; + while (*fname != '\0' && *fname != '.' && *fname != MAILDIR_INFO_SEP) { + if (*fname++ == 'M') { + while (*fname >= '0' && *fname <= '9') { + usecs = usecs * 10 + (*fname - '0'); + fname++; + } + *usecs_r = usecs; + return TRUE; + } + } + return FALSE; +} + +int maildir_filename_sort_cmp(const char *fname1, const char *fname2) +{ + const char *s1, *s2; + time_t secs1 = 0, secs2 = 0; + int ret, usecs1, usecs2; + + /* sort primarily by the timestamp in file name */ + for (s1 = fname1; *s1 >= '0' && *s1 <= '9'; s1++) + secs1 = secs1 * 10 + (*s1 - '0'); + for (s2 = fname2; *s2 >= '0' && *s2 <= '9'; s2++) + secs2 = secs2 * 10 + (*s2 - '0'); + + ret = (int)((long)secs1 - (long)secs2); + if (ret == 0) { + /* sort secondarily by microseconds, if they exist */ + if (maildir_fname_get_usecs(s1, &usecs1) && + maildir_fname_get_usecs(s2, &usecs2)) + ret = usecs1 - usecs2; + + if (ret == 0) { + /* fallback to comparing the base file name */ + ret = maildir_filename_base_cmp(s1, s2); + } + } + return ret; +} diff --git a/src/lib-storage/index/maildir/maildir-filename.h b/src/lib-storage/index/maildir/maildir-filename.h new file mode 100644 index 0000000..a691dea --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename.h @@ -0,0 +1,14 @@ +#ifndef MAILDIR_FILENAME_H +#define MAILDIR_FILENAME_H + +struct maildir_keywords_sync_ctx; + +const char *maildir_filename_generate(void); + +bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r); + +unsigned int maildir_filename_base_hash(const char *fname); +int maildir_filename_base_cmp(const char *fname1, const char *fname2); +int maildir_filename_sort_cmp(const char *fname1, const char *fname2); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-keywords.c b/src/lib-storage/index/maildir/maildir-keywords.c new file mode 100644 index 0000000..a25d112 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-keywords.c @@ -0,0 +1,499 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +/* note that everything here depends on uidlist file being locked the whole + time. that's why we don't have any locking of our own, or that we do things + that would be racy otherwise. */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "hash.h" +#include "str.h" +#include "istream.h" +#include "eacces-error.h" +#include "file-dotlock.h" +#include "write-full.h" +#include "nfs-workarounds.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" + +#include <sys/stat.h> +#include <utime.h> + +/* how many seconds to wait before overriding dovecot-keywords.lock */ +#define KEYWORDS_LOCK_STALE_TIMEOUT (60*2) + +struct maildir_keywords { + struct maildir_mailbox *mbox; + struct mail_storage *storage; + char *path; + + pool_t pool; + ARRAY_TYPE(keywords) list; + HASH_TABLE(char *, void *) hash; /* name -> idx+1 */ + + struct dotlock_settings dotlock_settings; + + time_t synced_mtime; + bool synced:1; + bool changed:1; +}; + +struct maildir_keywords_sync_ctx { + struct maildir_keywords *mk; + struct mail_index *index; + + const ARRAY_TYPE(keywords) *keywords; + ARRAY(char) idx_to_chr; + unsigned int chridx_to_idx[MAILDIR_MAX_KEYWORDS]; + bool readonly; +}; + +struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox) +{ + struct maildir_keywords *mk; + + mk = maildir_keywords_init_readonly(&mbox->box); + mk->mbox = mbox; + return mk; +} + +struct maildir_keywords * +maildir_keywords_init_readonly(struct mailbox *box) +{ + struct maildir_keywords *mk; + const char *dir; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, &dir) <= 0) + i_unreached(); + + mk = i_new(struct maildir_keywords, 1); + mk->storage = box->storage; + mk->path = i_strconcat(dir, "/" MAILDIR_KEYWORDS_NAME, NULL); + mk->pool = pool_alloconly_create("maildir keywords", 512); + i_array_init(&mk->list, MAILDIR_MAX_KEYWORDS); + hash_table_create(&mk->hash, mk->pool, 0, strcase_hash, strcasecmp); + + mk->dotlock_settings.use_excl_lock = + box->storage->set->dotlock_use_excl; + mk->dotlock_settings.nfs_flush = + box->storage->set->mail_nfs_storage; + mk->dotlock_settings.timeout = + mail_storage_get_lock_timeout(box->storage, + KEYWORDS_LOCK_STALE_TIMEOUT + 2); + mk->dotlock_settings.stale_timeout = KEYWORDS_LOCK_STALE_TIMEOUT; + mk->dotlock_settings.temp_prefix = + mailbox_list_get_temp_prefix(box->list); + return mk; +} + +void maildir_keywords_deinit(struct maildir_keywords **_mk) +{ + struct maildir_keywords *mk = *_mk; + + *_mk = NULL; + hash_table_destroy(&mk->hash); + array_free(&mk->list); + pool_unref(&mk->pool); + i_free(mk->path); + i_free(mk); +} + +static void maildir_keywords_clear(struct maildir_keywords *mk) +{ + array_clear(&mk->list); + hash_table_clear(mk->hash, TRUE); + p_clear(mk->pool); +} + +static int maildir_keywords_sync(struct maildir_keywords *mk) +{ + struct istream *input; + struct stat st; + char *line, *p, *new_name; + const char **strp; + unsigned int idx; + int fd; + + /* Remember that we rely on uidlist file locking in here. That's why + we rely on stat()'s timestamp and don't bother handling ESTALE + errors. */ + + if (mk->storage->set->mail_nfs_storage) { + /* file is updated only by replacing it, no need to flush + attribute cache */ + nfs_flush_file_handle_cache(mk->path); + } + + if (nfs_safe_stat(mk->path, &st) < 0) { + if (errno == ENOENT) { + maildir_keywords_clear(mk); + mk->synced = TRUE; + return 0; + } + mailbox_set_critical(&mk->mbox->box, + "stat(%s) failed: %m", mk->path); + return -1; + } + + if (st.st_mtime == mk->synced_mtime) { + /* hasn't changed */ + mk->synced = TRUE; + return 0; + } + mk->synced_mtime = st.st_mtime; + + fd = open(mk->path, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) { + maildir_keywords_clear(mk); + mk->synced = TRUE; + return 0; + } + mailbox_set_critical(&mk->mbox->box, + "open(%s) failed: %m", mk->path); + return -1; + } + + maildir_keywords_clear(mk); + input = i_stream_create_fd(fd, 1024); + while ((line = i_stream_read_next_line(input)) != NULL) { + p = strchr(line, ' '); + if (p == NULL) { + /* note that when converting .customflags file this + case happens in the first line. */ + continue; + } + *p++ = '\0'; + + if (str_to_uint(line, &idx) < 0 || + idx >= MAILDIR_MAX_KEYWORDS || *p == '\0' || + hash_table_lookup(mk->hash, p) != NULL) { + /* shouldn't happen */ + continue; + } + + /* save it */ + new_name = p_strdup(mk->pool, p); + hash_table_insert(mk->hash, new_name, POINTER_CAST(idx + 1)); + + strp = array_idx_get_space(&mk->list, idx); + *strp = new_name; + } + i_stream_destroy(&input); + + if (close(fd) < 0) { + mailbox_set_critical(&mk->mbox->box, + "close(%s) failed: %m", mk->path); + return -1; + } + + mk->synced = TRUE; + return 0; +} + +static int +maildir_keywords_lookup(struct maildir_keywords *mk, const char *name, + unsigned int *chridx_r) +{ + void *value; + + value = hash_table_lookup(mk->hash, name); + if (value == NULL) { + if (mk->synced) + return 0; + + if (maildir_keywords_sync(mk) < 0) + return -1; + i_assert(mk->synced); + + value = hash_table_lookup(mk->hash, name); + if (value == NULL) + return 0; + } + + *chridx_r = POINTER_CAST_TO(value, unsigned int)-1; + return 1; +} + +static void +maildir_keywords_create(struct maildir_keywords *mk, const char *name, + unsigned int chridx) +{ + const char **strp; + char *new_name; + + i_assert(chridx < MAILDIR_MAX_KEYWORDS); + + new_name = p_strdup(mk->pool, name); + hash_table_insert(mk->hash, new_name, POINTER_CAST(chridx + 1)); + + strp = array_idx_get_space(&mk->list, chridx); + *strp = new_name; + + mk->changed = TRUE; +} + +static int +maildir_keywords_lookup_or_create(struct maildir_keywords *mk, const char *name, + unsigned int *chridx_r) +{ + const char *const *keywords; + unsigned int i, count; + int ret; + + if ((ret = maildir_keywords_lookup(mk, name, chridx_r)) != 0) + return ret; + + /* see if we are full */ + keywords = array_get(&mk->list, &count); + for (i = 0; i < count; i++) { + if (keywords[i] == NULL) + break; + } + + if (i == count && count >= MAILDIR_MAX_KEYWORDS) + return -1; + + if (!maildir_uidlist_is_locked(mk->mbox->uidlist)) + return -1; + + maildir_keywords_create(mk, name, i); + *chridx_r = i; + return 1; +} + +static const char * +maildir_keywords_idx(struct maildir_keywords *mk, unsigned int idx) +{ + const char *const *keywords; + unsigned int count; + + keywords = array_get(&mk->list, &count); + if (idx >= count) { + if (mk->synced) + return NULL; + + if (maildir_keywords_sync(mk) < 0) + return NULL; + i_assert(mk->synced); + + keywords = array_get(&mk->list, &count); + } + return idx >= count ? NULL : keywords[idx]; +} + +static int maildir_keywords_write_fd(struct maildir_keywords *mk, + const char *path, int fd) +{ + struct maildir_mailbox *mbox = mk->mbox; + struct mailbox *box = &mbox->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *const *keywords; + unsigned int i, count; + string_t *str; + struct stat st; + + str = t_str_new(256); + keywords = array_get(&mk->list, &count); + for (i = 0; i < count; i++) { + if (keywords[i] != NULL) + str_printfa(str, "%u %s\n", i, keywords[i]); + } + if (write_full(fd, str_data(str), str_len(str)) < 0) { + mailbox_set_critical(&mk->mbox->box, + "write_full(%s) failed: %m", path); + return -1; + } + + if (fstat(fd, &st) < 0) { + mailbox_set_critical(&mk->mbox->box, + "fstat(%s) failed: %m", path); + return -1; + } + + if (st.st_gid != perm->file_create_gid && + perm->file_create_gid != (gid_t)-1) { + if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(&mk->mbox->box, "%s", + eperm_error_get_chgrp("fchown", path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(&mk->mbox->box, + "fchown(%s) failed: %m", path); + } + } + } + + /* mtime must grow every time */ + if (st.st_mtime <= mk->synced_mtime) { + struct utimbuf ut; + + mk->synced_mtime = ioloop_time <= mk->synced_mtime ? + mk->synced_mtime + 1 : ioloop_time; + ut.actime = ioloop_time; + ut.modtime = mk->synced_mtime; + if (utime(path, &ut) < 0) { + mailbox_set_critical(&mk->mbox->box, + "utime(%s) failed: %m", path); + return -1; + } + } + + if (fsync(fd) < 0) { + mailbox_set_critical(&mk->mbox->box, + "fsync(%s) failed: %m", path); + return -1; + } + return 0; +} + +static int maildir_keywords_commit(struct maildir_keywords *mk) +{ + const struct mailbox_permissions *perm; + struct dotlock *dotlock; + const char *lock_path; + mode_t old_mask; + int i, fd; + + mk->synced = FALSE; + + if (!mk->changed || mk->mbox == NULL) + return 0; + + lock_path = t_strconcat(mk->path, ".lock", NULL); + i_unlink_if_exists(lock_path); + + perm = mailbox_get_permissions(&mk->mbox->box); + for (i = 0;; i++) { + /* we could just create the temp file directly, but doing it + this ways avoids potential problems with overwriting + contents in malicious symlinks */ + old_mask = umask(0777 & ~perm->file_create_mode); + fd = file_dotlock_open(&mk->dotlock_settings, mk->path, + DOTLOCK_CREATE_FLAG_NONBLOCK, &dotlock); + umask(old_mask); + if (fd != -1) + break; + + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + mailbox_set_critical(&mk->mbox->box, + "file_dotlock_open(%s) failed: %m", mk->path); + return -1; + } + /* the control dir doesn't exist. create it unless the whole + mailbox was just deleted. */ + if (!maildir_set_deleted(&mk->mbox->box)) + return -1; + } + + if (maildir_keywords_write_fd(mk, lock_path, fd) < 0) { + file_dotlock_delete(&dotlock); + return -1; + } + + if (file_dotlock_replace(&dotlock, 0) < 0) { + mailbox_set_critical(&mk->mbox->box, + "file_dotlock_replace(%s) failed: %m", mk->path); + return -1; + } + + mk->changed = FALSE; + return 0; +} + +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init(struct maildir_keywords *mk, + struct mail_index *index) +{ + struct maildir_keywords_sync_ctx *ctx; + + ctx = i_new(struct maildir_keywords_sync_ctx, 1); + ctx->mk = mk; + ctx->index = index; + ctx->keywords = mail_index_get_keywords(index); + i_array_init(&ctx->idx_to_chr, MAILDIR_MAX_KEYWORDS); + return ctx; +} + +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init_readonly(struct maildir_keywords *mk, + struct mail_index *index) +{ + struct maildir_keywords_sync_ctx *ctx; + + ctx = maildir_keywords_sync_init(mk, index); + ctx->readonly = TRUE; + return ctx; +} + +void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **_ctx) +{ + struct maildir_keywords_sync_ctx *ctx = *_ctx; + + *_ctx = NULL; + + T_BEGIN { + (void)maildir_keywords_commit(ctx->mk); + } T_END; + + array_free(&ctx->idx_to_chr); + i_free(ctx); +} + +unsigned int maildir_keywords_char_idx(struct maildir_keywords_sync_ctx *ctx, + char keyword) +{ + const char *name; + unsigned int chridx, idx; + + i_assert(keyword >= MAILDIR_KEYWORD_FIRST && + keyword <= MAILDIR_KEYWORD_LAST); + chridx = keyword - MAILDIR_KEYWORD_FIRST; + + if (ctx->chridx_to_idx[chridx] != 0) + return ctx->chridx_to_idx[chridx]; + + /* lookup / create */ + name = maildir_keywords_idx(ctx->mk, chridx); + if (name == NULL) { + /* name is lost. just generate one ourself. */ + name = t_strdup_printf("unknown-%u", chridx); + while (maildir_keywords_lookup(ctx->mk, name, &idx) > 0) { + /* don't create a duplicate name. + keep changing the name until it doesn't exist */ + name = t_strconcat(name, "?", NULL); + } + maildir_keywords_create(ctx->mk, name, chridx); + } + + mail_index_keyword_lookup_or_create(ctx->index, name, &idx); + ctx->chridx_to_idx[chridx] = idx; + return idx; +} + +char maildir_keywords_idx_char(struct maildir_keywords_sync_ctx *ctx, + unsigned int idx) +{ + const char *name; + char *chr_p; + unsigned int chridx; + int ret; + + chr_p = array_idx_get_space(&ctx->idx_to_chr, idx); + if (*chr_p != '\0') + return *chr_p; + + name = array_idx_elem(ctx->keywords, idx); + ret = !ctx->readonly ? + maildir_keywords_lookup_or_create(ctx->mk, name, &chridx) : + maildir_keywords_lookup(ctx->mk, name, &chridx); + if (ret <= 0) + return '\0'; + + *chr_p = chridx + MAILDIR_KEYWORD_FIRST; + return *chr_p; +} diff --git a/src/lib-storage/index/maildir/maildir-keywords.h b/src/lib-storage/index/maildir/maildir-keywords.h new file mode 100644 index 0000000..43e47d7 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-keywords.h @@ -0,0 +1,36 @@ +#ifndef MAILDIR_KEYWORDS_H +#define MAILDIR_KEYWORDS_H + +#define MAILDIR_KEYWORDS_NAME "dovecot-keywords" + +struct maildir_mailbox; +struct maildir_keywords; +struct maildir_keywords_sync_ctx; + +struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox); +void maildir_keywords_deinit(struct maildir_keywords **mk); + +/* Initialize a read-only maildir_keywords instance. Mailbox needs to contain + the dovecot-keywords file, but otherwise it doesn't have to be in maildir + format. */ +struct maildir_keywords * +maildir_keywords_init_readonly(struct mailbox *box); + +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init(struct maildir_keywords *mk, + struct mail_index *index); +/* Don't try to add any nonexistent keywords */ +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init_readonly(struct maildir_keywords *mk, + struct mail_index *index); +void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **ctx); + +/* Returns keyword index. */ +unsigned int maildir_keywords_char_idx(struct maildir_keywords_sync_ctx *ctx, + char keyword); +/* Returns keyword character for given index, or \0 if keyword couldn't be + added. */ +char maildir_keywords_idx_char(struct maildir_keywords_sync_ctx *ctx, + unsigned int idx); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-mail.c b/src/lib-storage/index/maildir/maildir-mail.c new file mode 100644 index 0000000..c3abbd3 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-mail.c @@ -0,0 +1,809 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "nfs-workarounds.h" +#include "index-mail.h" +#include "maildir-storage.h" +#include "maildir-filename.h" +#include "maildir-uidlist.h" +#include "maildir-sync.h" + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> + +struct maildir_open_context { + int fd; + char *path; +}; + +static int +do_open(struct maildir_mailbox *mbox, const char *path, + struct maildir_open_context *ctx) +{ + ctx->fd = nfs_safe_open(path, O_RDONLY); + if (ctx->fd != -1) { + ctx->path = i_strdup(path); + return 1; + } + if (errno == ENOENT) + return 0; + + if (errno == EACCES) { + mailbox_set_critical(&mbox->box, "%s", + mail_error_eacces_msg("open", path)); + } else { + mailbox_set_critical(&mbox->box, + "open(%s) failed: %m", path); + } + return -1; +} + +static int +do_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st) +{ + if (stat(path, st) == 0) + return 1; + if (errno == ENOENT) + return 0; + + if (errno == EACCES) { + mailbox_set_critical(&mbox->box, "%s", + mail_error_eacces_msg("stat", path)); + } else { + mailbox_set_critical(&mbox->box, "stat(%s) failed: %m", path); + } + return -1; +} + +static struct istream * +maildir_open_mail(struct maildir_mailbox *mbox, struct mail *mail, + bool *deleted_r) +{ + struct istream *input; + const char *path; + struct maildir_open_context ctx; + + *deleted_r = FALSE; + + if (!mail_stream_access_start(mail)) + return NULL; + + ctx.fd = -1; + ctx.path = NULL; + + mail->transaction->stats.open_lookup_count++; + if (!mail->saving) { + if (maildir_file_do(mbox, mail->uid, do_open, &ctx) < 0) + return NULL; + } else { + path = maildir_save_file_get_path(mail->transaction, mail->seq); + if (do_open(mbox, path, &ctx) <= 0) + return NULL; + } + + if (ctx.fd == -1) { + *deleted_r = TRUE; + return NULL; + } + + input = i_stream_create_fd_autoclose(&ctx.fd, 0); + if (input->stream_errno == EISDIR) { + i_stream_destroy(&input); + if (maildir_lose_unexpected_dir(&mbox->storage->storage, + ctx.path) >= 0) + *deleted_r = TRUE; + } else { + i_stream_set_name(input, ctx.path); + index_mail_set_read_buffer_size(mail, input); + } + i_free(ctx.path); + return input; +} + +static int maildir_mail_stat(struct mail *mail, struct stat *st_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box); + struct index_mail *imail = INDEX_MAIL(mail); + const char *path; + int fd, ret; + + if (!mail_metadata_access_start(mail)) + return -1; + + if (imail->data.access_part != 0 && + imail->data.stream == NULL) { + /* we're going to open the mail anyway */ + struct istream *input; + + (void)mail_get_stream(mail, NULL, NULL, &input); + } + + if (imail->data.stream != NULL && + (fd = i_stream_get_fd(imail->data.stream)) != -1) { + mail->transaction->stats.fstat_lookup_count++; + if (fstat(fd, st_r) < 0) { + mail_set_critical(mail, "fstat(%s) failed: %m", + i_stream_get_name(imail->data.stream)); + return -1; + } + } else if (!mail->saving) { + mail->transaction->stats.stat_lookup_count++; + ret = maildir_file_do(mbox, mail->uid, do_stat, st_r); + if (ret <= 0) { + if (ret == 0) + mail_set_expunged(mail); + return -1; + } + } else { + mail->transaction->stats.stat_lookup_count++; + path = maildir_save_file_get_path(mail->transaction, mail->seq); + if (stat(path, st_r) < 0) { + mail_set_critical(mail, "stat(%s) failed: %m", path); + return -1; + } + } + return 0; +} + +static int maildir_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct stat st; + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + + *date_r = data->received_date = st.st_mtime; + return 0; +} + +static int maildir_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct stat st; + + if (index_mail_get_save_date(_mail, date_r) > 0) + return 1; + + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + + *date_r = data->save_date = st.st_ctime; + return 1; +} + +static int +maildir_mail_get_fname(struct maildir_mailbox *mbox, struct mail *mail, + const char **fname_r) +{ + enum maildir_uidlist_rec_flag flags; + struct mail_index_view *view; + uint32_t seq; + bool exists; + int ret; + + ret = maildir_sync_lookup(mbox, mail->uid, &flags, fname_r); + if (ret != 0) + return ret; + + /* file exists in index file, but not in dovecot-uidlist anymore. */ + mail_set_expunged(mail); + + /* one reason this could happen is if we delayed opening + dovecot-uidlist and we're trying to open a mail that got recently + expunged. Let's test this theory first: */ + mail_index_refresh(mbox->box.index); + view = mail_index_view_open(mbox->box.index); + exists = mail_index_lookup_seq(view, mail->uid, &seq); + mail_index_view_close(&view); + + if (exists) { + /* the message still exists in index. this means there's some + kind of a desync, which doesn't get fixed if cur/ mtime is + the same as in index. fix this by forcing a resync. */ + (void)maildir_storage_sync_force(mbox, mail->uid); + } + return 0; +} + +static int maildir_get_pop3_state(struct index_mail *mail) +{ + struct mailbox *box = mail->mail.mail.box; + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const struct mail_cache_field *fields; + unsigned int i, count, psize_idx, vsize_idx; + enum mail_cache_decision_type dec, vsize_dec; + enum mail_fetch_field allowed_pop3_fields; + bool not_pop3_only = FALSE; + + if (mail->pop3_state_set) + return mail->pop3_state; + + /* if this mail itself has non-pop3 fields we know we're not + pop3-only */ + allowed_pop3_fields = MAIL_FETCH_FLAGS | MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY | MAIL_FETCH_STORAGE_ID | + MAIL_FETCH_VIRTUAL_SIZE; + + if (mail->data.wanted_headers != NULL || + (mail->data.wanted_fields & ENUM_NEGATE(allowed_pop3_fields)) != 0) + not_pop3_only = TRUE; + + /* get vsize decisions */ + psize_idx = ibox->cache_fields[MAIL_CACHE_PHYSICAL_FULL_SIZE].idx; + vsize_idx = ibox->cache_fields[MAIL_CACHE_VIRTUAL_FULL_SIZE].idx; + if (not_pop3_only) { + vsize_dec = mail_cache_field_get_decision(box->cache, + vsize_idx); + vsize_dec &= ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + } else { + /* also check if there are any non-[pv]size cached fields */ + vsize_dec = MAIL_CACHE_DECISION_NO; + fields = mail_cache_register_get_list(box->cache, + pool_datastack_create(), + &count); + for (i = 0; i < count; i++) { + dec = fields[i].decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + if (fields[i].idx == vsize_idx) + vsize_dec = dec; + else if (dec != MAIL_CACHE_DECISION_NO && + fields[i].idx != psize_idx) + not_pop3_only = TRUE; + } + } + + if (index_mail_get_vsize_extension(&mail->mail.mail) != NULL) { + /* having a vsize extension in index is the same as having + vsize's caching decision YES */ + vsize_dec = MAIL_CACHE_DECISION_YES; + } + + if (!not_pop3_only) { + /* either nothing is cached, or only vsize is cached. */ + mail->pop3_state = 1; + } else if (vsize_dec != MAIL_CACHE_DECISION_YES && + (box->flags & MAILBOX_FLAG_POP3_SESSION) == 0) { + /* if virtual size isn't cached permanently, + POP3 isn't being used */ + mail->pop3_state = -1; + } else { + /* possibly a mixed pop3/imap */ + mail->pop3_state = 0; + } + mail->pop3_state_set = TRUE; + return mail->pop3_state; +} + +static int maildir_quick_size_lookup(struct index_mail *mail, bool vsize, + uoff_t *size_r) +{ + struct mail *_mail = &mail->mail.mail; + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + enum maildir_uidlist_rec_ext_key key; + const char *path, *fname, *value; + + if (!_mail->saving) { + if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) + return -1; + } else { + if (maildir_save_file_get_size(_mail->transaction, _mail->seq, + vsize, size_r) == 0) + return 1; + + path = maildir_save_file_get_path(_mail->transaction, + _mail->seq); + fname = strrchr(path, '/'); + fname = fname != NULL ? fname + 1 : path; + } + + /* size can be included in filename */ + if (vsize || !mbox->storage->set->maildir_broken_filename_sizes) { + if (maildir_filename_get_size(fname, + vsize ? MAILDIR_EXTRA_VIRTUAL_SIZE : + MAILDIR_EXTRA_FILE_SIZE, size_r)) + return 1; + } + + /* size can be included in uidlist entry */ + if (!_mail->saving) { + key = vsize ? MAILDIR_UIDLIST_REC_EXT_VSIZE : + MAILDIR_UIDLIST_REC_EXT_PSIZE; + value = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + key); + if (value != NULL && str_to_uoff(value, size_r) == 0) + return 1; + } + return 0; +} + +static void +maildir_handle_size_caching(struct index_mail *mail, bool quick_check, + bool vsize) +{ + struct mailbox *box = mail->mail.mail.box; + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + enum mail_fetch_field field; + uoff_t size; + int pop3_state; + + field = vsize ? MAIL_FETCH_VIRTUAL_SIZE : MAIL_FETCH_PHYSICAL_SIZE; + if ((mail->data.dont_cache_fetch_fields & field) != 0) + return; + + if (quick_check && maildir_quick_size_lookup(mail, vsize, &size) > 0) { + /* already in filename / uidlist. don't add it anywhere, + including to the uidlist if it's already in filename. + do some extra checks here to catch potential cache bugs. */ + if (vsize && mail->data.virtual_size != size) { + mail_set_mail_cache_corrupted(&mail->mail.mail, + "Corrupted virtual size: " + "%"PRIuUOFF_T" != %"PRIuUOFF_T, + mail->data.virtual_size, size); + mail->data.virtual_size = size; + } else if (!vsize && mail->data.physical_size != size) { + mail_set_mail_cache_corrupted(&mail->mail.mail, + "Corrupted physical size: " + "%"PRIuUOFF_T" != %"PRIuUOFF_T, + mail->data.physical_size, size); + mail->data.physical_size = size; + } + mail->data.dont_cache_fetch_fields |= field; + return; + } + + /* 1 = pop3-only, 0 = mixed, -1 = no pop3 */ + pop3_state = maildir_get_pop3_state(mail); + if (pop3_state >= 0 && mail->mail.mail.uid != 0) { + /* if size is wanted permanently, store it to uidlist + so that in case cache file gets lost we can get it quickly */ + mail->data.dont_cache_fetch_fields |= field; + size = vsize ? mail->data.virtual_size : + mail->data.physical_size; + maildir_uidlist_set_ext(mbox->uidlist, mail->mail.mail.uid, + vsize ? MAILDIR_UIDLIST_REC_EXT_VSIZE : + MAILDIR_UIDLIST_REC_EXT_PSIZE, + dec2str(size)); + } +} + +static int maildir_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct message_size hdr_size, body_size; + struct istream *input; + uoff_t old_offset; + + if (maildir_uidlist_is_read(mbox->uidlist) || + (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) { + /* try to get the size from uidlist. this is especially useful + with pop3 to avoid unnecessarily opening the cache file. */ + if (maildir_quick_size_lookup(mail, TRUE, + &data->virtual_size) < 0) + return -1; + } + + if (data->virtual_size == UOFF_T_MAX) { + if (index_mail_get_cached_virtual_size(mail, size_r)) { + i_assert(mail->data.virtual_size != UOFF_T_MAX); + maildir_handle_size_caching(mail, TRUE, TRUE); + return 0; + } + if (maildir_quick_size_lookup(mail, TRUE, + &data->virtual_size) < 0) + return -1; + } + if (data->virtual_size != UOFF_T_MAX) { + data->dont_cache_fetch_fields |= MAIL_FETCH_VIRTUAL_SIZE; + *size_r = data->virtual_size; + return 0; + } + + /* fallback to reading the file */ + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0) + return -1; + i_stream_seek(data->stream, old_offset); + + maildir_handle_size_caching(mail, FALSE, TRUE); + *size_r = data->virtual_size; + return 0; +} + +static int maildir_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + struct index_mail_data *data = &mail->data; + struct stat st; + struct message_size hdr_size, body_size; + struct istream *input; + const char *path; + int ret; + + if (maildir_uidlist_is_read(mbox->uidlist) || + (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) { + /* try to get the size from uidlist (see virtual size above) */ + if (maildir_quick_size_lookup(mail, FALSE, + &data->physical_size) < 0) + return -1; + } + + if (data->physical_size == UOFF_T_MAX) { + if (index_mail_get_physical_size(_mail, size_r) == 0) { + i_assert(mail->data.physical_size != UOFF_T_MAX); + maildir_handle_size_caching(mail, TRUE, FALSE); + return 0; + } + if (maildir_quick_size_lookup(mail, FALSE, + &data->physical_size) < 0) + return -1; + } + if (data->physical_size != UOFF_T_MAX) { + data->dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE; + *size_r = data->physical_size; + return 0; + } + + if (mail->mail.v.istream_opened != NULL) { + /* we can't use stat(), because this may be a mail that some + plugin has changed (e.g. zlib). need to do it the slow + way. */ + if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0) + return -1; + st.st_size = hdr_size.physical_size + body_size.physical_size; + } else if (!_mail->saving) { + ret = maildir_file_do(mbox, _mail->uid, do_stat, &st); + if (ret <= 0) { + if (ret == 0) + mail_set_expunged(_mail); + return -1; + } + } else { + /* saved mail which hasn't been committed yet */ + path = maildir_save_file_get_path(_mail->transaction, + _mail->seq); + if (stat(path, &st) < 0) { + mail_set_critical(_mail, "stat(%s) failed: %m", path); + return -1; + } + } + + data->physical_size = st.st_size; + maildir_handle_size_caching(mail, FALSE, FALSE); + *size_r = st.st_size; + return 0; +} + +static int +maildir_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + const char *path, *fname = NULL, *end, *guid, *uidl, *order; + struct stat st; + + switch (field) { + case MAIL_FETCH_GUID: + /* use GUID from uidlist if it exists */ + i_assert(!_mail->saving); + + if (mail->data.guid != NULL) { + *value_r = mail->data.guid; + return 0; + } + + /* first make sure that we have a refreshed uidlist */ + if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) + return -1; + + guid = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_GUID); + if (guid != NULL) { + if (*guid != '\0') { + *value_r = mail->data.guid = + p_strdup(mail->mail.data_pool, guid); + return 0; + } + + mail_set_critical(_mail, + "Maildir: Corrupted dovecot-uidlist: " + "UID had empty GUID, clearing it"); + maildir_uidlist_unset_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_GUID); + } + + /* default to base filename: */ + if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID, + value_r) < 0) + return -1; + mail->data.guid = mail->data.filename; + return 0; + case MAIL_FETCH_STORAGE_ID: + if (mail->data.filename != NULL) { + *value_r = mail->data.filename; + return 0; + } + if (fname != NULL) { + /* we came here from MAIL_FETCH_GUID, + avoid a second lookup */ + } else if (!_mail->saving) { + if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) + return -1; + } else { + path = maildir_save_file_get_path(_mail->transaction, + _mail->seq); + fname = strrchr(path, '/'); + fname = fname != NULL ? fname + 1 : path; + } + end = strchr(fname, MAILDIR_INFO_SEP); + mail->data.filename = end == NULL ? + p_strdup(mail->mail.data_pool, fname) : + p_strdup_until(mail->mail.data_pool, fname, end); + *value_r = mail->data.filename; + return 0; + case MAIL_FETCH_UIDL_BACKEND: + uidl = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); + if (uidl == NULL) { + /* use the default */ + *value_r = ""; + } else if (*uidl == '\0') { + /* special optimization case: use the base file name */ + return maildir_mail_get_special(_mail, + MAIL_FETCH_STORAGE_ID, value_r); + } else { + *value_r = p_strdup(mail->mail.data_pool, uidl); + } + return 0; + case MAIL_FETCH_POP3_ORDER: + order = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_POP3_ORDER); + if (order == NULL) { + *value_r = ""; + } else { + *value_r = p_strdup(mail->mail.data_pool, order); + } + return 0; + case MAIL_FETCH_REFCOUNT: + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + *value_r = p_strdup_printf(mail->mail.data_pool, "%lu", + (unsigned long)st.st_nlink); + return 0; + case MAIL_FETCH_REFCOUNT_ID: + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + *value_r = p_strdup_printf(mail->mail.data_pool, "%llu", + (unsigned long long)st.st_ino); + return 0; + default: + return index_mail_get_special(_mail, field, value_r); + } +} + +static int +maildir_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + struct index_mail_data *data = &mail->data; + bool deleted; + + if (data->stream == NULL) { + data->stream = maildir_open_mail(mbox, _mail, &deleted); + if (data->stream == NULL) { + if (deleted) + mail_set_expunged(_mail); + return -1; + } + if (mail->mail.v.istream_opened != NULL) { + if (mail->mail.v.istream_opened(_mail, + &data->stream) < 0) { + i_stream_unref(&data->stream); + return -1; + } + } + } + + return index_mail_init_stream(mail, hdr_size, body_size, stream_r); +} + +static void maildir_update_pop3_uidl(struct mail *_mail, const char *uidl) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + const char *fname; + + if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID, + &fname) == 0 && + strcmp(uidl, fname) == 0) { + /* special case optimization: empty UIDL means the same + as base filename */ + uidl = ""; + } + + maildir_uidlist_set_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL, uidl); +} + +static void maildir_mail_remove_sizes_from_uidlist(struct mail *mail) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box); + + if (maildir_uidlist_lookup_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_VSIZE) != NULL) { + maildir_uidlist_unset_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_VSIZE); + } + if (maildir_uidlist_lookup_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_PSIZE) != NULL) { + maildir_uidlist_unset_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_PSIZE); + } +} + +struct maildir_size_fix_ctx { + uoff_t physical_size; + char wrong_key; +}; + +static int +do_fix_size(struct maildir_mailbox *mbox, const char *path, + struct maildir_size_fix_ctx *ctx) +{ + const char *fname, *newpath, *extra, *info, *dir; + struct stat st; + + fname = strrchr(path, '/'); + i_assert(fname != NULL); + dir = t_strdup_until(path, fname++); + + extra = strchr(fname, MAILDIR_EXTRA_SEP); + i_assert(extra != NULL); + info = strchr(fname, MAILDIR_INFO_SEP); + if (info == NULL) info = ""; + + if (ctx->physical_size == UOFF_T_MAX) { + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + mailbox_set_critical(&mbox->box, + "stat(%s) failed: %m", path); + return -1; + } + ctx->physical_size = st.st_size; + } + + newpath = t_strdup_printf("%s/%s,S=%"PRIuUOFF_T"%s", dir, + t_strdup_until(fname, extra), + ctx->physical_size, info); + + if (rename(path, newpath) == 0) { + mailbox_set_critical(&mbox->box, + "Maildir filename has wrong %c value, " + "renamed the file from %s to %s", + ctx->wrong_key, path, newpath); + return 1; + } + if (errno == ENOENT) + return 0; + + mailbox_set_critical(&mbox->box, "rename(%s, %s) failed: %m", + path, newpath); + return -1; +} + +static void +maildir_mail_remove_sizes_from_filename(struct mail *mail, + enum mail_fetch_field field) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box); + struct mail_private *pmail = (struct mail_private *)mail; + enum maildir_uidlist_rec_flag flags; + const char *fname; + uoff_t size; + struct maildir_size_fix_ctx ctx; + + if (mbox->storage->set->maildir_broken_filename_sizes) { + /* never try to fix sizes in maildir filenames */ + return; + } + + if (maildir_sync_lookup(mbox, mail->uid, &flags, &fname) <= 0) + return; + if (strchr(fname, MAILDIR_EXTRA_SEP) == NULL) + return; + + i_zero(&ctx); + ctx.physical_size = UOFF_T_MAX; + if (field == MAIL_FETCH_VIRTUAL_SIZE && + maildir_filename_get_size(fname, MAILDIR_EXTRA_VIRTUAL_SIZE, + &size)) { + ctx.wrong_key = 'W'; + } else if (field == MAIL_FETCH_PHYSICAL_SIZE && + maildir_filename_get_size(fname, MAILDIR_EXTRA_FILE_SIZE, + &size)) { + ctx.wrong_key = 'S'; + } else { + /* the broken size isn't in filename */ + return; + } + + if (pmail->v.istream_opened != NULL) { + /* the mail could be e.g. compressed. get the physical size + the slow way by actually reading the mail. */ + struct istream *input; + const struct stat *stp; + + if (mail_get_stream(mail, NULL, NULL, &input) < 0) + return; + if (i_stream_stat(input, TRUE, &stp) < 0) + return; + ctx.physical_size = stp->st_size; + } + + (void)maildir_file_do(mbox, mail->uid, do_fix_size, &ctx); +} + +static void maildir_mail_set_cache_corrupted(struct mail *_mail, + enum mail_fetch_field field, + const char *reason) +{ + if (field == MAIL_FETCH_PHYSICAL_SIZE || + field == MAIL_FETCH_VIRTUAL_SIZE) { + maildir_mail_remove_sizes_from_uidlist(_mail); + maildir_mail_remove_sizes_from_filename(_mail, field); + } + index_mail_set_cache_corrupted(_mail, field, reason); +} + +struct mail_vfuncs maildir_mail_vfuncs = { + index_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + maildir_mail_get_received_date, + maildir_mail_get_save_date, + maildir_mail_get_virtual_size, + maildir_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + maildir_mail_get_stream, + index_mail_get_binary_stream, + maildir_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + maildir_update_pop3_uidl, + index_mail_expunge, + maildir_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c new file mode 100644 index 0000000..5cf7d6a --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-save.c @@ -0,0 +1,1084 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "fdatasync-path.h" +#include "eacces-error.h" +#include "str.h" +#include "index-mail.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename.h" +#include "maildir-filename-flags.h" +#include "maildir-sync.h" +#include "mailbox-recent-flags.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <utime.h> +#include <sys/stat.h> + +#define MAILDIR_FILENAME_FLAG_MOVED 0x10000000 + +struct maildir_filename { + struct maildir_filename *next; + const char *tmp_name, *dest_basename; + const char *pop3_uidl, *guid; + + uoff_t size, vsize; + enum mail_flags flags; + unsigned int pop3_order; + bool preserve_filename:1; + ARRAY_TYPE(keyword_indexes) keywords; +}; + +struct maildir_save_context { + struct mail_save_context ctx; + pool_t pool; + + struct maildir_mailbox *mbox; + struct mail_index_transaction *trans; + struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; + struct maildir_keywords_sync_ctx *keywords_sync_ctx; + struct maildir_index_sync_context *sync_ctx; + struct mail *cur_dest_mail; + + const char *tmpdir, *newdir, *curdir; + struct maildir_filename *files, **files_tail, *file_last; + unsigned int files_count; + + struct istream *input; + int fd; + uint32_t first_seq, seq, last_nonrecent_uid; + + bool have_keywords:1; + bool have_preserved_filenames:1; + bool locked:1; + bool failed:1; + bool last_save_finished:1; + bool locked_uidlist_refresh:1; +}; + +#define MAILDIR_SAVECTX(s) container_of(s, struct maildir_save_context, ctx) + +static int maildir_file_move(struct maildir_save_context *ctx, + struct maildir_filename *mf, const char *destname, + bool newdir) +{ + struct mail_storage *storage = &ctx->mbox->storage->storage; + const char *tmp_path, *new_path; + + i_assert(*destname != '\0'); + i_assert(*mf->tmp_name != '\0'); + + /* if we have flags, we'll move it to cur/ directly, because files in + new/ directory can't have flags. alternative would be to write it + in new/ and set the flags dirty in index file, but in that case + external MUAs would see wrong flags. */ + tmp_path = t_strconcat(ctx->tmpdir, "/", mf->tmp_name, NULL); + new_path = newdir ? + t_strconcat(ctx->newdir, "/", destname, NULL) : + t_strconcat(ctx->curdir, "/", destname, NULL); + + /* maildir spec says we should use link() + unlink() here. however + since our filename is guaranteed to be unique, rename() works just + as well, except faster. even if the filename wasn't unique, the + problem could still happen if the file was already moved from + new/ to cur/, so link() doesn't really provide any safety anyway. + + Besides the small temporary performance benefits, this rename() is + almost required with OSX's HFS+ filesystem, since it implements + hard links in a pretty ugly way, which makes the performance crawl + when a lot of hard links are used. */ + if (rename(tmp_path, new_path) == 0) { + mf->flags |= MAILDIR_FILENAME_FLAG_MOVED; + return 0; + } else if (ENOQUOTA(errno)) { + mail_storage_set_error(storage, MAIL_ERROR_NOQUOTA, + MAIL_ERRSTR_NO_QUOTA); + return -1; + } else { + mailbox_set_critical(&ctx->mbox->box, + "rename(%s, %s) failed: %m", + tmp_path, new_path); + return -1; + } +} + +static struct mail_save_context * +maildir_save_transaction_init(struct mailbox_transaction_context *t) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(t->box); + struct maildir_save_context *ctx; + const char *path; + pool_t pool; + + pool = pool_alloconly_create("maildir_save_context", 4096); + ctx = p_new(pool, struct maildir_save_context, 1); + ctx->ctx.transaction = t; + ctx->pool = pool; + ctx->mbox = mbox; + ctx->trans = t->itrans; + ctx->files_tail = &ctx->files; + ctx->fd = -1; + + path = mailbox_get_path(&mbox->box); + ctx->tmpdir = p_strconcat(pool, path, "/tmp", NULL); + ctx->newdir = p_strconcat(pool, path, "/new", NULL); + ctx->curdir = p_strconcat(pool, path, "/cur", NULL); + + ctx->last_save_finished = TRUE; + return &ctx->ctx; +} + +struct maildir_filename * +maildir_save_add(struct mail_save_context *_ctx, const char *tmp_fname, + struct mail *src_mail) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct mail_save_data *mdata = &_ctx->data; + struct maildir_filename *mf; + struct istream *input; + + i_assert(*tmp_fname != '\0'); + + /* allow caller to specify recent flag only when uid is specified + (we're replicating, converting, etc.). */ + if (mdata->uid == 0) + mdata->flags |= MAIL_RECENT; + else if ((mdata->flags & MAIL_RECENT) == 0 && + ctx->last_nonrecent_uid < mdata->uid) + ctx->last_nonrecent_uid = mdata->uid; + + /* now, we want to be able to rollback the whole append session, + so we'll just store the name of this temp file and move it later + into new/ or cur/. */ + mf = p_new(ctx->pool, struct maildir_filename, 1); + mf->tmp_name = mf->dest_basename = p_strdup(ctx->pool, tmp_fname); + mf->flags = mdata->flags; + mf->size = UOFF_T_MAX; + mf->vsize = UOFF_T_MAX; + + ctx->file_last = mf; + i_assert(*ctx->files_tail == NULL); + *ctx->files_tail = mf; + ctx->files_tail = &mf->next; + ctx->files_count++; + + if (mdata->pop3_uidl != NULL) + mf->pop3_uidl = p_strdup(ctx->pool, mdata->pop3_uidl); + mf->pop3_order = mdata->pop3_order; + + /* insert into index */ + mail_index_append(ctx->trans, mdata->uid, &ctx->seq); + mail_index_update_flags(ctx->trans, ctx->seq, + MODIFY_REPLACE, + mdata->flags & ENUM_NEGATE(MAIL_RECENT)); + if (mdata->keywords != NULL) { + mail_index_update_keywords(ctx->trans, ctx->seq, + MODIFY_REPLACE, mdata->keywords); + } + if (mdata->min_modseq != 0) { + mail_index_update_modseq(ctx->trans, ctx->seq, + mdata->min_modseq); + } + + if (ctx->first_seq == 0) { + ctx->first_seq = ctx->seq; + i_assert(ctx->files->next == NULL); + } + + mail_set_seq_saving(_ctx->dest_mail, ctx->seq); + + if (ctx->input == NULL) { + /* copying with hardlinking. */ + i_assert(src_mail != NULL); + index_copy_cache_fields(_ctx, src_mail, ctx->seq); + ctx->cur_dest_mail = NULL; + } else { + input = index_mail_cache_parse_init(_ctx->dest_mail, + ctx->input); + i_stream_unref(&ctx->input); + ctx->input = input; + ctx->cur_dest_mail = _ctx->dest_mail; + } + return mf; +} + +void maildir_save_set_dest_basename(struct mail_save_context *_ctx, + struct maildir_filename *mf, + const char *basename) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + mf->preserve_filename = TRUE; + mf->dest_basename = p_strdup(ctx->pool, basename); + ctx->have_preserved_filenames = TRUE; +} + +void maildir_save_set_sizes(struct maildir_filename *mf, + uoff_t size, uoff_t vsize) +{ + mf->size = size; + mf->vsize = vsize; +} + +static bool +maildir_get_dest_filename(struct maildir_save_context *ctx, + struct maildir_filename *mf, + const char **fname_r) +{ + const char *basename = mf->dest_basename; + + if (mf->size != UOFF_T_MAX && !mf->preserve_filename) { + basename = t_strdup_printf("%s,%c=%"PRIuUOFF_T, basename, + MAILDIR_EXTRA_FILE_SIZE, mf->size); + } + + if (mf->vsize != UOFF_T_MAX && !mf->preserve_filename) { + basename = t_strdup_printf("%s,%c=%"PRIuUOFF_T, basename, + MAILDIR_EXTRA_VIRTUAL_SIZE, + mf->vsize); + } + + if (!array_is_created(&mf->keywords) || array_count(&mf->keywords) == 0) { + if ((mf->flags & MAIL_FLAGS_MASK) == MAIL_RECENT) { + *fname_r = basename; + return TRUE; + } + + *fname_r = maildir_filename_flags_set(basename, + mf->flags & MAIL_FLAGS_MASK); + return FALSE; + } + + i_assert(ctx->keywords_sync_ctx != NULL || + !array_is_created(&mf->keywords) || array_count(&mf->keywords) == 0); + *fname_r = maildir_filename_flags_kw_set(ctx->keywords_sync_ctx, + basename, + mf->flags & MAIL_FLAGS_MASK, + &mf->keywords); + return FALSE; +} + +static const char *maildir_mf_get_path(struct maildir_save_context *ctx, + struct maildir_filename *mf) +{ + const char *fname, *dir; + + if ((mf->flags & MAILDIR_FILENAME_FLAG_MOVED) == 0) { + /* file is still in tmp/ */ + return t_strdup_printf("%s/%s", ctx->tmpdir, mf->tmp_name); + } + + /* already moved to new/ or cur/ */ + dir = maildir_get_dest_filename(ctx, mf, &fname) ? + ctx->newdir : ctx->curdir; + return t_strdup_printf("%s/%s", dir, fname); +} + + +static struct maildir_filename * +maildir_save_get_mf(struct mailbox_transaction_context *t, uint32_t seq) +{ + struct maildir_save_context *save_ctx = MAILDIR_SAVECTX(t->save_ctx); + struct maildir_filename *mf; + + i_assert(seq >= save_ctx->first_seq); + + seq -= save_ctx->first_seq; + mf = save_ctx->files; + while (seq > 0) { + mf = mf->next; + i_assert(mf != NULL); + seq--; + } + return mf; +} + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r) +{ + struct maildir_filename *mf = maildir_save_get_mf(t, seq); + + *size_r = vsize ? mf->vsize : mf->size; + return *size_r == UOFF_T_MAX ? -1 : 0; +} + +const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, + uint32_t seq) +{ + struct maildir_save_context *save_ctx = MAILDIR_SAVECTX(t->save_ctx); + struct maildir_filename *mf = maildir_save_get_mf(t, seq); + + return maildir_mf_get_path(save_ctx, mf); +} + +static int maildir_create_tmp(struct maildir_mailbox *mbox, const char *dir, + const char **fname_r) +{ + struct mailbox *box = &mbox->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + size_t prefix_len; + const char *tmp_fname; + string_t *path; + mode_t old_mask; + int fd; + + path = t_str_new(256); + str_append(path, dir); + str_append_c(path, '/'); + prefix_len = str_len(path); + + do { + tmp_fname = maildir_filename_generate(); + str_truncate(path, prefix_len); + str_append(path, tmp_fname); + + /* the generated filename is unique. the only reason why it + might return an existing filename is if the time moved + backwards. so we'll use O_EXCL anyway, although it's mostly + useless. */ + old_mask = umask(0777 & ~perm->file_create_mode); + fd = open(str_c(path), + O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0777); + umask(old_mask); + } while (fd == -1 && errno == EEXIST); + + *fname_r = tmp_fname; + if (fd == -1) { + if (ENOQUOTA(errno)) { + mail_storage_set_error(box->storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + } else { + mailbox_set_critical(box, + "open(%s) failed: %m", str_c(path)); + } + } else if (perm->file_create_gid != (gid_t)-1) { + if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", + str_c(path), + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, + "fchown(%s) failed: %m", str_c(path)); + } + } + } + + return fd; +} + +struct mail_save_context * +maildir_save_alloc(struct mailbox_transaction_context *t) +{ + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx == NULL) + t->save_ctx = maildir_save_transaction_init(t); + return t->save_ctx; +} + +int maildir_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct maildir_filename *mf; + + /* new mail, new failure state */ + ctx->failed = FALSE; + + T_BEGIN { + /* create a new file in tmp/ directory */ + const char *fname; + + ctx->fd = maildir_create_tmp(ctx->mbox, ctx->tmpdir, &fname); + if (ctx->fd == -1) + ctx->failed = TRUE; + else { + if (ctx->mbox->storage->storage.set->mail_save_crlf) + ctx->input = i_stream_create_crlf(input); + else + ctx->input = i_stream_create_lf(input); + mf = maildir_save_add(_ctx, fname, NULL); + if (_ctx->data.guid != NULL) { + maildir_save_set_dest_basename(_ctx, mf, + _ctx->data.guid); + } + } + } T_END; + + if (!ctx->failed) { + _ctx->data.output = o_stream_create_fd_file(ctx->fd, 0, FALSE); + o_stream_set_name(_ctx->data.output, t_strdup_printf( + "%s/%s", ctx->tmpdir, ctx->file_last->tmp_name)); + o_stream_cork(_ctx->data.output); + ctx->last_save_finished = FALSE; + } + return ctx->failed ? -1 : 0; +} + +int maildir_save_continue(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + if (ctx->failed) + return -1; + + if (index_storage_save_continue(_ctx, ctx->input, + ctx->cur_dest_mail) < 0) { + ctx->failed = TRUE; + return -1; + } + return 0; +} + +static int maildir_save_finish_received_date(struct maildir_save_context *ctx, + const char *path) +{ + struct utimbuf buf; + struct stat st; + + if (ctx->ctx.data.received_date != (time_t)-1) { + /* set the received_date by modifying mtime */ + buf.actime = ioloop_time; + buf.modtime = ctx->ctx.data.received_date; + + if (utime(path, &buf) < 0) { + mail_set_critical(ctx->ctx.dest_mail, + "utime(%s) failed: %m", path); + return -1; + } + } else if (ctx->fd != -1) { + if (fstat(ctx->fd, &st) == 0) + ctx->ctx.data.received_date = st.st_mtime; + else { + mail_set_critical(ctx->ctx.dest_mail, + "fstat(%s) failed: %m", path); + return -1; + } + } else { + /* hardlinked */ + if (stat(path, &st) == 0) + ctx->ctx.data.received_date = st.st_mtime; + else { + mail_set_critical(ctx->ctx.dest_mail, + "stat(%s) failed: %m", path); + return -1; + } + } + return 0; +} + +static void maildir_save_remove_last_filename(struct maildir_save_context *ctx) +{ + struct maildir_filename **fm; + + index_storage_save_abort_last(&ctx->ctx, ctx->seq); + ctx->seq--; + + for (fm = &ctx->files; (*fm)->next != NULL; fm = &(*fm)->next) ; + i_assert(*fm == ctx->file_last); + *fm = NULL; + + ctx->files_tail = fm; + ctx->file_last = NULL; + ctx->files_count--; +} + +void maildir_save_finish_keywords(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + ARRAY_TYPE(keyword_indexes) keyword_idx; + t_array_init(&keyword_idx, 8); + mail_index_lookup_keywords(ctx->ctx.transaction->view, ctx->seq, + &keyword_idx); + + if (array_count(&keyword_idx) > 0) { + /* copy keywords */ + p_array_init(&ctx->file_last->keywords, ctx->pool, + array_count(&keyword_idx)); + array_copy(&ctx->file_last->keywords.arr, 0, &keyword_idx.arr, 0, + array_count(&keyword_idx)); + ctx->have_keywords = TRUE; + } +} + +static int maildir_save_finish_real(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct mail_storage *storage = &ctx->mbox->storage->storage; + const char *path, *output_errstr; + off_t real_size; + uoff_t size; + int output_errno; + + ctx->last_save_finished = TRUE; + if (ctx->failed && ctx->fd == -1) { + /* tmp file creation failed */ + return -1; + } + + path = t_strconcat(ctx->tmpdir, "/", ctx->file_last->tmp_name, NULL); + if (o_stream_finish(_ctx->data.output) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(_ctx->dest_mail, + "write(%s) failed: %s", path, + o_stream_get_error(_ctx->data.output)); + } + ctx->failed = TRUE; + } + + if (_ctx->data.save_date != (time_t)-1) { + /* we can't change ctime, but we can add the date to cache */ + struct index_mail *mail = INDEX_MAIL(_ctx->dest_mail); + uint32_t t = _ctx->data.save_date; + + index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t)); + } + + if (maildir_save_finish_received_date(ctx, path) < 0) + ctx->failed = TRUE; + + if (ctx->cur_dest_mail != NULL) { + index_mail_cache_parse_deinit(ctx->cur_dest_mail, + ctx->ctx.data.received_date, + !ctx->failed); + } + i_stream_unref(&ctx->input); + + /* remember the size in case we want to add it to filename */ + ctx->file_last->size = _ctx->data.output->offset; + if (ctx->cur_dest_mail == NULL || + mail_get_virtual_size(ctx->cur_dest_mail, + &ctx->file_last->vsize) < 0) + ctx->file_last->vsize = UOFF_T_MAX; + + output_errno = _ctx->data.output->stream_errno; + output_errstr = t_strdup(o_stream_get_error(_ctx->data.output)); + o_stream_destroy(&_ctx->data.output); + + maildir_save_finish_keywords(_ctx); + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER && + !ctx->failed) { + if (fsync(ctx->fd) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(_ctx->dest_mail, + "fsync(%s) failed: %m", path); + } + ctx->failed = TRUE; + } + } + real_size = lseek(ctx->fd, 0, SEEK_END); + if (real_size == (off_t)-1) { + mail_set_critical(_ctx->dest_mail, "lseek(%s) failed: %m", path); + } else if (real_size != (off_t)ctx->file_last->size && + (!maildir_filename_get_size(ctx->file_last->dest_basename, + MAILDIR_EXTRA_FILE_SIZE, &size) || + size != ctx->file_last->size)) { + /* e.g. zlib plugin was used. the "physical size" must be in + the maildir filename, since stat() will return wrong size */ + ctx->file_last->preserve_filename = FALSE; + /* preserve the GUID if needed */ + if (ctx->file_last->guid == NULL) + ctx->file_last->guid = ctx->file_last->dest_basename; + /* reset the base name as well, just in case there's a + ,W=vsize */ + ctx->file_last->dest_basename = ctx->file_last->tmp_name; + } + if (close(ctx->fd) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(_ctx->dest_mail, + "close(%s) failed: %m", path); + } + ctx->failed = TRUE; + } + ctx->fd = -1; + + if (ctx->failed) { + /* delete the tmp file */ + i_unlink_if_exists(path); + + if (ENOQUOTA(output_errno)) { + mail_storage_set_error(storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + } else if (output_errno != 0) { + mail_set_critical(_ctx->dest_mail, + "write(%s) failed: %s", path, output_errstr); + } + + maildir_save_remove_last_filename(ctx); + return -1; + } + + ctx->file_last = NULL; + return 0; +} + +int maildir_save_finish(struct mail_save_context *ctx) +{ + int ret; + + T_BEGIN { + ret = maildir_save_finish_real(ctx); + } T_END; + index_save_context_free(ctx); + return ret; +} + +void maildir_save_cancel(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)maildir_save_finish(_ctx); +} + +static void +maildir_save_unlink_files(struct maildir_save_context *ctx) +{ + struct maildir_filename *mf; + + for (mf = ctx->files; mf != NULL; mf = mf->next) T_BEGIN { + i_unlink(maildir_mf_get_path(ctx, mf)); + } T_END; + ctx->files = NULL; +} + +static int maildir_transaction_fsync_dirs(struct maildir_save_context *ctx, + bool new_changed, bool cur_changed) +{ + struct mail_storage *storage = &ctx->mbox->storage->storage; + + if (storage->set->parsed_fsync_mode == FSYNC_MODE_NEVER) + return 0; + + if (new_changed) { + if (fdatasync_path(ctx->newdir) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "fdatasync_path(%s) failed: %m", ctx->newdir); + return -1; + } + } + if (cur_changed) { + if (fdatasync_path(ctx->curdir) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "fdatasync_path(%s) failed: %m", ctx->curdir); + return -1; + } + } + return 0; +} + +static int seq_range_cmp(const struct seq_range *r1, const struct seq_range *r2) +{ + if (r1->seq1 < r2->seq2) + return -1; + else if (r1->seq1 > r2->seq2) + return 1; + else + return 0; +} + +static uint32_t +maildir_save_set_recent_flags(struct maildir_save_context *ctx) +{ + struct maildir_mailbox *mbox = ctx->mbox; + ARRAY_TYPE(seq_range) saved_sorted_uids; + const struct seq_range *uids; + unsigned int i, count; + uint32_t uid; + + count = array_count(&ctx->ctx.transaction->changes->saved_uids); + if (count == 0) + return 0; + + t_array_init(&saved_sorted_uids, count); + array_append_array(&saved_sorted_uids, + &ctx->ctx.transaction->changes->saved_uids); + array_sort(&saved_sorted_uids, seq_range_cmp); + + uids = array_get(&saved_sorted_uids, &count); + for (i = 0; i < count; i++) { + for (uid = uids[i].seq1; uid <= uids[i].seq2; uid++) + mailbox_recent_flags_set_uid(&mbox->box, uid); + } + return uids[count-1].seq2 + 1; +} + +static int +maildir_save_sync_index(struct maildir_save_context *ctx) +{ + struct mailbox_transaction_context *_t = ctx->ctx.transaction; + struct maildir_mailbox *mbox = ctx->mbox; + uint32_t first_uid, next_uid, first_recent_uid; + int ret; + + /* we'll need to keep the lock past the sync deinit */ + ret = maildir_uidlist_lock(mbox->uidlist); + i_assert(ret > 0); + + if (maildir_sync_header_refresh(mbox) < 0) + return -1; + if ((ret = maildir_uidlist_refresh_fast_init(mbox->uidlist)) < 0) + return -1; + + if (ret == 0) { + /* uidlist doesn't exist. make sure all existing message + are added to uidlist first. */ + (void)maildir_storage_sync_force(mbox, 0); + } + + if (maildir_sync_index_begin(mbox, NULL, &ctx->sync_ctx) < 0) + return -1; + ctx->keywords_sync_ctx = + maildir_sync_get_keywords_sync_ctx(ctx->sync_ctx); + + /* now that uidlist is locked, make sure all the existing mails + have been added to index. we don't really look into the + maildir, just add all the new mails listed in + dovecot-uidlist to index. */ + if (maildir_sync_index(ctx->sync_ctx, TRUE) < 0) + return -1; + + /* if messages were added to index, assign them UIDs */ + first_uid = maildir_uidlist_get_next_uid(mbox->uidlist); + i_assert(first_uid != 0); + mail_index_append_finish_uids(ctx->trans, first_uid, + &_t->changes->saved_uids); + i_assert(ctx->files_count == seq_range_count(&_t->changes->saved_uids)); + + /* these mails are all recent in our session */ + T_BEGIN { + next_uid = maildir_save_set_recent_flags(ctx); + } T_END; + + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) + first_recent_uid = next_uid; + else if (ctx->last_nonrecent_uid != 0) + first_recent_uid = ctx->last_nonrecent_uid + 1; + else + first_recent_uid = 0; + + if (first_recent_uid != 0) { + /* maildir_sync_index() dropped recent flags from + existing messages. we'll still need to drop recent + flags from these newly added messages. */ + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, + first_recent_uid), + &first_recent_uid, + sizeof(first_recent_uid), FALSE); + } + return 0; +} + +static void +maildir_save_rollback_index_changes(struct maildir_save_context *ctx) +{ + uint32_t seq; + + if (ctx->seq == 0) + return; + + for (seq = ctx->seq; seq >= ctx->first_seq; seq--) + mail_index_expunge(ctx->trans, seq); + + mail_cache_transaction_reset(ctx->ctx.transaction->cache_trans); +} + +static bool maildir_filename_has_conflict(struct maildir_filename *mf, + struct maildir_filename *prev_mf) +{ + if (strcmp(mf->dest_basename, prev_mf->dest_basename) == 0) { + /* already used this */ + return TRUE; + } + if (prev_mf->guid != NULL && + strcmp(mf->dest_basename, prev_mf->guid) == 0) { + /* previous filename also had a conflict */ + return TRUE; + } + return FALSE; +} + +static void +maildir_filename_check_conflicts(struct maildir_save_context *ctx, + struct maildir_filename *mf, + struct maildir_filename *prev_mf) +{ + uoff_t size, vsize; + + if (!ctx->locked_uidlist_refresh && ctx->locked) { + (void)maildir_uidlist_refresh(ctx->mbox->uidlist); + ctx->locked_uidlist_refresh = TRUE; + } + + if (!maildir_filename_get_size(mf->dest_basename, + MAILDIR_EXTRA_FILE_SIZE, &size)) + size = UOFF_T_MAX; + if (!maildir_filename_get_size(mf->dest_basename, + MAILDIR_EXTRA_VIRTUAL_SIZE, &vsize)) + vsize = UOFF_T_MAX; + + if (size != mf->size || vsize != mf->vsize || + !ctx->locked_uidlist_refresh || + (prev_mf != NULL && maildir_filename_has_conflict(mf, prev_mf)) || + maildir_uidlist_get_full_filename(ctx->mbox->uidlist, + mf->dest_basename) != NULL) { + /* a) dest_basename didn't contain the (correct) size/vsize. + they're required for good performance. + + b) file already exists. give it another name. + but preserve the size/vsize in the filename if possible */ + if (mf->size == UOFF_T_MAX) + mf->size = size; + if (mf->vsize == UOFF_T_MAX) + mf->vsize = size; + + mf->guid = mf->dest_basename; + mf->dest_basename = p_strdup(ctx->pool, + maildir_filename_generate()); + mf->preserve_filename = FALSE; + } +} + +static int +maildir_filename_dest_basename_cmp(struct maildir_filename *const *f1, + struct maildir_filename *const *f2) +{ + return strcmp((*f1)->dest_basename, (*f2)->dest_basename); +} + +static int +maildir_save_move_files_to_newcur(struct maildir_save_context *ctx) +{ + ARRAY(struct maildir_filename *) files; + struct maildir_filename *mf, *prev_mf; + bool newdir, new_changed, cur_changed; + int ret; + + /* put files into an array sorted by the destination filename. + this way we can easily check if there are duplicate destination + filenames within this transaction. */ + t_array_init(&files, ctx->files_count); + for (mf = ctx->files; mf != NULL; mf = mf->next) + array_push_back(&files, &mf); + array_sort(&files, maildir_filename_dest_basename_cmp); + + new_changed = cur_changed = FALSE; + prev_mf = NULL; + array_foreach_elem(&files, mf) { + T_BEGIN { + const char *dest; + + if (mf->preserve_filename) { + maildir_filename_check_conflicts(ctx, mf, + prev_mf); + } + + newdir = maildir_get_dest_filename(ctx, mf, &dest); + if (newdir) + new_changed = TRUE; + else + cur_changed = TRUE; + ret = maildir_file_move(ctx, mf, dest, newdir); + } T_END; + if (ret < 0) + return -1; + prev_mf = mf; + } + + if (ctx->locked) { + i_assert(ctx->sync_ctx != NULL); + maildir_sync_set_new_msgs_count(ctx->sync_ctx, + array_count(&files)); + } + return maildir_transaction_fsync_dirs(ctx, new_changed, cur_changed); +} + +static void maildir_save_sync_uidlist(struct maildir_save_context *ctx) +{ + struct mailbox_transaction_context *t = ctx->ctx.transaction; + struct maildir_filename *mf; + struct seq_range_iter iter; + enum maildir_uidlist_rec_flag flags; + struct maildir_uidlist_rec *rec; + unsigned int n = 0; + uint32_t uid; + bool newdir, bret; + int ret; + + seq_range_array_iter_init(&iter, &t->changes->saved_uids); + for (mf = ctx->files; mf != NULL; mf = mf->next) T_BEGIN { + const char *dest; + + bret = seq_range_array_iter_nth(&iter, n++, &uid); + i_assert(bret); + + newdir = maildir_get_dest_filename(ctx, mf, &dest); + flags = MAILDIR_UIDLIST_REC_FLAG_RECENT; + if (newdir) + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + ret = maildir_uidlist_sync_next_uid(ctx->uidlist_sync_ctx, + dest, uid, flags, &rec); + i_assert(ret > 0); + i_assert(rec != NULL); + if (mf->guid != NULL) { + maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec, + MAILDIR_UIDLIST_REC_EXT_GUID, mf->guid); + } + if (mf->pop3_uidl != NULL) { + maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec, + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL, + mf->pop3_uidl); + } + if (mf->pop3_order > 0) { + maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec, + MAILDIR_UIDLIST_REC_EXT_POP3_ORDER, + t_strdup_printf("%u", mf->pop3_order)); + } + } T_END; + i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); +} + +int maildir_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + enum maildir_uidlist_sync_flags sync_flags; + int ret; + + i_assert(_ctx->data.output == NULL); + i_assert(ctx->last_save_finished); + + if (ctx->files_count == 0) + return 0; + + sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL | + MAILDIR_UIDLIST_SYNC_NOREFRESH; + + if ((_t->flags & MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS) != 0) { + /* we want to assign UIDs, we must lock uidlist */ + } else if (ctx->have_keywords) { + /* keywords file updating relies on uidlist lock. */ + } else if (ctx->have_preserved_filenames) { + /* we're trying to use some potentially existing filenames. + we must lock to avoid race conditions where two sessions + try to save the same filename. */ + } else { + /* no requirement to lock uidlist. if we happen to get a lock, + assign uids. */ + sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK; + } + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags, + &ctx->uidlist_sync_ctx); + if (ret > 0) { + ctx->locked = TRUE; + if (maildir_save_sync_index(ctx) < 0) { + maildir_transaction_save_rollback(_ctx); + return -1; + } + } else if (ret == 0 && + (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0) { + ctx->locked = FALSE; + i_assert(ctx->uidlist_sync_ctx == NULL); + /* since we couldn't lock uidlist, we'll have to drop the + appends to index. */ + maildir_save_rollback_index_changes(ctx); + } else { + maildir_transaction_save_rollback(_ctx); + return -1; + } + + T_BEGIN { + ret = maildir_save_move_files_to_newcur(ctx); + } T_END; + if (ctx->locked) { + if (ret == 0) { + /* update dovecot-uidlist file. */ + maildir_save_sync_uidlist(ctx); + } + + if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, + ret == 0) < 0) + ret = -1; + } + + _t->changes->uid_validity = + maildir_uidlist_get_uid_validity(ctx->mbox->uidlist); + + if (ctx->locked) { + /* It doesn't matter if index syncing fails */ + ctx->keywords_sync_ctx = NULL; + if (ret < 0) + maildir_sync_index_rollback(&ctx->sync_ctx); + else + (void)maildir_sync_index_commit(&ctx->sync_ctx); + } + + if (ret < 0) { + ctx->keywords_sync_ctx = !ctx->have_keywords ? NULL : + maildir_keywords_sync_init(ctx->mbox->keywords, + ctx->mbox->box.index); + + /* unlink the files we just moved in an attempt to rollback + the transaction. uidlist is still locked, so at least other + Dovecot instances haven't yet seen the files. we need + to have the keywords sync context to be able to generate + the destination filenames if keywords were used. */ + maildir_save_unlink_files(ctx); + + if (ctx->keywords_sync_ctx != NULL) + maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx); + /* returning failure finishes the save_context */ + maildir_transaction_save_rollback(_ctx); + return -1; + } + return 0; +} + +void maildir_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result ATTR_UNUSED) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + _ctx->transaction = NULL; /* transaction is already freed */ + + if (ctx->locked) + maildir_uidlist_unlock(ctx->mbox->uidlist); + pool_unref(&ctx->pool); +} + +void maildir_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + i_assert(_ctx->data.output == NULL); + + if (!ctx->last_save_finished) + maildir_save_cancel(&ctx->ctx); + + /* delete files in tmp/ */ + maildir_save_unlink_files(ctx); + + if (ctx->uidlist_sync_ctx != NULL) + (void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE); + if (ctx->sync_ctx != NULL) + maildir_sync_index_rollback(&ctx->sync_ctx); + if (ctx->locked) + maildir_uidlist_unlock(ctx->mbox->uidlist); + + pool_unref(&ctx->pool); +} diff --git a/src/lib-storage/index/maildir/maildir-settings.c b/src/lib-storage/index/maildir/maildir-settings.c new file mode 100644 index 0000000..d986d18 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-settings.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "maildir-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct maildir_settings) + +static const struct setting_define maildir_setting_defines[] = { + DEF(BOOL, maildir_copy_with_hardlinks), + DEF(BOOL, maildir_very_dirty_syncs), + DEF(BOOL, maildir_broken_filename_sizes), + DEF(BOOL, maildir_empty_new), + + SETTING_DEFINE_LIST_END +}; + +static const struct maildir_settings maildir_default_settings = { + .maildir_copy_with_hardlinks = TRUE, + .maildir_very_dirty_syncs = FALSE, + .maildir_broken_filename_sizes = FALSE, + .maildir_empty_new = FALSE +}; + +static const struct setting_parser_info maildir_setting_parser_info = { + .module_name = "maildir", + .defines = maildir_setting_defines, + .defaults = &maildir_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct maildir_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info +}; + +const struct setting_parser_info *maildir_get_setting_parser_info(void) +{ + return &maildir_setting_parser_info; +} + diff --git a/src/lib-storage/index/maildir/maildir-settings.h b/src/lib-storage/index/maildir/maildir-settings.h new file mode 100644 index 0000000..cfcb732 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-settings.h @@ -0,0 +1,13 @@ +#ifndef MAILDIR_SETTINGS_H +#define MAILDIR_SETTINGS_H + +struct maildir_settings { + bool maildir_copy_with_hardlinks; + bool maildir_very_dirty_syncs; + bool maildir_broken_filename_sizes; + bool maildir_empty_new; +}; + +const struct setting_parser_info *maildir_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-storage.c b/src/lib-storage/index/maildir/maildir-storage.c new file mode 100644 index 0000000..f6537d8 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-storage.c @@ -0,0 +1,795 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "eacces-error.h" +#include "unlink-old-files.h" +#include "mailbox-uidvalidity.h" +#include "mailbox-list-private.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-sync.h" +#include "index-mail.h" + +#include <sys/stat.h> + +#define MAILDIR_LIST_CONTEXT(obj) \ + MODULE_CONTEXT(obj, maildir_mailbox_list_module) +#define MAILDIR_SUBFOLDER_FILENAME "maildirfolder" + +struct maildir_mailbox_list_context { + union mailbox_list_module_context module_ctx; + const struct maildir_settings *set; +}; + +extern struct mail_storage maildir_storage; +extern struct mailbox maildir_mailbox; + +static struct event_category event_category_maildir = { + .name = "maildir", + .parent = &event_category_storage, +}; + +static MODULE_CONTEXT_DEFINE_INIT(maildir_mailbox_list_module, + &mailbox_list_module_register); +static const char *maildir_subdirs[] = { "cur", "new", "tmp" }; + +static void maildir_mailbox_close(struct mailbox *box); + +static struct mail_storage *maildir_storage_alloc(void) +{ + struct maildir_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("maildir storage", 512+256); + storage = p_new(pool, struct maildir_storage, 1); + storage->storage = maildir_storage; + storage->storage.pool = pool; + return &storage->storage; +} + +static int +maildir_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, + const char **error_r ATTR_UNUSED) +{ + struct maildir_storage *storage = MAILDIR_STORAGE(_storage); + struct mailbox_list *list = ns->list; + const char *dir; + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + + storage->temp_prefix = p_strdup(_storage->pool, + mailbox_list_get_temp_prefix(list)); + + if (list->set.control_dir == NULL && list->set.inbox_path == NULL && + (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) { + /* put the temp files into tmp/ directory preferably */ + storage->temp_prefix = p_strconcat(_storage->pool, "tmp/", + storage->temp_prefix, NULL); + dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_DIR); + } else { + /* control dir should also be writable */ + dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + } + _storage->temp_path_prefix = p_strconcat(_storage->pool, dir, "/", + storage->temp_prefix, NULL); + return 0; +} + +static void maildir_storage_get_list_settings(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_MAILDIRPLUSPLUS; + if (set->subscription_fname == NULL) + set->subscription_fname = MAILDIR_SUBSCRIPTION_FILE_NAME; + + if (set->inbox_path == NULL && *set->maildir_name == '\0' && + (strcmp(set->layout, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) == 0 || + strcmp(set->layout, MAILBOX_LIST_NAME_FS) == 0) && + (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) { + /* Maildir++ INBOX is the Maildir base itself */ + set->inbox_path = set->root_dir; + } +} + +static const char * +maildir_storage_find_root_dir(const struct mail_namespace *ns) +{ + bool debug = ns->mail_set->mail_debug; + const char *home, *path; + + /* we'll need to figure out the maildir location ourself. + It's ~/Maildir unless we are chrooted. */ + if (ns->owner != NULL && + mail_user_get_home(ns->owner, &home) > 0) { + path = t_strconcat(home, "/Maildir", NULL); + if (access(path, R_OK|W_OK|X_OK) == 0) { + if (debug) + i_debug("maildir: root exists (%s)", path); + return path; + } + if (debug) + i_debug("maildir: access(%s, rwx): failed: %m", path); + } else { + if (debug) + i_debug("maildir: Home directory not set"); + if (access("/cur", R_OK|W_OK|X_OK) == 0) { + if (debug) + i_debug("maildir: /cur exists, assuming chroot"); + return "/"; + } + } + return NULL; +} + +static bool maildir_storage_autodetect(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + bool debug = ns->mail_set->mail_debug; + struct stat st; + const char *path, *root_dir; + + if (set->root_dir != NULL) + root_dir = set->root_dir; + else { + root_dir = maildir_storage_find_root_dir(ns); + if (root_dir == NULL) { + if (debug) + i_debug("maildir: couldn't find root dir"); + return FALSE; + } + } + + path = t_strconcat(root_dir, "/cur", NULL); + if (stat(path, &st) < 0) { + if (debug) + i_debug("maildir autodetect: stat(%s) failed: %m", path); + return FALSE; + } + + if (!S_ISDIR(st.st_mode)) { + if (debug) + i_debug("maildir autodetect: %s not a directory", path); + return FALSE; + } + + set->root_dir = root_dir; + maildir_storage_get_list_settings(ns, set); + return TRUE; +} + +static int +mkdir_verify(struct mailbox *box, const char *dir, bool verify) +{ + const struct mailbox_permissions *perm; + struct stat st; + + if (verify) { + if (stat(dir, &st) == 0) + return 0; + + if (errno != ENOENT) { + mailbox_set_critical(box, "stat(%s) failed: %m", dir); + return -1; + } + } + + perm = mailbox_get_permissions(box); + if (mkdir_parents_chgrp(dir, perm->dir_create_mode, + perm->file_create_gid, + perm->file_create_gid_origin) == 0) + return 0; + + if (errno == EEXIST) { + if (verify) + return 0; + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + } else if (errno == ENOENT) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + "Mailbox was deleted while it was being created"); + } else if (errno == EACCES) { + if (box->list->ns->type == MAIL_NAMESPACE_TYPE_SHARED) { + /* shared namespace, don't log permission errors */ + mail_storage_set_error(box->storage, MAIL_ERROR_PERM, + MAIL_ERRSTR_NO_PERMISSION); + return -1; + } + mailbox_set_critical(box, "%s", + mail_error_create_eacces_msg("mkdir", dir)); + } else { + mailbox_set_critical(box, "mkdir(%s) failed: %m", dir); + } + return -1; +} + +static int maildir_check_tmp(struct mail_storage *storage, const char *dir) +{ + unsigned int interval = storage->set->mail_temp_scan_interval; + const char *path; + struct stat st; + + /* if tmp/ directory exists, we need to clean it up once in a while */ + path = t_strconcat(dir, "/tmp", NULL); + if (stat(path, &st) < 0) { + if (errno == ENOENT || errno == ENAMETOOLONG) + return 0; + if (errno == EACCES) { + mail_storage_set_critical(storage, "%s", + mail_error_eacces_msg("stat", path)); + return -1; + } + mail_storage_set_critical(storage, "stat(%s) failed: %m", path); + return -1; + } + + if (interval == 0) { + /* disabled */ + } else if (st.st_atime > st.st_ctime + MAILDIR_TMP_DELETE_SECS) { + /* the directory should be empty. we won't do anything + until ctime changes. */ + } else if (st.st_atime < ioloop_time - (time_t)interval) { + /* time to scan */ + (void)unlink_old_files(path, "", + ioloop_time - MAILDIR_TMP_DELETE_SECS); + } + return 1; +} + +/* create or fix maildir, ignore if it already exists */ +static int create_maildir_subdirs(struct mailbox *box, bool verify) +{ + const char *path, *box_path; + unsigned int i; + enum mail_error error; + int ret = 0; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &box_path) < 0) + return -1; + + for (i = 0; i < N_ELEMENTS(maildir_subdirs); i++) { + path = t_strconcat(box_path, "/", maildir_subdirs[i], NULL); + if (mkdir_verify(box, path, verify) < 0) { + error = mailbox_get_last_mail_error(box); + if (error != MAIL_ERROR_EXISTS) + return -1; + /* try to create all of the directories in case one + of them doesn't exist */ + ret = -1; + } + } + return ret; +} + +static void maildir_lock_touch_timeout(struct maildir_mailbox *mbox) +{ + (void)maildir_uidlist_lock_touch(mbox->uidlist); +} + +static struct mailbox * +maildir_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct maildir_mailbox *mbox; + pool_t pool; + + pool = pool_alloconly_create("maildir mailbox", 1024*3); + mbox = p_new(pool, struct maildir_mailbox, 1); + mbox->box = maildir_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &maildir_mail_vfuncs; + mbox->maildir_list_index_ext_id = (uint32_t)-1; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = MAILDIR_STORAGE(storage); + return &mbox->box; +} + +static int maildir_mailbox_open_existing(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + mbox->uidlist = maildir_uidlist_init(mbox); + mbox->keywords = maildir_keywords_init(mbox); + + if ((box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0) { + if (maildir_uidlist_lock(mbox->uidlist) <= 0) { + maildir_mailbox_close(box); + return -1; + } + mbox->keep_lock_to = timeout_add(MAILDIR_LOCK_TOUCH_SECS * 1000, + maildir_lock_touch_timeout, + mbox); + } + + if (index_storage_mailbox_open(box, FALSE) < 0) { + maildir_mailbox_close(box); + return -1; + } + + mbox->maildir_ext_id = + mail_index_ext_register(mbox->box.index, "maildir", + sizeof(mbox->maildir_hdr), 0, 0); + return 0; +} + +static bool maildir_storage_is_readonly(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + if (index_storage_is_readonly(box)) + return TRUE; + + if (maildir_is_backend_readonly(mbox)) { + /* return read-only only if there are no private flags + (that are stored in index files) */ + if (mailbox_get_private_flags_mask(box) == 0) + return TRUE; + } + return FALSE; +} + +static int +maildir_mailbox_exists(struct mailbox *box, bool auto_boxes, + enum mailbox_existence *existence_r) +{ + if (auto_boxes && mailbox_is_autocreated(box)) { + *existence_r = MAILBOX_EXISTENCE_SELECT; + return 0; + } + + return index_storage_mailbox_exists_full(box, "cur", existence_r); +} + +static bool maildir_has_any_subdir(const char *box_path, + const char **error_path) +{ + const char *path; + unsigned int i; + struct stat st; + + for (i = 0; i < N_ELEMENTS(maildir_subdirs); i++) { + path = t_strconcat(box_path, "/", maildir_subdirs[i], NULL); + if (stat(path, &st) == 0) { + /* Return if there is any subdir */ + return TRUE; + } else if (errno == ENOENT || errno == ENAMETOOLONG) { + /* Some subdirs may not exist. */ + } else { + *error_path = path; + return FALSE; + } + } + + return FALSE; +} + +static bool maildir_is_selectable(const char *box_path, + const char *root_dir, + const char **error_path_r) +{ + struct stat st; + + *error_path_r = box_path; + if (strcmp(box_path, root_dir) == 0) + return maildir_has_any_subdir(box_path, error_path_r); + else + return stat(box_path, &st) == 0; +} + +static int maildir_mailbox_open(struct mailbox *box) +{ + const char *box_path = mailbox_get_path(box); + const char *root_dir; + int ret; + + /* begin by checking if tmp/ directory exists and if it should be + cleaned up. */ + ret = maildir_check_tmp(box->storage, box_path); + if (ret > 0) { + /* exists */ + return maildir_mailbox_open_existing(box); + } + if (ret < 0) + return -1; + + /* tmp/ directory doesn't exist. does the maildir? autocreate missing + dirs only with Maildir++ and imapdir layouts. */ + if (strcmp(box->list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) != 0 && + strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPDIR) != 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } + root_dir = mailbox_list_get_root_forced(box->list, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + + /* This code path is executed only for Maildir++ and imapdir layouts, + which don't support \Noselect mailboxes. If the mailbox root + directory exists, automatically create any missing cur/new/tmp + directories. Otherwise the mailbox would show up as selectable + in the mailbox list, but not actually be selectable. + + As a special case we don't do this when the mailbox root directory + is the same as the namespace root directory. This especially means + that we don't autocreate Maildir INBOX when ~/Maildir directory + exists. Instead, we return that mailbox doesn't exist, so the + caller goes to the INBOX autocreation code path similarly as with + other mailboxes. This is needed e.g. for welcome plugin to work. */ + const char *error_path; + if (maildir_is_selectable(box_path, root_dir, &error_path)) { + /* yes, we'll need to create the missing dirs */ + if (create_maildir_subdirs(box, TRUE) < 0) + return -1; + + return maildir_mailbox_open_existing(box); + } + + if (errno == ENOENT || errno == ENAMETOOLONG) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } else { + mailbox_set_critical(box, "stat(%s) failed: %m", error_path); + return -1; + } +} + +static int maildir_create_shared(struct mailbox *box) +{ + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *path; + mode_t old_mask; + int fd, ret; + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &path); + if (ret < 0) + return -1; + i_assert(ret > 0); + + old_mask = umask(0); + path = t_strconcat(path, "/dovecot-shared", NULL); + fd = open(path, O_WRONLY | O_CREAT, perm->file_create_mode); + umask(old_mask); + + if (fd == -1) { + mailbox_set_critical(box, "open(%s) failed: %m", path); + return -1; + } + + if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, + "fchown(%s) failed: %m", path); + } + } + i_close_fd(&fd); + return 0; +} + +static int +maildir_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + struct maildir_uidlist *uidlist; + bool locked = FALSE; + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + uidlist = mbox->uidlist; + + if (update->uid_validity != 0 || update->min_next_uid != 0 || + !guid_128_is_empty(update->mailbox_guid)) { + if (maildir_uidlist_lock(uidlist) <= 0) + return -1; + + locked = TRUE; + if (!guid_128_is_empty(update->mailbox_guid)) + maildir_uidlist_set_mailbox_guid(uidlist, update->mailbox_guid); + if (update->uid_validity != 0) + maildir_uidlist_set_uid_validity(uidlist, update->uid_validity); + if (update->min_next_uid != 0) { + maildir_uidlist_set_next_uid(uidlist, update->min_next_uid, + FALSE); + } + ret = maildir_uidlist_update(uidlist); + } + if (ret == 0) + ret = index_storage_mailbox_update(box, update); + if (locked) + maildir_uidlist_unlock(uidlist); + return ret; +} + +static int maildir_create_maildirfolder_file(struct mailbox *box) +{ + const struct mailbox_permissions *perm; + const char *path; + mode_t old_mask; + int fd; + + /* Maildir++ spec wants that maildirfolder named file is created for + all subfolders. Do this only with Maildir++ layout. */ + if (strcmp(box->list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) != 0) + return 0; + perm = mailbox_get_permissions(box); + + path = t_strconcat(mailbox_get_path(box), + "/"MAILDIR_SUBFOLDER_FILENAME, NULL); + old_mask = umask(0); + fd = open(path, O_CREAT | O_WRONLY, perm->file_create_mode); + umask(old_mask); + if (fd != -1) { + /* ok */ + } else if (errno == ENOENT) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + "Mailbox was deleted while it was being created"); + return -1; + } else { + mailbox_set_critical(box, "open(%s, O_CREAT) failed: %m", path); + return -1; + } + + if (perm->file_create_gid != (gid_t)-1) { + if (fchown(fd, (uid_t)-1, perm->file_create_gid) == 0) { + /* ok */ + } else if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, "fchown(%s) failed: %m", path); + } + } + i_close_fd(&fd); + return 0; +} + +static int +maildir_mailbox_create(struct mailbox *box, const struct mailbox_update *update, + bool directory) +{ + const char *root_dir, *shared_path; + /* allow physical location to exist when we rebuild list index, this + happens with LAYOUT=INDEX only. */ + bool verify = box->storage->rebuilding_list_index; + struct stat st; + int ret; + + if ((ret = index_storage_mailbox_create(box, directory)) <= 0) + return ret; + ret = 0; + /* the maildir is created now. finish the creation as best as we can */ + if (create_maildir_subdirs(box, verify) < 0) + ret = -1; + if (maildir_create_maildirfolder_file(box) < 0) + ret = -1; + /* if dovecot-shared exists in the root dir, copy it to newly + created mailboxes */ + root_dir = mailbox_list_get_root_forced(box->list, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + shared_path = t_strconcat(root_dir, "/dovecot-shared", NULL); + if (stat(shared_path, &st) == 0) { + if (maildir_create_shared(box) < 0) + ret = -1; + } + if (ret == 0 && update != NULL) { + if (maildir_mailbox_update(box, update) < 0) + ret = -1; + } + return ret; +} + +static int +maildir_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + + if ((items & MAILBOX_METADATA_GUID) != 0) { + if (maildir_uidlist_get_mailbox_guid(mbox->uidlist, + metadata_r->guid) < 0) + return -1; + } + return 0; +} + +static void maildir_mailbox_close(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + if (mbox->keep_lock_to != NULL) { + maildir_uidlist_unlock(mbox->uidlist); + timeout_remove(&mbox->keep_lock_to); + } + + if (mbox->flags_view != NULL) + mail_index_view_close(&mbox->flags_view); + if (mbox->keywords != NULL) + maildir_keywords_deinit(&mbox->keywords); + if (mbox->uidlist != NULL) + maildir_uidlist_deinit(&mbox->uidlist); + index_storage_mailbox_close(box); +} + +static void maildir_notify_changes(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + const char *box_path = mailbox_get_path(box); + + if (box->notify_callback == NULL) + mailbox_watch_remove_all(&mbox->box); + else { + mailbox_watch_add(&mbox->box, + t_strconcat(box_path, "/new", NULL)); + mailbox_watch_add(&mbox->box, + t_strconcat(box_path, "/cur", NULL)); + } +} + +static bool +maildir_is_internal_name(struct mailbox_list *list ATTR_UNUSED, + const char *name) +{ + return strcmp(name, "cur") == 0 || + strcmp(name, "new") == 0 || + strcmp(name, "tmp") == 0; +} + +static void maildir_storage_add_list(struct mail_storage *storage, + struct mailbox_list *list) +{ + struct maildir_mailbox_list_context *mlist; + + mlist = p_new(list->pool, struct maildir_mailbox_list_context, 1); + mlist->module_ctx.super = list->v; + mlist->set = mail_namespace_get_driver_settings(list->ns, storage); + + list->v.is_internal_name = maildir_is_internal_name; + MODULE_CONTEXT_SET(list, maildir_mailbox_list_module, mlist); +} + +uint32_t maildir_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"MAILDIR_UIDVALIDITY_FNAME, NULL); + return mailbox_uidvalidity_next(list, path); +} + +static enum mail_flags maildir_get_private_flags_mask(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + const char *path, *path2; + struct stat st; + + if (mbox->private_flags_mask_set) + return mbox->_private_flags_mask; + mbox->private_flags_mask_set = TRUE; + + path = mailbox_list_get_root_forced(box->list, MAILBOX_LIST_PATH_TYPE_MAILBOX); + if (box->list->set.index_pvt_dir != NULL) { + /* private index directory is set. we'll definitely have + private flags. */ + mbox->_private_flags_mask = MAIL_SEEN; + } else if (!mailbox_list_get_root_path(box->list, + MAILBOX_LIST_PATH_TYPE_INDEX, + &path2) || + strcmp(path, path2) == 0) { + /* no separate index directory. we can't have private flags, + so don't even bother checking if dovecot-shared exists */ + } else { + path = t_strconcat(mailbox_get_path(box), + "/dovecot-shared", NULL); + if (stat(path, &st) == 0) + mbox->_private_flags_mask = MAIL_SEEN; + } + return mbox->_private_flags_mask; +} + +bool maildir_is_backend_readonly(struct maildir_mailbox *mbox) +{ + if (!mbox->backend_readonly_set) { + const char *box_path = mailbox_get_path(&mbox->box); + + mbox->backend_readonly_set = TRUE; + if (access(t_strconcat(box_path, "/cur", NULL), W_OK) < 0 && + errno == EACCES) + mbox->backend_readonly = TRUE; + } + return mbox->backend_readonly; +} + +struct mail_storage maildir_storage = { + .name = MAILDIR_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + .event_category = &event_category_maildir, + + .v = { + maildir_get_setting_parser_info, + maildir_storage_alloc, + maildir_storage_create, + index_storage_destroy, + maildir_storage_add_list, + maildir_storage_get_list_settings, + maildir_storage_autodetect, + maildir_mailbox_alloc, + NULL, + mail_storage_list_index_rebuild, + } +}; + +struct mailbox maildir_mailbox = { + .v = { + maildir_storage_is_readonly, + index_storage_mailbox_enable, + maildir_mailbox_exists, + maildir_mailbox_open, + maildir_mailbox_close, + index_storage_mailbox_free, + maildir_mailbox_create, + maildir_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + maildir_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + maildir_list_index_has_changed, + maildir_list_index_update_sync, + maildir_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + maildir_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + maildir_get_private_flags_mask, + index_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + maildir_save_alloc, + maildir_save_begin, + maildir_save_continue, + maildir_save_finish, + maildir_save_cancel, + maildir_copy, + maildir_transaction_save_commit_pre, + maildir_transaction_save_commit_post, + maildir_transaction_save_rollback, + index_storage_is_inconsistent + } +}; diff --git a/src/lib-storage/index/maildir/maildir-storage.h b/src/lib-storage/index/maildir/maildir-storage.h new file mode 100644 index 0000000..da92866 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-storage.h @@ -0,0 +1,148 @@ +#ifndef MAILDIR_STORAGE_H +#define MAILDIR_STORAGE_H + +#include "maildir-settings.h" + +#define MAILDIR_STORAGE_NAME "maildir" +#define MAILDIR_SUBSCRIPTION_FILE_NAME "subscriptions" +#define MAILDIR_UIDVALIDITY_FNAME "dovecot-uidvalidity" + +/* "base,S=123:2," means: + <base> [<extra sep> <extra data> [..]] <info sep> 2 <flags sep> */ +#define MAILDIR_INFO_SEP ':' +#define MAILDIR_EXTRA_SEP ',' +#define MAILDIR_FLAGS_SEP ',' + +#define MAILDIR_INFO_SEP_S ":" +#define MAILDIR_EXTRA_SEP_S "," +#define MAILDIR_FLAGS_SEP_S "," + +/* ":2," is the standard flags separator */ +#define MAILDIR_FLAGS_FULL_SEP MAILDIR_INFO_SEP_S "2" MAILDIR_FLAGS_SEP_S + +#define MAILDIR_KEYWORD_FIRST 'a' +#define MAILDIR_KEYWORD_LAST 'z' +#define MAILDIR_MAX_KEYWORDS (MAILDIR_KEYWORD_LAST - MAILDIR_KEYWORD_FIRST + 1) + +/* Maildir++ extension: include file size in the filename to avoid stat() */ +#define MAILDIR_EXTRA_FILE_SIZE 'S' +/* Something (can't remember what anymore) could use 'W' in filename to avoid + calculating file's virtual size (added missing CRs). */ +#define MAILDIR_EXTRA_VIRTUAL_SIZE 'W' + +/* Delete files having ctime older than this from tmp/. 36h is standard. */ +#define MAILDIR_TMP_DELETE_SECS (36*60*60) + +/* How often to touch the uidlist lock file when it's locked. + This is done both when using KEEP_LOCKED flag and when syncing a large + maildir. */ +#define MAILDIR_LOCK_TOUCH_SECS 10 + +/* If an operation fails with ENOENT, we'll check if the mailbox is deleted + or if some directory is just missing. If it's missing, we'll create the + directories and try again this many times before failing. */ +#define MAILDIR_DELETE_RETRY_COUNT 3 + +#include "index-storage.h" + +struct timeval; +struct maildir_save_context; +struct maildir_copy_context; + +struct maildir_index_header { + uint32_t new_check_time, new_mtime, new_mtime_nsecs; + uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs; + uint32_t uidlist_mtime, uidlist_mtime_nsecs, uidlist_size; +}; + +struct maildir_list_index_record { + uint32_t new_mtime, cur_mtime; +}; + +struct maildir_storage { + struct mail_storage storage; + + const struct maildir_settings *set; + const char *temp_prefix; +}; + +struct maildir_mailbox { + struct mailbox box; + struct maildir_storage *storage; + struct mail_index_view *flags_view; + + struct timeout *keep_lock_to; + + /* Filled lazily by mailbox_get_private_flags_mask() */ + enum mail_flags _private_flags_mask; + + /* maildir sync: */ + struct maildir_uidlist *uidlist; + struct maildir_keywords *keywords; + + struct maildir_index_header maildir_hdr; + uint32_t maildir_ext_id; + uint32_t maildir_list_index_ext_id; + + bool synced:1; + bool syncing_commit:1; + bool private_flags_mask_set:1; + bool backend_readonly:1; + bool backend_readonly_set:1; + bool sync_uidlist_refreshed:1; +}; + +#define MAILDIR_STORAGE(s) container_of(s, struct maildir_storage, storage) +#define MAILDIR_MAILBOX(s) container_of(s, struct maildir_mailbox, box) + +extern struct mail_vfuncs maildir_mail_vfuncs; + +/* Return -1 = error, 0 = file not found, 1 = ok */ +typedef int maildir_file_do_func(struct maildir_mailbox *mbox, + const char *path, void *context); + +int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid, + maildir_file_do_func *callback, void *context); +#define maildir_file_do(mbox, seq, callback, context) \ + maildir_file_do(mbox, seq - \ + CALLBACK_TYPECHECK(callback, int (*)( \ + struct maildir_mailbox *, const char *, typeof(context))), \ + (maildir_file_do_func *)callback, context) + +bool maildir_set_deleted(struct mailbox *box); +uint32_t maildir_get_uidvalidity_next(struct mailbox_list *list); +int maildir_lose_unexpected_dir(struct mail_storage *storage, const char *path); +bool maildir_is_backend_readonly(struct maildir_mailbox *mbox); + +struct mail_save_context * +maildir_save_alloc(struct mailbox_transaction_context *_t); +int maildir_save_begin(struct mail_save_context *ctx, struct istream *input); +int maildir_save_continue(struct mail_save_context *ctx); +void maildir_save_finish_keywords(struct mail_save_context *ctx); +int maildir_save_finish(struct mail_save_context *ctx); +void maildir_save_cancel(struct mail_save_context *ctx); + +struct maildir_filename * +maildir_save_add(struct mail_save_context *_ctx, const char *tmp_fname, + struct mail *src_mail) ATTR_NULL(3); +void maildir_save_set_dest_basename(struct mail_save_context *ctx, + struct maildir_filename *mf, + const char *basename); +void maildir_save_set_sizes(struct maildir_filename *mf, + uoff_t size, uoff_t vsize); + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r); +const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, + uint32_t seq); + +int maildir_transaction_save_commit_pre(struct mail_save_context *ctx); +void maildir_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void maildir_transaction_save_rollback(struct mail_save_context *ctx); + +int maildir_copy(struct mail_save_context *ctx, struct mail *mail); +int maildir_transaction_copy_commit(struct maildir_copy_context *ctx); +void maildir_transaction_copy_rollback(struct maildir_copy_context *ctx); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-sync-index.c b/src/lib-storage/index/maildir/maildir-sync-index.c new file mode 100644 index 0000000..da0e40c --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-sync-index.c @@ -0,0 +1,810 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "maildir-storage.h" +#include "index-sync-changes.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename-flags.h" +#include "maildir-sync.h" +#include "mailbox-recent-flags.h" + +#include <stdio.h> +#include <unistd.h> + +struct maildir_index_sync_context { + struct maildir_mailbox *mbox; + struct maildir_sync_context *maildir_sync_ctx; + + struct mail_index_view *view; + struct mail_index_sync_ctx *sync_ctx; + struct maildir_keywords_sync_ctx *keywords_sync_ctx; + struct mail_index_transaction *trans; + + struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; + struct index_sync_changes_context *sync_changes; + enum mail_flags flags; + ARRAY_TYPE(keyword_indexes) keywords, idx_keywords; + + uint32_t uid; + bool update_maildir_hdr_cur; + + time_t start_time; + unsigned int flag_change_count, expunge_count, new_msgs_count; +}; + +struct maildir_keywords_sync_ctx * +maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx) +{ + return ctx->keywords_sync_ctx; +} + +void maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx, + unsigned int count) +{ + ctx->new_msgs_count = count; +} + +static bool +maildir_expunge_is_valid_guid(struct maildir_index_sync_context *ctx, + uint32_t uid, const char *filename, + guid_128_t expunged_guid_128) +{ + guid_128_t guid_128; + const char *guid; + + if (guid_128_is_empty(expunged_guid_128)) { + /* no GUID associated with expunge */ + return TRUE; + } + + T_BEGIN { + guid = maildir_uidlist_lookup_ext(ctx->mbox->uidlist, uid, + MAILDIR_UIDLIST_REC_EXT_GUID); + if (guid == NULL) + guid = t_strcut(filename, *MAILDIR_INFO_SEP_S); + mail_generate_guid_128_hash(guid, guid_128); + } T_END; + + if (memcmp(guid_128, expunged_guid_128, sizeof(guid_128)) == 0) + return TRUE; + + mailbox_set_critical(&ctx->mbox->box, + "Expunged GUID mismatch for UID %u: %s vs %s", + ctx->uid, guid_128_to_string(guid_128), + guid_128_to_string(expunged_guid_128)); + return FALSE; +} + +static int maildir_expunge(struct maildir_mailbox *mbox, const char *path, + struct maildir_index_sync_context *ctx) +{ + struct mailbox *box = &mbox->box; + + ctx->expunge_count++; + + if (unlink(path) == 0) { + mailbox_sync_notify(box, ctx->uid, MAILBOX_SYNC_TYPE_EXPUNGE); + return 1; + } + if (errno == ENOENT) + return 0; + if (UNLINK_EISDIR(errno)) + return maildir_lose_unexpected_dir(box->storage, path); + + mailbox_set_critical(&mbox->box, "unlink(%s) failed: %m", path); + return -1; +} + +static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path, + struct maildir_index_sync_context *ctx) +{ + struct mailbox *box = &mbox->box; + struct stat st; + const char *dir, *fname, *newfname, *newpath; + enum mail_index_sync_type sync_type; + uint8_t flags8; + + ctx->flag_change_count++; + + fname = strrchr(path, '/'); + i_assert(fname != NULL); + fname++; + dir = t_strdup_until(path, fname); + + i_assert(*fname != '\0'); + + /* get the current flags and keywords */ + maildir_filename_flags_get(ctx->keywords_sync_ctx, + fname, &ctx->flags, &ctx->keywords); + + /* apply changes */ + flags8 = ctx->flags; + index_sync_changes_apply(ctx->sync_changes, NULL, + &flags8, &ctx->keywords, &sync_type); + ctx->flags = flags8; + + /* and try renaming with the new name */ + newfname = maildir_filename_flags_kw_set(ctx->keywords_sync_ctx, fname, + ctx->flags, &ctx->keywords); + newpath = t_strconcat(dir, newfname, NULL); + if (strcmp(path, newpath) == 0) { + /* just make sure that the file still exists. avoid rename() + here because it's slow on HFS. */ + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return -1; + } + } else { + if (rename(path, newpath) < 0) { + if (errno == ENOENT) + return 0; + if (!ENOSPACE(errno) && errno != EACCES) { + mailbox_set_critical(box, + "rename(%s, %s) failed: %m", + path, newpath); + } + return -1; + } + } + mailbox_sync_notify(box, ctx->uid, index_sync_type_convert(sync_type)); + return 1; +} + +static int maildir_handle_uid_insertion(struct maildir_index_sync_context *ctx, + enum maildir_uidlist_rec_flag uflags, + const char *filename, uint32_t uid) +{ + int ret; + + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + return 0; + } + + /* most likely a race condition: we read the maildir, then someone else + expunged messages and committed changes to index. so, this message + shouldn't actually exist. */ + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) == 0) { + /* mark it racy and check in next sync */ + ctx->mbox->maildir_hdr.cur_check_time = 0; + maildir_sync_set_racing(ctx->maildir_sync_ctx); + maildir_uidlist_add_flags(ctx->mbox->uidlist, filename, + MAILDIR_UIDLIST_REC_FLAG_RACING); + return 0; + } + + if (ctx->uidlist_sync_ctx == NULL) { + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, + MAILDIR_UIDLIST_SYNC_PARTIAL | + MAILDIR_UIDLIST_SYNC_KEEP_STATE, + &ctx->uidlist_sync_ctx); + if (ret <= 0) + return -1; + } + + uflags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + maildir_uidlist_sync_remove(ctx->uidlist_sync_ctx, filename); + ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx, + filename, uflags); + i_assert(ret > 0); + + /* give the new UID to it immediately */ + maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx); + + i_warning("Maildir %s: Expunged message reappeared, giving a new UID " + "(old uid=%u, file=%s)%s", mailbox_get_path(&ctx->mbox->box), + uid, filename, !str_begins(filename, "msg.") ? "" : + " (Your MDA is saving MH files into Maildir?)"); + return 0; +} + +int maildir_sync_index_begin(struct maildir_mailbox *mbox, + struct maildir_sync_context *maildir_sync_ctx, + struct maildir_index_sync_context **ctx_r) +{ + struct mailbox *_box = &mbox->box; + struct maildir_index_sync_context *ctx; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + enum mail_index_sync_flags sync_flags; + + sync_flags = index_storage_get_sync_flags(&mbox->box); + /* don't drop recent messages if we're saving messages */ + if (maildir_sync_ctx == NULL) + sync_flags &= ENUM_NEGATE(MAIL_INDEX_SYNC_FLAG_DROP_RECENT); + + if (index_storage_expunged_sync_begin(_box, &sync_ctx, &view, + &trans, sync_flags) < 0) + return -1; + + ctx = i_new(struct maildir_index_sync_context, 1); + ctx->mbox = mbox; + ctx->maildir_sync_ctx = maildir_sync_ctx; + ctx->sync_ctx = sync_ctx; + ctx->view = view; + ctx->trans = trans; + ctx->keywords_sync_ctx = + maildir_keywords_sync_init(mbox->keywords, _box->index); + ctx->sync_changes = + index_sync_changes_init(ctx->sync_ctx, ctx->view, ctx->trans, + maildir_is_backend_readonly(mbox)); + ctx->start_time = time(NULL); + + *ctx_r = ctx; + return 0; +} + +static bool +maildir_index_header_has_changed(const struct maildir_index_header *old_hdr, + const struct maildir_index_header *new_hdr) +{ +#define DIR_DELAYED_REFRESH(hdr, name) \ + ((hdr)->name ## _check_time <= \ + (hdr)->name ## _mtime + MAILDIR_SYNC_SECS) + + if (old_hdr->new_mtime != new_hdr->new_mtime || + old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs || + old_hdr->cur_mtime != new_hdr->cur_mtime || + old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs || + old_hdr->uidlist_mtime != new_hdr->uidlist_mtime || + old_hdr->uidlist_mtime_nsecs != new_hdr->uidlist_mtime_nsecs || + old_hdr->uidlist_size != new_hdr->uidlist_size) + return TRUE; + + return DIR_DELAYED_REFRESH(old_hdr, new) != + DIR_DELAYED_REFRESH(new_hdr, new) || + DIR_DELAYED_REFRESH(old_hdr, cur) != + DIR_DELAYED_REFRESH(new_hdr, cur); +} + +static void +maildir_sync_index_update_ext_header(struct maildir_index_sync_context *ctx) +{ + struct maildir_mailbox *mbox = ctx->mbox; + const char *cur_path; + const void *data; + size_t data_size; + struct stat st; + + cur_path = t_strconcat(mailbox_get_path(&mbox->box), "/cur", NULL); + if (ctx->update_maildir_hdr_cur && stat(cur_path, &st) == 0) { + if ((time_t)mbox->maildir_hdr.cur_check_time < st.st_mtime) + mbox->maildir_hdr.cur_check_time = st.st_mtime; + mbox->maildir_hdr.cur_mtime = st.st_mtime; + mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st); + } + + mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id, + &data, &data_size); + if (data_size != sizeof(mbox->maildir_hdr) || + maildir_index_header_has_changed(data, &mbox->maildir_hdr)) { + mail_index_update_header_ext(ctx->trans, mbox->maildir_ext_id, + 0, &mbox->maildir_hdr, + sizeof(mbox->maildir_hdr)); + } +} + +static int maildir_sync_index_finish(struct maildir_index_sync_context *ctx, + bool success) +{ + struct maildir_mailbox *mbox = ctx->mbox; + unsigned int time_diff; + int ret = success ? 0 : -1; + + time_diff = time(NULL) - ctx->start_time; + if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) { + i_warning("Maildir %s: Synchronization took %u seconds " + "(%u new msgs, %u flag change attempts, " + "%u expunge attempts)", + mailbox_get_path(&ctx->mbox->box), time_diff, + ctx->new_msgs_count, ctx->flag_change_count, + ctx->expunge_count); + mail_index_sync_no_warning(ctx->sync_ctx); + } + + if (ret < 0) + mail_index_sync_rollback(&ctx->sync_ctx); + else { + maildir_sync_index_update_ext_header(ctx); + + /* Set syncing_commit=TRUE so that if any sync callbacks try + to access mails which got lost (eg. expunge callback trying + to open the file which was just unlinked) we don't try to + start a second index sync and crash. */ + mbox->syncing_commit = TRUE; + if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } + mbox->syncing_commit = FALSE; + } + + index_storage_expunging_deinit(&mbox->box); + maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx); + index_sync_changes_deinit(&ctx->sync_changes); + i_free(ctx); + return ret; +} + +int maildir_sync_index_commit(struct maildir_index_sync_context **_ctx) +{ + struct maildir_index_sync_context *ctx = *_ctx; + + *_ctx = NULL; + return maildir_sync_index_finish(ctx, TRUE); +} + +void maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx) +{ + struct maildir_index_sync_context *ctx = *_ctx; + + *_ctx = NULL; + (void)maildir_sync_index_finish(ctx, FALSE); +} + +static int uint_cmp(const unsigned int *i1, const unsigned int *i2) +{ + if (*i1 < *i2) + return -1; + else if (*i1 > *i2) + return 1; + else + return 0; +} + +static void +maildir_sync_mail_keywords(struct maildir_index_sync_context *ctx, uint32_t seq) +{ + struct mailbox *box = &ctx->mbox->box; + struct mail_keywords *kw; + unsigned int i, j, old_count, new_count; + const unsigned int *old_indexes, *new_indexes; + bool have_indexonly_keywords; + int diff; + + mail_index_lookup_keywords(ctx->view, seq, &ctx->idx_keywords); + if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords)) { + /* no changes - we should get here usually */ + return; + } + + /* sort the keywords */ + array_sort(&ctx->idx_keywords, uint_cmp); + array_sort(&ctx->keywords, uint_cmp); + + /* drop keywords that are in index-only. we don't want to touch them. */ + old_indexes = array_get(&ctx->idx_keywords, &old_count); + have_indexonly_keywords = FALSE; + for (i = old_count; i > 0; i--) { + if (maildir_keywords_idx_char(ctx->keywords_sync_ctx, + old_indexes[i-1]) == '\0') { + have_indexonly_keywords = TRUE; + array_delete(&ctx->idx_keywords, i-1, 1); + } + } + + if (!have_indexonly_keywords) { + /* no index-only keywords found, so something changed. + just replace them all. */ + kw = mail_index_keywords_create_from_indexes(box->index, + &ctx->keywords); + mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, kw); + mail_index_keywords_unref(&kw); + return; + } + + /* check again if non-index-only keywords changed */ + if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords)) + return; + + /* we can't reset all the keywords or we'd drop indexonly keywords too. + so first remove the unwanted keywords and then add back the wanted + ones. we can get these lists easily by removing common elements + from old and new keywords. */ + new_indexes = array_get(&ctx->keywords, &new_count); + for (i = j = 0; i < old_count && j < new_count; ) { + diff = (int)old_indexes[i] - (int)new_indexes[j]; + if (diff == 0) { + array_delete(&ctx->keywords, j, 1); + array_delete(&ctx->idx_keywords, i, 1); + old_indexes = array_get(&ctx->idx_keywords, &old_count); + new_indexes = array_get(&ctx->keywords, &new_count); + } else if (diff < 0) { + i++; + } else { + j++; + } + } + + if (array_count(&ctx->idx_keywords) > 0) { + kw = mail_index_keywords_create_from_indexes(box->index, + &ctx->idx_keywords); + mail_index_update_keywords(ctx->trans, seq, MODIFY_REMOVE, kw); + mail_index_keywords_unref(&kw); + } + + if (array_count(&ctx->keywords) > 0) { + kw = mail_index_keywords_create_from_indexes(box->index, + &ctx->keywords); + mail_index_update_keywords(ctx->trans, seq, MODIFY_ADD, kw); + mail_index_keywords_unref(&kw); + } +} + +int maildir_sync_index(struct maildir_index_sync_context *ctx, + bool partial) +{ + struct maildir_mailbox *mbox = ctx->mbox; + struct mail_index_view *view = ctx->view; + struct mail_index_view *view2; + struct maildir_uidlist_iter_ctx *iter; + struct mail_index_transaction *trans = ctx->trans; + const struct mail_index_header *hdr; + struct mail_index_header empty_hdr; + const struct mail_index_record *rec; + uint32_t seq, seq2, uid, prev_uid; + enum maildir_uidlist_rec_flag uflags; + const char *filename; + uint32_t uid_validity, next_uid, hdr_next_uid, first_recent_uid; + uint32_t first_uid; + unsigned int changes = 0; + int ret = 0; + time_t time_before_sync; + guid_128_t expunged_guid_128; + enum mail_flags private_flags_mask; + bool expunged, full_rescan = FALSE; + + i_assert(!mbox->syncing_commit); + + first_uid = 1; + hdr = mail_index_get_header(view); + uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); + if (uid_validity != hdr->uid_validity && + uid_validity != 0 && hdr->uid_validity != 0) { + /* uidvalidity changed and index isn't being synced for the + first time, reset the index so we can add all messages as + new */ + i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)", + mailbox_get_path(&ctx->mbox->box), + hdr->uid_validity, uid_validity); + mail_index_reset(trans); + mailbox_recent_flags_reset(&mbox->box); + + first_uid = hdr->messages_count + 1; + i_zero(&empty_hdr); + empty_hdr.next_uid = 1; + hdr = &empty_hdr; + } + hdr_next_uid = hdr->next_uid; + + ctx->mbox->box.tmp_sync_view = view; + private_flags_mask = mailbox_get_private_flags_mask(&mbox->box); + time_before_sync = time(NULL); + mbox->syncing_commit = TRUE; + seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1); + i_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS); + i_array_init(&ctx->idx_keywords, MAILDIR_MAX_KEYWORDS); + iter = maildir_uidlist_iter_init(mbox->uidlist); + while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) { + maildir_filename_flags_get(ctx->keywords_sync_ctx, filename, + &ctx->flags, &ctx->keywords); + + i_assert(uid > prev_uid); + prev_uid = uid; + + /* the private flags are kept only in indexes. don't use them + at all even for newly seen mails */ + ctx->flags &= ENUM_NEGATE(private_flags_mask); + + again: + seq++; + ctx->uid = uid; + + if (seq > hdr->messages_count) { + if (uid < hdr_next_uid) { + if (maildir_handle_uid_insertion(ctx, uflags, + filename, + uid) < 0) + ret = -1; + seq--; + continue; + } + + /* Trust uidlist recent flags only for newly added + messages. When saving/copying messages with flags + they're stored to cur/ and uidlist treats them + as non-recent. */ + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) == 0) { + if (uid >= first_recent_uid) + first_recent_uid = uid + 1; + } + + hdr_next_uid = uid + 1; + mail_index_append(trans, uid, &seq); + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + ctx->flags); + if (array_count(&ctx->keywords) > 0) { + struct mail_keywords *kw; + + kw = mail_index_keywords_create_from_indexes( + mbox->box.index, &ctx->keywords); + mail_index_update_keywords(trans, seq, + MODIFY_REPLACE, kw); + mail_index_keywords_unref(&kw); + } + continue; + } + + rec = mail_index_lookup(view, seq); + if (uid > rec->uid) { + /* already expunged (no point in showing guid in the + expunge record anymore) */ + mail_index_expunge(ctx->trans, seq); + goto again; + } + + if (uid < rec->uid) { + if (maildir_handle_uid_insertion(ctx, uflags, + filename, uid) < 0) + ret = -1; + seq--; + continue; + } + + index_sync_changes_read(ctx->sync_changes, ctx->uid, &expunged, + expunged_guid_128); + if (expunged) { + if (!maildir_expunge_is_valid_guid(ctx, ctx->uid, + filename, + expunged_guid_128)) + continue; + if (maildir_file_do(mbox, ctx->uid, + maildir_expunge, ctx) >= 0) { + /* successful expunge */ + mail_index_expunge(ctx->trans, seq); + } + if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx->maildir_sync_ctx); + continue; + } + + /* the private flags are stored only in indexes, keep them */ + ctx->flags |= rec->flags & private_flags_mask; + + if (index_sync_changes_have(ctx->sync_changes)) { + /* apply flag changes to maildir */ + if (maildir_file_do(mbox, ctx->uid, + maildir_sync_flags, ctx) < 0) + ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx->maildir_sync_ctx); + } + + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) { + /* we last saw this mail in new/, but it's + not there anymore. possibly expunged, + make sure. */ + full_rescan = TRUE; + } + continue; + } + + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* we haven't been able to update maildir with this + record's flag changes. don't sync them. */ + continue; + } + + if (ctx->flags != (rec->flags & MAIL_FLAGS_NONRECENT)) { + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + ctx->flags); + } + + maildir_sync_mail_keywords(ctx, seq); + } + maildir_uidlist_iter_deinit(&iter); + + if (!partial) { + /* expunge the rest */ + for (seq++; seq <= hdr->messages_count; seq++) + mail_index_expunge(ctx->trans, seq); + } + + /* add \Recent flags. use updated view so it contains newly + appended messages. */ + view2 = mail_index_transaction_open_updated_view(trans); + if (mail_index_lookup_seq_range(view2, first_recent_uid, (uint32_t)-1, + &seq, &seq2) && seq2 >= first_uid) { + if (seq < first_uid) { + /* UIDVALIDITY changed, skip over the old messages */ + seq = first_uid; + } + mailbox_recent_flags_set_seqs(&mbox->box, view2, seq, seq2); + } + mail_index_view_close(&view2); + + if (ctx->uidlist_sync_ctx != NULL) { + if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, + TRUE) < 0) + ret = -1; + } + + mailbox_sync_notify(&mbox->box, 0, 0); + ctx->mbox->box.tmp_sync_view = NULL; + + /* check cur/ mtime later. if we came here from saving messages they + could still be moved to cur/ directory. */ + ctx->update_maildir_hdr_cur = TRUE; + mbox->maildir_hdr.cur_check_time = time_before_sync; + + if (uid_validity == 0) { + uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity : + maildir_get_uidvalidity_next(mbox->box.list); + maildir_uidlist_set_uid_validity(mbox->uidlist, uid_validity); + } + maildir_uidlist_set_next_uid(mbox->uidlist, hdr_next_uid, FALSE); + + if (uid_validity != hdr->uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + next_uid = maildir_uidlist_get_next_uid(mbox->uidlist); + if (hdr_next_uid < next_uid) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &next_uid, sizeof(next_uid), FALSE); + } + + i_assert(hdr->first_recent_uid <= first_recent_uid); + if (hdr->first_recent_uid < first_recent_uid) { + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + array_free(&ctx->keywords); + array_free(&ctx->idx_keywords); + mbox->syncing_commit = FALSE; + return ret < 0 ? -1 : (full_rescan ? 0 : 1); +} + +static unsigned int +maildir_list_get_ext_id(struct maildir_mailbox *mbox, + struct mail_index_view *view) +{ + if (mbox->maildir_list_index_ext_id == (uint32_t)-1) { + mbox->maildir_list_index_ext_id = + mail_index_ext_register(mail_index_view_get_index(view), + "maildir", 0, + sizeof(struct maildir_list_index_record), + sizeof(uint32_t)); + } + return mbox->maildir_list_index_ext_id; +} + +int maildir_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, + const char **reason_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + const struct maildir_list_index_record *rec; + const void *data; + const char *root_dir, *new_dir, *cur_dir; + struct stat st; + uint32_t ext_id; + bool expunged; + int ret; + + ret = index_storage_list_index_has_changed(box, list_view, seq, + quick, reason_r); + if (ret != 0 || box->storage->set->mailbox_list_index_very_dirty_syncs) + return ret; + if (mbox->storage->set->maildir_very_dirty_syncs) { + /* we don't track cur/new directories with dirty syncs */ + return 0; + } + + ext_id = maildir_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + rec = data; + + if (rec == NULL) { + *reason_r = "Maildir record is missing"; + return 1; + } else if (expunged) { + *reason_r = "Maildir record is expunged"; + return 1; + } else if (rec->new_mtime == 0) { + /* not synced */ + *reason_r = "Maildir record new_mtime=0"; + return 1; + } else if (rec->cur_mtime == 0) { + /* dirty-synced */ + *reason_r = "Maildir record cur_mtime=0"; + return 1; + } + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &root_dir); + if (ret < 0) + return ret; + i_assert(ret > 0); + + /* check if new/ changed */ + new_dir = t_strconcat(root_dir, "/new", NULL); + if (stat(new_dir, &st) < 0) { + mailbox_set_critical(box, "stat(%s) failed: %m", new_dir); + return -1; + } + if ((time_t)rec->new_mtime != st.st_mtime) { + *reason_r = t_strdup_printf( + "Maildir new_mtime changed %u != %"PRIdTIME_T, + rec->new_mtime, st.st_mtime); + return 1; + } + + /* check if cur/ changed */ + cur_dir = t_strconcat(root_dir, "/cur", NULL); + if (stat(cur_dir, &st) < 0) { + mailbox_set_critical(box, "stat(%s) failed: %m", cur_dir); + return -1; + } + if ((time_t)rec->cur_mtime != st.st_mtime) { + *reason_r = t_strdup_printf( + "Maildir cur_mtime changed %u != %"PRIdTIME_T, + rec->cur_mtime, st.st_mtime); + return 1; + } + return 0; +} + +void maildir_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + struct mail_index_view *list_view; + const struct maildir_index_header *mhdr = &mbox->maildir_hdr; + const struct maildir_list_index_record *old_rec; + struct maildir_list_index_record new_rec; + const void *data; + uint32_t ext_id; + bool expunged; + + index_storage_list_index_update_sync(box, trans, seq); + if (mbox->storage->set->maildir_very_dirty_syncs) { + /* we don't track cur/new directories with dirty syncs */ + return; + } + + /* get the current record */ + list_view = mail_index_transaction_get_view(trans); + ext_id = maildir_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + if (expunged) + return; + old_rec = data; + + i_zero(&new_rec); + if (mhdr->new_check_time <= mhdr->new_mtime + MAILDIR_SYNC_SECS || + mhdr->cur_check_time <= mhdr->cur_mtime + MAILDIR_SYNC_SECS) { + /* dirty, we need a refresh next time */ + } else { + new_rec.new_mtime = mhdr->new_mtime; + new_rec.cur_mtime = mhdr->cur_mtime; + } + + if (old_rec == NULL || + memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0) + mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL); +} diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c new file mode 100644 index 0000000..0814415 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-sync.c @@ -0,0 +1,1132 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* + Here's a description of how we handle Maildir synchronization and + it's problems: + + We want to be as efficient as we can. The most efficient way to + check if changes have occurred is to stat() the new/ and cur/ + directories and uidlist file - if their mtimes haven't changed, + there's no changes and we don't need to do anything. + + Problem 1: Multiple changes can happen within a single second - + nothing guarantees that once we synced it, someone else didn't just + then make a modification. Such modifications wouldn't get noticed + until a new modification occurred later. + + Problem 2: Syncing cur/ directory is much more costly than syncing + new/. Moving mails from new/ to cur/ will always change mtime of + cur/ causing us to sync it as well. + + Problem 3: We may not be able to move mail from new/ to cur/ + because we're out of quota, or simply because we're accessing a + read-only mailbox. + + + MAILDIR_SYNC_SECS + ----------------- + + Several checks below use MAILDIR_SYNC_SECS, which should be maximum + clock drift between all computers accessing the maildir (eg. via + NFS), rounded up to next second. Our default is 1 second, since + everyone should be using NTP. + + Note that setting it to 0 works only if there's only one computer + accessing the maildir. It's practically impossible to make two + clocks _exactly_ synchronized. + + It might be possible to only use file server's clock by looking at + the atime field, but I don't know how well that would actually work. + + cur directory + ------------- + + We have dirty_cur_time variable which is set to cur/ directory's + mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have + synchronized the directory. + + When dirty_cur_time is non-zero, we don't synchronize the cur/ + directory until + + a) cur/'s mtime changes + b) opening a mail fails with ENOENT + c) time() > dirty_cur_time + MAILDIR_SYNC_SECS + + This allows us to modify the maildir multiple times without having + to sync it at every change. The sync will eventually be done to + make sure we didn't miss any external changes. + + The dirty_cur_time is set when: + + - we change message flags + - we expunge messages + - we move mail from new/ to cur/ + - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS + + It's unset when we do the final syncing, ie. when mtime is + older than time() - MAILDIR_SYNC_SECS. + + new directory + ------------- + + If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize + it. dirty_cur_time-like feature might save us a few syncs, but + that might break a client which saves a mail in one connection and + tries to fetch it in another one. new/ directory is almost always + empty, so syncing it should be very fast anyway. Actually this can + still happen if we sync only new/ dir while another client is also + moving mails from it to cur/ - it takes us a while to see them. + That's pretty unlikely to happen however, and only way to fix it + would be to always synchronize cur/ after new/. + + Normally we move all mails from new/ to cur/ whenever we sync it. If + it's not possible for some reason, we mark the mail with "probably + exists in new/ directory" flag. + + If rename() still fails because of ENOSPC or EDQUOT, we still save + the flag changes in index with dirty-flag on. When moving the mail + to cur/ directory, or when we notice it's already moved there, we + apply the flag changes to the filename, rename it and remove the + dirty flag. If there's dirty flags, this should be tried every time + after expunge or when closing the mailbox. + + uidlist + ------- + + This file contains UID <-> filename mappings. It's updated only when + new mail arrives, so it may contain filenames that have already been + deleted. Updating is done by getting uidlist.lock file, writing the + whole uidlist into it and rename()ing it over the old uidlist. This + means there's no need to lock the file for reading. + + Whenever uidlist is rewritten, it's mtime must be larger than the old + one's. Use utime() before rename() if needed. Note that inode checking + wouldn't have been sufficient as inode numbers can be reused. + + This file is usually read the first time you need to know filename for + given UID. After that it's not re-read unless new mails come that we + don't know about. + + broken clients + -------------- + + Originally the middle identifier in Maildir filename was specified + only as <process id>_<delivery counter>. That however created a + problem with randomized PIDs which made it possible that the same + PID was reused within one second. + + So if within one second a mail was delivered, MUA moved it to cur/ + and another mail was delivered by a new process using same PID as + the first one, we likely ended up overwriting the first mail when + the second mail was moved over it. + + Nowadays everyone should be giving a bit more specific identifier, + for example include microseconds in it which Dovecot does. + + There's a simple way to prevent this from happening in some cases: + Don't move the mail from new/ to cur/ if it's mtime is >= time() - + MAILDIR_SYNC_SECS. The second delivery's link() call then fails + because the file is already in new/, and it will then use a + different filename. There's a few problems with this however: + + - it requires extra stat() call which is unneeded extra I/O + - another MUA might still move the mail to cur/ + - if first file's flags are modified by either Dovecot or another + MUA, it's moved to cur/ (you _could_ just do the dirty-flagging + but that'd be ugly) + + Because this is useful only for very few people and it requires + extra I/O, I decided not to implement this. It should be however + quite easy to do since we need to be able to deal with files in new/ + in any case. + + It's also possible to never accidentally overwrite a mail by using + link() + unlink() rather than rename(). This however isn't very + good idea as it introduces potential race conditions when multiple + clients are accessing the mailbox: + + Trying to move the same mail from new/ to cur/ at the same time: + + a) Client 1 uses slightly different filename than client 2, + for example one sets read-flag on but the other doesn't. + You have the same mail duplicated now. + + b) Client 3 sees the mail between Client 1's and 2's link() calls + and changes it's flag. You have the same mail duplicated now. + + And it gets worse when they're unlink()ing in cur/ directory: + + c) Client 1 changes mails's flag and client 2 changes it back + between 1's link() and unlink(). The mail is now expunged. + + d) If you try to deal with the duplicates by unlink()ing another + one of them, you might end up unlinking both of them. + + So, what should we do then if we notice a duplicate? First of all, + it might not be a duplicate at all, readdir() might have just + returned it twice because it was just renamed. What we should do is + create a completely new base name for it and rename() it to that. + If the call fails with ENOENT, it only means that it wasn't a + duplicate after all. +*/ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "hash.h" +#include "str.h" +#include "eacces-error.h" +#include "nfs-workarounds.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-filename.h" +#include "maildir-sync.h" + +#include <stdio.h> +#include <stddef.h> +#include <unistd.h> +#include <dirent.h> +#include <sys/stat.h> + +#define MAILDIR_FILENAME_FLAG_FOUND 128 + +/* When rename()ing many files from new/ to cur/, it's possible that next + readdir() skips some files. we don't of course wish to lose them, so we + go and rescan the new/ directory again from beginning until no files are + left. This value is just an optimization to avoid checking the directory + twice needlessly. usually only NFS is the problem case. 1 is the safest + bet here, but I guess 5 will do just fine too. */ +#define MAILDIR_RENAME_RESCAN_COUNT 5 + +/* This is mostly to avoid infinite looping when rename() destination already + exists as the hard link of the file itself. */ +#define MAILDIR_SCAN_DIR_MAX_COUNT 5 + +#define DUPE_LINKS_DELETE_SECS 30 + +enum maildir_scan_why { + WHY_FORCED = 0x01, + WHY_FIRSTSYNC = 0x02, + WHY_NEWCHANGED = 0x04, + WHY_CURCHANGED = 0x08, + WHY_DROPRECENT = 0x10, + WHY_FINDRECENT = 0x20, + WHY_DELAYEDNEW = 0x40, + WHY_DELAYEDCUR = 0x80 +}; + +struct maildir_sync_context { + struct maildir_mailbox *mbox; + const char *new_dir, *cur_dir; + + enum mailbox_sync_flags flags; + time_t last_touch, last_notify; + + struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; + struct maildir_index_sync_context *index_sync_ctx; + + bool partial:1; + bool locked:1; + bool racing:1; +}; + +void maildir_sync_set_racing(struct maildir_sync_context *ctx) +{ + ctx->racing = TRUE; +} + +void maildir_sync_notify(struct maildir_sync_context *ctx) +{ + time_t now; + + if (ctx == NULL) { + /* we got here from maildir-save.c. it has no + maildir_sync_context, */ + return; + } + + now = time(NULL); + if (now - ctx->last_touch > MAILDIR_LOCK_TOUCH_SECS && ctx->locked) { + (void)maildir_uidlist_lock_touch(ctx->mbox->uidlist); + ctx->last_touch = now; + } + if (now - ctx->last_notify > MAIL_STORAGE_STAYALIVE_SECS) { + struct mailbox *box = &ctx->mbox->box; + + if (box->storage->callbacks.notify_ok != NULL) { + box->storage->callbacks. + notify_ok(box, "Hang in there..", + box->storage->callback_context); + } + ctx->last_notify = now; + } +} + +static struct maildir_sync_context * +maildir_sync_context_new(struct maildir_mailbox *mbox, + enum mailbox_sync_flags flags) +{ + struct maildir_sync_context *ctx; + + ctx = t_new(struct maildir_sync_context, 1); + ctx->mbox = mbox; + ctx->new_dir = t_strconcat(mailbox_get_path(&mbox->box), "/new", NULL); + ctx->cur_dir = t_strconcat(mailbox_get_path(&mbox->box), "/cur", NULL); + ctx->last_touch = ioloop_time; + ctx->last_notify = ioloop_time; + ctx->flags = flags; + return ctx; +} + +static void maildir_sync_deinit(struct maildir_sync_context *ctx) +{ + if (ctx->uidlist_sync_ctx != NULL) + (void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE); + if (ctx->index_sync_ctx != NULL) + maildir_sync_index_rollback(&ctx->index_sync_ctx); + if (ctx->mbox->storage->storage.rebuild_list_index) + (void)mail_storage_list_index_rebuild_and_set_uncorrupted(&ctx->mbox->storage->storage); +} + +static int maildir_fix_duplicate(struct maildir_sync_context *ctx, + const char *dir, const char *fname2) +{ + const char *fname1, *path1, *path2; + const char *new_fname, *new_path; + struct stat st1, st2; + uoff_t size; + + fname1 = maildir_uidlist_sync_get_full_filename(ctx->uidlist_sync_ctx, + fname2); + i_assert(fname1 != NULL); + + path1 = t_strconcat(dir, "/", fname1, NULL); + path2 = t_strconcat(dir, "/", fname2, NULL); + + if (stat(path1, &st1) < 0 || stat(path2, &st2) < 0) { + /* most likely the files just don't exist anymore. + don't really care about other errors much. */ + return 0; + } + if (st1.st_ino == st2.st_ino && + CMP_DEV_T(st1.st_dev, st2.st_dev)) { + /* Files are the same. this means either a race condition + between stat() calls, or that the files were link()ed. */ + if (st1.st_nlink > 1 && st2.st_nlink == st1.st_nlink && + st1.st_ctime == st2.st_ctime && + st1.st_ctime < ioloop_time - DUPE_LINKS_DELETE_SECS) { + /* The file has hard links and it hasn't had any + changes (such as renames) for a while, so this + isn't a race condition. + + rename()ing one file on top of the other would fix + this safely, except POSIX decided that rename() + doesn't work that way. So we'll have unlink() one + and hope that another process didn't just decide to + unlink() the other (uidlist lock prevents this from + happening) */ + if (i_unlink(path2) == 0) + i_warning("Unlinked a duplicate: %s", path2); + } + return 0; + } + + new_fname = maildir_filename_generate(); + /* preserve S= and W= sizes if they're available. + (S=size is required for zlib plugin to work) */ + if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_FILE_SIZE, &size)) { + new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T, + new_fname, MAILDIR_EXTRA_FILE_SIZE, size); + } + if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) { + new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T, + new_fname, MAILDIR_EXTRA_VIRTUAL_SIZE, size); + } + new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box), + "/new/", new_fname, NULL); + + if (rename(path2, new_path) == 0) + i_warning("Fixed a duplicate: %s -> %s", path2, new_fname); + else if (errno != ENOENT) { + mailbox_set_critical(&ctx->mbox->box, + "Couldn't fix a duplicate: rename(%s, %s) failed: %m", + path2, new_path); + return -1; + } + return 0; +} + +static int +maildir_rename_empty_basename(struct maildir_sync_context *ctx, + const char *dir, const char *fname) +{ + const char *old_path, *new_fname, *new_path; + + old_path = t_strconcat(dir, "/", fname, NULL); + new_fname = maildir_filename_generate(); + new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box), + "/new/", new_fname, NULL); + if (rename(old_path, new_path) == 0) + i_warning("Fixed broken filename: %s -> %s", old_path, new_fname); + else if (errno != ENOENT) { + mailbox_set_critical(&ctx->mbox->box, + "Couldn't fix a broken filename: rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + return 0; +} + +static int +maildir_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st_r) +{ + struct mailbox *box = &mbox->box; + int i; + + for (i = 0;; i++) { + if (nfs_safe_stat(path, st_r) == 0) + return 0; + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) + break; + + if (!maildir_set_deleted(box)) + return -1; + /* try again */ + } + + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return -1; +} + +static int +maildir_scan_dir(struct maildir_sync_context *ctx, bool new_dir, bool final, + enum maildir_scan_why why) +{ + const char *path; + DIR *dirp; + string_t *src, *dest; + struct dirent *dp; + struct stat st; + enum maildir_uidlist_rec_flag flags; + unsigned int time_diff, i, readdir_count = 0, move_count = 0; + time_t start_time; + int ret = 1; + bool move_new, dir_changed = FALSE; + + path = new_dir ? ctx->new_dir : ctx->cur_dir; + for (i = 0;; i++) { + dirp = opendir(path); + if (dirp != NULL) + break; + + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + if (errno == EACCES) { + mailbox_set_critical(&ctx->mbox->box, "%s", + eacces_error_get("opendir", path)); + } else { + mailbox_set_critical(&ctx->mbox->box, + "opendir(%s) failed: %m", path); + } + return -1; + } + + if (!maildir_set_deleted(&ctx->mbox->box)) + return -1; + /* try again */ + } + +#ifdef HAVE_DIRFD + if (fstat(dirfd(dirp), &st) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "fstat(%s) failed: %m", path); + (void)closedir(dirp); + return -1; + } +#else + if (maildir_stat(ctx->mbox, path, &st) < 0) { + (void)closedir(dirp); + return -1; + } +#endif + + start_time = time(NULL); + if (new_dir) { + ctx->mbox->maildir_hdr.new_check_time = start_time; + ctx->mbox->maildir_hdr.new_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.new_mtime_nsecs = ST_MTIME_NSEC(st); + } else { + ctx->mbox->maildir_hdr.cur_check_time = start_time; + ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st); + } + + src = t_str_new(1024); + dest = t_str_new(1024); + + move_new = new_dir && ctx->locked && + ((ctx->mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0 || + ctx->mbox->storage->set->maildir_empty_new); + + errno = 0; + for (; (dp = readdir(dirp)) != NULL; errno = 0) { + if (dp->d_name[0] == '.') + continue; + + if (dp->d_name[0] == MAILDIR_INFO_SEP) { + /* don't even try to use file with empty base name */ + if (maildir_rename_empty_basename(ctx, path, + dp->d_name) < 0) + break; + continue; + } + + flags = 0; + if (move_new) { + i_assert(dp->d_name[0] != '\0'); + + str_truncate(src, 0); + str_truncate(dest, 0); + str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name); + str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name); + if (strchr(dp->d_name, MAILDIR_INFO_SEP) == NULL) { + str_append(dest, MAILDIR_FLAGS_FULL_SEP); + } + if (rename(str_c(src), str_c(dest)) == 0) { + /* we moved it - it's \Recent for us */ + dir_changed = TRUE; + move_count++; + flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED | + MAILDIR_UIDLIST_REC_FLAG_RECENT; + } else if (ENOTFOUND(errno)) { + /* someone else moved it already */ + dir_changed = TRUE; + move_count++; + flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED | + MAILDIR_UIDLIST_REC_FLAG_RECENT; + } else if (ENOSPACE(errno) || errno == EACCES) { + /* not enough disk space / read-only maildir, + leave here */ + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + move_new = FALSE; + } else { + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + mailbox_set_critical(&ctx->mbox->box, + "rename(%s, %s) failed: %m", + str_c(src), str_c(dest)); + } + if ((move_count % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx); + } else if (new_dir) { + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | + MAILDIR_UIDLIST_REC_FLAG_RECENT; + } + + readdir_count++; + if ((readdir_count % MAILDIR_SLOW_CHECK_COUNT) == 0) + maildir_sync_notify(ctx); + + ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx, + dp->d_name, flags); + if (ret <= 0) { + if (ret < 0) + break; + + /* possibly duplicate - try fixing it */ + T_BEGIN { + ret = maildir_fix_duplicate(ctx, path, + dp->d_name); + } T_END; + if (ret < 0) + break; + } + } + +#ifdef __APPLE__ + if (errno == EINVAL && move_count > 0 && !final) { + /* OS X HFS+: readdir() fails sometimes when rename() + have been done. */ + move_count = MAILDIR_RENAME_RESCAN_COUNT + 1; + } else +#endif + + if (errno != 0) { + mailbox_set_critical(&ctx->mbox->box, + "readdir(%s) failed: %m", path); + ret = -1; + } + + if (closedir(dirp) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "closedir(%s) failed: %m", path); + ret = -1; + } + + if (dir_changed) { + /* save the exact new times. the new mtimes should be >= + "start_time", but just in case something weird happens and + mtime doesn't update, use "start_time". */ + if (stat(ctx->new_dir, &st) == 0) { + ctx->mbox->maildir_hdr.new_check_time = + I_MAX(st.st_mtime, start_time); + ctx->mbox->maildir_hdr.new_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.new_mtime_nsecs = + ST_MTIME_NSEC(st); + } + if (stat(ctx->cur_dir, &st) == 0) { + ctx->mbox->maildir_hdr.new_check_time = + I_MAX(st.st_mtime, start_time); + ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.cur_mtime_nsecs = + ST_MTIME_NSEC(st); + } + } + time_diff = time(NULL) - start_time; + if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) { + i_warning("Maildir: Scanning %s took %u seconds " + "(%u readdir()s, %u rename()s to cur/, why=0x%x)", + path, time_diff, readdir_count, move_count, why); + } + + return ret < 0 ? -1 : + (move_count <= MAILDIR_RENAME_RESCAN_COUNT || final ? 0 : 1); +} + +static void maildir_sync_get_header(struct maildir_mailbox *mbox) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id, + &data, &data_size); + if (data_size == 0) { + /* header doesn't exist */ + } else { + memcpy(&mbox->maildir_hdr, data, + I_MIN(sizeof(mbox->maildir_hdr), data_size)); + } +} + +int maildir_sync_header_refresh(struct maildir_mailbox *mbox) +{ + if (mail_index_refresh(mbox->box.index) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + maildir_sync_get_header(mbox); + return 0; +} + +static int maildir_sync_quick_check(struct maildir_mailbox *mbox, bool undirty, + const char *new_dir, const char *cur_dir, + bool *new_changed_r, bool *cur_changed_r, + enum maildir_scan_why *why_r) +{ +#define DIR_DELAYED_REFRESH(hdr, name) \ + ((hdr)->name ## _check_time <= \ + (hdr)->name ## _mtime + MAILDIR_SYNC_SECS && \ + (undirty || \ + (time_t)(hdr)->name ## _check_time < ioloop_time - MAILDIR_SYNC_SECS)) + +#define DIR_MTIME_CHANGED(st, hdr, name) \ + ((st).st_mtime != (time_t)(hdr)->name ## _mtime || \ + !ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), (hdr)->name ## _mtime_nsecs)) + + struct maildir_index_header *hdr = &mbox->maildir_hdr; + struct stat new_st, cur_st; + bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE; + + *why_r = 0; + + if (mbox->maildir_hdr.new_mtime == 0) { + maildir_sync_get_header(mbox); + if (mbox->maildir_hdr.new_mtime == 0) { + /* first sync */ + *why_r |= WHY_FIRSTSYNC; + *new_changed_r = *cur_changed_r = TRUE; + return 0; + } + } + + *new_changed_r = *cur_changed_r = FALSE; + + /* try to avoid stat()ing by first checking delayed changes */ + if (DIR_DELAYED_REFRESH(hdr, new) || + (DIR_DELAYED_REFRESH(hdr, cur) && + !mbox->storage->set->maildir_very_dirty_syncs)) { + /* refresh index and try again */ + if (maildir_sync_header_refresh(mbox) < 0) + return -1; + refreshed = TRUE; + + if (DIR_DELAYED_REFRESH(hdr, new)) { + *why_r |= WHY_DELAYEDNEW; + *new_changed_r = TRUE; + } + if (DIR_DELAYED_REFRESH(hdr, cur) && + !mbox->storage->set->maildir_very_dirty_syncs) { + *why_r |= WHY_DELAYEDCUR; + *cur_changed_r = TRUE; + } + if (*new_changed_r && *cur_changed_r) + return 0; + } + + if (!*new_changed_r) { + if (maildir_stat(mbox, new_dir, &new_st) < 0) + return -1; + check_new = TRUE; + } + if (!*cur_changed_r) { + if (maildir_stat(mbox, cur_dir, &cur_st) < 0) + return -1; + check_cur = TRUE; + } + + for (;;) { + if (check_new) { + *new_changed_r = DIR_MTIME_CHANGED(new_st, hdr, new); + if (*new_changed_r) + *why_r |= WHY_NEWCHANGED; + } + if (check_cur) { + *cur_changed_r = DIR_MTIME_CHANGED(cur_st, hdr, cur); + if (*cur_changed_r) + *why_r |= WHY_CURCHANGED; + } + + if ((!*new_changed_r && !*cur_changed_r) || refreshed) + break; + + /* refresh index and try again */ + if (maildir_sync_header_refresh(mbox) < 0) + return -1; + refreshed = TRUE; + } + + return 0; +} + +static void maildir_sync_update_next_uid(struct maildir_mailbox *mbox) +{ + const struct mail_index_header *hdr; + uint32_t uid_validity; + + hdr = mail_index_get_header(mbox->box.view); + if (hdr->uid_validity == 0) + return; + + uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); + if (uid_validity == hdr->uid_validity || uid_validity == 0) { + /* make sure uidlist's next_uid is at least as large as + index file's. typically this happens only if uidlist gets + deleted. */ + maildir_uidlist_set_uid_validity(mbox->uidlist, + hdr->uid_validity); + maildir_uidlist_set_next_uid(mbox->uidlist, + hdr->next_uid, FALSE); + } +} + +static bool +have_recent_messages(struct maildir_sync_context *ctx, bool seen_changes) +{ + const struct mail_index_header *hdr; + uint32_t next_uid; + + hdr = mail_index_get_header(ctx->mbox->box.view); + if (!seen_changes) { + /* index is up to date. get the next-uid from it */ + next_uid = hdr->next_uid; + } else { + (void)maildir_uidlist_refresh(ctx->mbox->uidlist); + next_uid = maildir_uidlist_get_next_uid(ctx->mbox->uidlist); + } + return hdr->first_recent_uid < next_uid; +} + +static int maildir_sync_get_changes(struct maildir_sync_context *ctx, + bool *new_changed_r, bool *cur_changed_r, + enum maildir_scan_why *why_r) +{ + struct maildir_mailbox *mbox = ctx->mbox; + enum mail_index_sync_flags flags = 0; + bool undirty = (ctx->flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0; + + *why_r = 0; + + if (maildir_sync_quick_check(mbox, undirty, ctx->new_dir, ctx->cur_dir, + new_changed_r, cur_changed_r, why_r) < 0) + return -1; + + /* if there are files in new/, we'll need to move them. we'll check + this by seeing if we have any recent messages */ + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) { + if (!*new_changed_r && have_recent_messages(ctx, FALSE)) { + *new_changed_r = TRUE; + *why_r |= WHY_DROPRECENT; + } + } else if (*new_changed_r) { + /* if recent messages have been externally deleted from new/, + we need to get them out of index. this requires that + we make sure they weren't just moved to cur/. */ + if (!*cur_changed_r && have_recent_messages(ctx, TRUE)) { + *cur_changed_r = TRUE; + *why_r |= WHY_FINDRECENT; + } + } + + if (*new_changed_r || *cur_changed_r) + return 1; + + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) + flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; + + if (mbox->synced) { + /* refresh index only after the first sync, i.e. avoid wasting + time on refreshing it immediately after it was just opened */ + mail_index_refresh(mbox->box.index); + } + return mail_index_sync_have_any(mbox->box.index, flags) ? 1 : 0; +} + +static int ATTR_NULL(3) +maildir_sync_context(struct maildir_sync_context *ctx, bool forced, + uint32_t *find_uid, bool *lost_files_r) +{ + enum maildir_uidlist_sync_flags sync_flags; + enum maildir_uidlist_rec_flag flags; + bool new_changed, cur_changed, lock_failure; + const char *fname; + enum maildir_scan_why why; + int ret; + + *lost_files_r = FALSE; + + if (forced) { + new_changed = cur_changed = TRUE; + why = WHY_FORCED; + } else { + ret = maildir_sync_get_changes(ctx, &new_changed, &cur_changed, + &why); + if (ret <= 0) + return ret; + } + + /* + Locking, locking, locking.. Wasn't maildir supposed to be lockless? + + We can get here either as beginning a real maildir sync, or when + committing changes to maildir but a file was lost (maybe renamed). + + So, we're going to need two locks. One for index and one for + uidlist. To avoid deadlocking do the uidlist lock always first. + + uidlist is needed only for figuring out UIDs for newly seen files, + so theoretically we wouldn't need to lock it unless there are new + files. It has a few problems though, assuming the index lock didn't + already protect it (eg. in-memory indexes): + + 1. Just because you see a new file which doesn't exist in uidlist + file, doesn't mean that the file really exists anymore, or that + your readdir() lists all new files. Meaning that this is possible: + + A: opendir(), readdir() -> new file ... + -- new files are written to the maildir -- + B: opendir(), readdir() -> new file, lock uidlist, + readdir() -> another new file, rewrite uidlist, unlock + A: ... lock uidlist, readdir() -> nothing left, rewrite uidlist, + unlock + + The second time running A didn't see the two new files. To handle + this correctly, it must not remove the new unseen files from + uidlist. This is possible to do, but adds extra complexity. + + 2. If another process is rename()ing files while we are + readdir()ing, it's possible that readdir() never lists some files, + causing Dovecot to assume they were expunged. In next sync they + would show up again, but client could have already been notified of + that and they would show up under new UIDs, so the damage is + already done. + + Both of the problems can be avoided if we simply lock the uidlist + before syncing and keep it until sync is finished. Typically this + would happen in any case, as there is the index lock.. + + The second case is still a problem with external changes though, + because maildir doesn't require any kind of locking. Luckily this + problem rarely happens except under high amount of modifications. + */ + + if (!cur_changed) { + ctx->partial = TRUE; + sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL; + } else { + ctx->partial = FALSE; + sync_flags = 0; + if (forced) + sync_flags |= MAILDIR_UIDLIST_SYNC_FORCE; + if ((ctx->flags & MAILBOX_SYNC_FLAG_FAST) != 0) + sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK; + } + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags, + &ctx->uidlist_sync_ctx); + lock_failure = ret <= 0; + if (ret <= 0) { + struct mail_storage *storage = ctx->mbox->box.storage; + + if (ret == 0) { + /* timeout */ + return 0; + } + /* locking failed. sync anyway without locking so that it's + possible to expunge messages when out of quota. */ + if (forced) { + /* we're already forcing a sync, we're trying to find + a message that was probably already expunged, don't + loop for a long time trying to find it. */ + return -1; + } + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags | + MAILDIR_UIDLIST_SYNC_NOLOCK, + &ctx->uidlist_sync_ctx); + if (ret <= 0) { + i_assert(ret != 0); + return -1; + } + + if (storage->callbacks.notify_no != NULL) { + storage->callbacks.notify_no(&ctx->mbox->box, + "Internal mailbox synchronization failure, " + "showing only old mails.", + storage->callback_context); + } + } + ctx->locked = maildir_uidlist_is_locked(ctx->mbox->uidlist); + if (!ctx->locked) + ctx->partial = TRUE; + + if (!ctx->mbox->syncing_commit && (ctx->locked || lock_failure)) { + if (maildir_sync_index_begin(ctx->mbox, ctx, + &ctx->index_sync_ctx) < 0) + return -1; + } + + if (new_changed || cur_changed) { + /* if we're going to check cur/ dir our current logic requires + that new/ dir is checked as well. it's a good idea anyway. */ + unsigned int count = 0; + bool final = FALSE; + + while ((ret = maildir_scan_dir(ctx, TRUE, final, why)) > 0) { + /* rename()d at least some files, which might have + caused some other files to be missed. check again + (see MAILDIR_RENAME_RESCAN_COUNT). */ + if (++count >= MAILDIR_SCAN_DIR_MAX_COUNT) + final = TRUE; + } + if (ret < 0) + return -1; + + if (cur_changed) { + if (maildir_scan_dir(ctx, FALSE, TRUE, why) < 0) + return -1; + } + + maildir_sync_update_next_uid(ctx->mbox); + + /* finish uidlist syncing, but keep it still locked */ + maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx); + } + + if (!ctx->locked) { + /* make sure we sync the maildir later */ + ctx->mbox->maildir_hdr.new_mtime = 0; + ctx->mbox->maildir_hdr.cur_mtime = 0; + } + + if (ctx->index_sync_ctx != NULL) { + /* NOTE: index syncing here might cause a re-sync due to + files getting lost, so this function might be called + reentrantly. */ + ret = maildir_sync_index(ctx->index_sync_ctx, ctx->partial); + if (ret < 0) + maildir_sync_index_rollback(&ctx->index_sync_ctx); + else if (maildir_sync_index_commit(&ctx->index_sync_ctx) < 0) + return -1; + + if (ret < 0) + return -1; + if (ret == 0) + *lost_files_r = TRUE; + + i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist) || + lock_failure); + } + + if (find_uid != NULL && *find_uid != 0) { + ret = maildir_uidlist_lookup(ctx->mbox->uidlist, + *find_uid, &flags, &fname); + if (ret < 0) + return -1; + if (ret == 0) { + /* UID is expunged */ + *find_uid = 0; + } else if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) { + *find_uid = 0; + } else { + /* we didn't find it, possibly expunged? */ + } + } + + return maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, TRUE); +} + +int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r) +{ + int ret; + + ret = maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r); + if (ret != 0) + return ret; + + if (maildir_uidlist_is_open(mbox->uidlist)) { + /* refresh uidlist and check again in case it was added + after the last mailbox sync */ + if (mbox->sync_uidlist_refreshed) { + /* we've already refreshed it, don't bother again */ + return ret; + } + mbox->sync_uidlist_refreshed = TRUE; + if (maildir_uidlist_refresh(mbox->uidlist) < 0) + return -1; + } else { + /* the uidlist doesn't exist. */ + if (maildir_storage_sync_force(mbox, uid) < 0) + return -1; + } + + /* try again */ + return maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r); +} + +static int maildir_sync_run(struct maildir_mailbox *mbox, + enum mailbox_sync_flags flags, bool force_resync, + uint32_t *uid, bool *lost_files_r) +{ + struct maildir_sync_context *ctx; + bool retry, lost_files; + int ret; + + T_BEGIN { + ctx = maildir_sync_context_new(mbox, flags); + ret = maildir_sync_context(ctx, force_resync, uid, lost_files_r); + retry = ctx->racing; + maildir_sync_deinit(ctx); + } T_END; + + if (retry) T_BEGIN { + /* we're racing some file. retry the sync again to see if the + file is really gone or not. if it is, this is a bit of + unnecessary work, but if it's not, this is necessary for + e.g. doveadm force-resync to work. */ + ctx = maildir_sync_context_new(mbox, 0); + ret = maildir_sync_context(ctx, TRUE, NULL, &lost_files); + maildir_sync_deinit(ctx); + } T_END; + return ret; +} + +int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid) +{ + bool lost_files; + int ret; + + ret = maildir_sync_run(mbox, MAILBOX_SYNC_FLAG_FAST, + TRUE, &uid, &lost_files); + if (uid != 0) { + /* maybe it's expunged. check again. */ + ret = maildir_sync_run(mbox, 0, TRUE, NULL, &lost_files); + } + return ret; +} + +int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox) +{ + struct mail_index_view_sync_ctx *sync_ctx; + bool delayed_expunges; + + mail_index_refresh(mbox->box.index); + if (mbox->flags_view == NULL) + mbox->flags_view = mail_index_view_open(mbox->box.index); + + sync_ctx = mail_index_view_sync_begin(mbox->flags_view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + if (mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + /* make sure the map stays in private memory */ + if (mbox->flags_view->map->refcount > 1) { + struct mail_index_map *map; + + map = mail_index_map_clone(mbox->flags_view->map); + mail_index_unmap(&mbox->flags_view->map); + mbox->flags_view->map = map; + } + mail_index_record_map_move_to_private(mbox->flags_view->map); + mail_index_map_move_to_memory(mbox->flags_view->map); + return 0; +} + +struct mailbox_sync_context * +maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + bool lost_files, force_resync; + int ret = 0; + + force_resync = (flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0; + if (index_mailbox_want_full_sync(&mbox->box, flags)) { + ret = maildir_sync_run(mbox, flags, force_resync, + NULL, &lost_files); + i_assert(!maildir_uidlist_is_locked(mbox->uidlist) || + (box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0); + + if (lost_files) { + /* lost some files from new/, see if they're in cur/ */ + ret = maildir_storage_sync_force(mbox, 0); + } + } + + if (mbox->storage->set->maildir_very_dirty_syncs) { + if (maildir_sync_refresh_flags_view(mbox) < 0) + ret = -1; + maildir_uidlist_set_all_nonsynced(mbox->uidlist); + } + mbox->synced = TRUE; + mbox->sync_uidlist_refreshed = FALSE; + return index_mailbox_sync_init(box, flags, ret < 0); +} + +int maildir_sync_is_synced(struct maildir_mailbox *mbox) +{ + bool new_changed, cur_changed; + enum maildir_scan_why why; + int ret; + + T_BEGIN { + const char *box_path = mailbox_get_path(&mbox->box); + const char *new_dir, *cur_dir; + + new_dir = t_strconcat(box_path, "/new", NULL); + cur_dir = t_strconcat(box_path, "/cur", NULL); + + ret = maildir_sync_quick_check(mbox, FALSE, new_dir, cur_dir, + &new_changed, &cur_changed, + &why); + } T_END; + return ret < 0 ? -1 : (!new_changed && !cur_changed); +} diff --git a/src/lib-storage/index/maildir/maildir-sync.h b/src/lib-storage/index/maildir/maildir-sync.h new file mode 100644 index 0000000..9bc6db4 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-sync.h @@ -0,0 +1,59 @@ +#ifndef MAILDIR_SYNC_H +#define MAILDIR_SYNC_H + +/* All systems accessing the filesystem must have their clock less than this + many seconds apart from each others. 0 works only for local filesystems. */ +#define MAILDIR_SYNC_SECS 1 + +/* After moving this many mails from new/ to cur/, check if we need to touch + the uidlist lock. */ +#define MAILDIR_SLOW_MOVE_COUNT 100 +/* readdir() should be pretty fast to do, but check anyway every n files + to see if we need to touch the uidlist lock. */ +#define MAILDIR_SLOW_CHECK_COUNT 10000 +/* If syncing takes longer than this, log a warning. */ +#define MAILDIR_SYNC_TIME_WARN_SECS MAIL_TRANSACTION_LOG_LOCK_WARN_SECS + +struct maildir_mailbox; +struct maildir_sync_context; +struct maildir_keywords_sync_ctx; +struct maildir_index_sync_context; + +int maildir_sync_is_synced(struct maildir_mailbox *mbox); + +struct mailbox_sync_context * +maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); +int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid); + +int maildir_sync_header_refresh(struct maildir_mailbox *mbox); + +int maildir_sync_index_begin(struct maildir_mailbox *mbox, + struct maildir_sync_context *maildir_sync_ctx, + struct maildir_index_sync_context **ctx_r) + ATTR_NULL(2); +int maildir_sync_index(struct maildir_index_sync_context *sync_ctx, + bool partial); +int maildir_sync_index_commit(struct maildir_index_sync_context **_ctx); +void maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx); + +struct maildir_keywords_sync_ctx * +maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx); +void maildir_sync_set_racing(struct maildir_sync_context *ctx); +void maildir_sync_notify(struct maildir_sync_context *ctx); +void maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx, + unsigned int count); +int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox); + +int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r); + +int maildir_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, + const char **reason_r); +void maildir_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c new file mode 100644 index 0000000..bb17b9c --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-uidlist.c @@ -0,0 +1,2151 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +/* + Version 1 format has been used for most versions of Dovecot up to v1.0.x. + It's also compatible with Courier IMAP's courierimapuiddb file. + The format is: + + header: 1 <uid validity> <next uid> + entry: <uid> <filename> + + -- + + Version 2 format was written by a few development Dovecot versions, but + v1.0.x still parses the format. The format has <flags> field after <uid>. + + -- + + Version 3 format is an extensible format used by Dovecot v1.1 and later. + It's also parsed by v1.0.2 (and later). The format is: + + header: 3 [<key><value> ...] + entry: <uid> [<key><value> ...] :<filename> + + See enum maildir_uidlist_*_ext_key for used keys. +*/ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "file-dotlock.h" +#include "nfs-workarounds.h" +#include "eacces-error.h" +#include "maildir-storage.h" +#include "maildir-filename.h" +#include "maildir-uidlist.h" + +#include <stdio.h> +#include <sys/stat.h> + +/* NFS: How many times to retry reading dovecot-uidlist file if ESTALE + error occurs in the middle of reading it */ +#define UIDLIST_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT + +#define UIDLIST_VERSION 3 +#define UIDLIST_COMPRESS_PERCENTAGE 75 + +#define UIDLIST_IS_LOCKED(uidlist) \ + ((uidlist)->lock_count > 0) + +struct maildir_uidlist_rec { + uint32_t uid; + uint32_t flags; + char *filename; + unsigned char *extensions; /* <data>\0[<data>\0 ...]\0 */ +}; +ARRAY_DEFINE_TYPE(maildir_uidlist_rec_p, struct maildir_uidlist_rec *); + +HASH_TABLE_DEFINE_TYPE(path_to_maildir_uidlist_rec, + char *, struct maildir_uidlist_rec *); + +struct maildir_uidlist { + struct mailbox *box; + char *path; + struct maildir_index_header *mhdr; + + int fd; + dev_t fd_dev; + ino_t fd_ino; + off_t fd_size; + + unsigned int lock_count; + + struct dotlock_settings dotlock_settings; + struct dotlock *dotlock; + + pool_t record_pool; + ARRAY_TYPE(maildir_uidlist_rec_p) records; + HASH_TABLE_TYPE(path_to_maildir_uidlist_rec) files; + unsigned int change_counter; + + unsigned int version; + unsigned int uid_validity, next_uid, prev_read_uid, last_seen_uid; + unsigned int hdr_next_uid; + unsigned int read_records_count, read_line_count; + uoff_t last_read_offset; + string_t *hdr_extensions; + + guid_128_t mailbox_guid; + + bool recreate:1; + bool recreate_on_change:1; + bool initial_read:1; + bool initial_hdr_read:1; + bool retry_rewind:1; + bool locked_refresh:1; + bool unsorted:1; + bool have_mailbox_guid:1; + bool opened_readonly:1; +}; + +struct maildir_uidlist_sync_ctx { + struct maildir_uidlist *uidlist; + enum maildir_uidlist_sync_flags sync_flags; + + pool_t record_pool; + ARRAY_TYPE(maildir_uidlist_rec_p) records; + HASH_TABLE_TYPE(path_to_maildir_uidlist_rec) files; + + unsigned int first_unwritten_pos, first_new_pos; + unsigned int new_files_count; + unsigned int finish_change_counter; + + bool partial:1; + bool finished:1; + bool changed:1; + bool failed:1; + bool locked:1; +}; + +struct maildir_uidlist_iter_ctx { + struct maildir_uidlist *uidlist; + struct maildir_uidlist_rec *const *next, *const *end; + + unsigned int change_counter; + uint32_t prev_uid; +}; + +static int maildir_uidlist_open_latest(struct maildir_uidlist *uidlist); +static bool maildir_uidlist_iter_next_rec(struct maildir_uidlist_iter_ctx *ctx, + struct maildir_uidlist_rec **rec_r); + +static int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist, + bool nonblock, bool refresh, + bool refresh_when_locked) +{ + struct mailbox *box = uidlist->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *path = uidlist->path; + mode_t old_mask; + const enum dotlock_create_flags dotlock_flags = + nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0; + int i, ret; + + if (uidlist->lock_count > 0) { + if (!uidlist->locked_refresh && refresh_when_locked) { + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } + uidlist->lock_count++; + return 1; + } + + index_storage_lock_notify_reset(box); + + for (i = 0;; i++) { + old_mask = umask(0777 & ~perm->file_create_mode); + ret = file_dotlock_create(&uidlist->dotlock_settings, path, + dotlock_flags, &uidlist->dotlock); + umask(old_mask); + if (ret > 0) + break; + + /* failure */ + if (ret == 0) { + mail_storage_set_error(box->storage, + MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT); + return 0; + } + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + if (errno == EACCES) { + mailbox_set_critical(box, "%s", + eacces_error_get_creating("file_dotlock_create", path)); + } else { + mailbox_set_critical(box, + "file_dotlock_create(%s) failed: %m", + path); + } + return -1; + } + /* the control dir doesn't exist. create it unless the whole + mailbox was just deleted. */ + if (!maildir_set_deleted(uidlist->box)) + return -1; + } + + uidlist->lock_count++; + uidlist->locked_refresh = FALSE; + + if (refresh) { + /* make sure we have the latest changes before + changing anything */ + if (maildir_uidlist_refresh(uidlist) < 0) { + maildir_uidlist_unlock(uidlist); + return -1; + } + } + return 1; +} + +int maildir_uidlist_lock(struct maildir_uidlist *uidlist) +{ + return maildir_uidlist_lock_timeout(uidlist, FALSE, TRUE, FALSE); +} + +int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist) +{ + return maildir_uidlist_lock_timeout(uidlist, TRUE, TRUE, FALSE); +} + +int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist) +{ + i_assert(UIDLIST_IS_LOCKED(uidlist)); + + return file_dotlock_touch(uidlist->dotlock); +} + +bool maildir_uidlist_is_locked(struct maildir_uidlist *uidlist) +{ + return UIDLIST_IS_LOCKED(uidlist); +} + +bool maildir_uidlist_is_read(struct maildir_uidlist *uidlist) +{ + return uidlist->initial_read; +} + +bool maildir_uidlist_is_open(struct maildir_uidlist *uidlist) +{ + return uidlist->fd != -1; +} + +void maildir_uidlist_unlock(struct maildir_uidlist *uidlist) +{ + i_assert(uidlist->lock_count > 0); + + if (--uidlist->lock_count > 0) + return; + + uidlist->locked_refresh = FALSE; + file_dotlock_delete(&uidlist->dotlock); +} + +static bool dotlock_callback(unsigned int secs_left, bool stale, void *context) +{ + struct mailbox *box = context; + + index_storage_lock_notify(box, stale ? + MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE : + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + secs_left); + return TRUE; +} + +struct maildir_uidlist *maildir_uidlist_init(struct maildir_mailbox *mbox) +{ + struct mailbox *box = &mbox->box; + struct maildir_uidlist *uidlist; + const char *control_dir; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, + &control_dir) <= 0) + i_unreached(); + + uidlist = i_new(struct maildir_uidlist, 1); + uidlist->box = box; + uidlist->mhdr = &mbox->maildir_hdr; + uidlist->fd = -1; + uidlist->path = i_strconcat(control_dir, "/"MAILDIR_UIDLIST_NAME, NULL); + i_array_init(&uidlist->records, 128); + hash_table_create(&uidlist->files, default_pool, 4096, + maildir_filename_base_hash, + maildir_filename_base_cmp); + uidlist->next_uid = 1; + uidlist->hdr_extensions = str_new(default_pool, 128); + + uidlist->dotlock_settings.use_io_notify = TRUE; + uidlist->dotlock_settings.use_excl_lock = + box->storage->set->dotlock_use_excl; + uidlist->dotlock_settings.nfs_flush = + box->storage->set->mail_nfs_storage; + uidlist->dotlock_settings.timeout = + mail_storage_get_lock_timeout(box->storage, + MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT + 2); + uidlist->dotlock_settings.stale_timeout = + MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT; + uidlist->dotlock_settings.callback = dotlock_callback; + uidlist->dotlock_settings.context = box; + uidlist->dotlock_settings.temp_prefix = mbox->storage->temp_prefix; + return uidlist; +} + +static void maildir_uidlist_close(struct maildir_uidlist *uidlist) +{ + if (uidlist->fd != -1) { + if (close(uidlist->fd) < 0) { + mailbox_set_critical(uidlist->box, + "close(%s) failed: %m", uidlist->path); + } + uidlist->fd = -1; + uidlist->fd_ino = 0; + } + uidlist->last_read_offset = 0; + uidlist->read_line_count = 0; +} + +static void maildir_uidlist_reset(struct maildir_uidlist *uidlist) +{ + maildir_uidlist_close(uidlist); + uidlist->last_seen_uid = 0; + uidlist->initial_hdr_read = FALSE; + uidlist->read_records_count = 0; + + hash_table_clear(uidlist->files, FALSE); + array_clear(&uidlist->records); +} + +void maildir_uidlist_deinit(struct maildir_uidlist **_uidlist) +{ + struct maildir_uidlist *uidlist = *_uidlist; + + i_assert(!UIDLIST_IS_LOCKED(uidlist)); + + *_uidlist = NULL; + (void)maildir_uidlist_update(uidlist); + maildir_uidlist_close(uidlist); + + hash_table_destroy(&uidlist->files); + pool_unref(&uidlist->record_pool); + + array_free(&uidlist->records); + str_free(&uidlist->hdr_extensions); + i_free(uidlist->path); + i_free(uidlist); +} + +static int maildir_uid_cmp(struct maildir_uidlist_rec *const *rec1, + struct maildir_uidlist_rec *const *rec2) +{ + return (*rec1)->uid < (*rec2)->uid ? -1 : + (*rec1)->uid > (*rec2)->uid ? 1 : 0; +} + +static void ATTR_FORMAT(2, 3) +maildir_uidlist_set_corrupted(struct maildir_uidlist *uidlist, + const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + if (uidlist->retry_rewind) { + mailbox_set_critical(uidlist->box, + "Broken or unexpectedly changed file %s " + "line %u: %s - re-reading from beginning", + uidlist->path, uidlist->read_line_count, + t_strdup_vprintf(fmt, args)); + } else { + mailbox_set_critical(uidlist->box, "Broken file %s line %u: %s", + uidlist->path, uidlist->read_line_count, + t_strdup_vprintf(fmt, args)); + } + va_end(args); +} + +static void maildir_uidlist_update_hdr(struct maildir_uidlist *uidlist, + const struct stat *st) +{ + struct maildir_index_header *mhdr = uidlist->mhdr; + + if (mhdr->uidlist_mtime == 0 && uidlist->version != UIDLIST_VERSION) { + /* upgrading from older version. don't update the + uidlist times until it uses the new format */ + uidlist->recreate = TRUE; + return; + } + mhdr->uidlist_mtime = st->st_mtime; + mhdr->uidlist_mtime_nsecs = ST_MTIME_NSEC(*st); + mhdr->uidlist_size = st->st_size; +} + +static unsigned int +maildir_uidlist_records_array_delete(struct maildir_uidlist *uidlist, + struct maildir_uidlist_rec *rec) +{ + struct maildir_uidlist_rec *const *recs, *const *pos; + unsigned int idx, count; + + recs = array_get(&uidlist->records, &count); + if (!uidlist->unsorted) { + pos = array_bsearch(&uidlist->records, &rec, maildir_uid_cmp); + i_assert(pos != NULL); + idx = pos - recs; + } else { + for (idx = 0; idx < count; idx++) { + if (recs[idx]->uid == rec->uid) + break; + } + i_assert(idx != count); + } + array_delete(&uidlist->records, idx, 1); + return idx; +} + +static bool +maildir_uidlist_read_extended(struct maildir_uidlist *uidlist, + const char **line_p, + struct maildir_uidlist_rec *rec) +{ + const char *start, *line = *line_p; + buffer_t *buf; + + buf = t_buffer_create(128); + while (*line != '\0' && *line != ':') { + /* skip over an extension field */ + start = line; + while (*line != ' ' && *line != '\0') line++; + if (MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*start)) { + buffer_append(buf, start, line - start); + buffer_append_c(buf, '\0'); + } else { + maildir_uidlist_set_corrupted(uidlist, + "Invalid extension record, removing: %s", + t_strdup_until(start, line)); + uidlist->recreate = TRUE; + } + while (*line == ' ') line++; + } + + if (buf->used > 0) { + /* save the extensions */ + buffer_append_c(buf, '\0'); + rec->extensions = p_malloc(uidlist->record_pool, buf->used); + memcpy(rec->extensions, buf->data, buf->used); + } + + if (*line == ':') + line++; + if (*line == '\0') + return FALSE; + + *line_p = line; + return TRUE; +} + +static bool maildir_uidlist_next(struct maildir_uidlist *uidlist, + const char *line) +{ + struct maildir_uidlist_rec *rec, *old_rec, *const *recs; + unsigned int count; + uint32_t uid; + + uid = 0; + while (*line >= '0' && *line <= '9') { + uid = uid*10 + (*line - '0'); + line++; + } + + if (uid == 0 || *line != ' ') { + /* invalid file */ + maildir_uidlist_set_corrupted(uidlist, "Invalid data: %s", + line); + return FALSE; + } + if (uid <= uidlist->prev_read_uid) { + maildir_uidlist_set_corrupted(uidlist, + "UIDs not ordered (%u >= %u)", + uid, uidlist->prev_read_uid); + return FALSE; + } + if (uid >= (uint32_t)-1) { + maildir_uidlist_set_corrupted(uidlist, + "UID too high (%u)", uid); + return FALSE; + } + uidlist->prev_read_uid = uid; + + if (uid <= uidlist->last_seen_uid) { + /* we already have this */ + return TRUE; + } + uidlist->last_seen_uid = uid; + + if (uid >= uidlist->next_uid && uidlist->version == 1) { + maildir_uidlist_set_corrupted(uidlist, + "UID larger than next_uid (%u >= %u)", + uid, uidlist->next_uid); + return FALSE; + } + + rec = p_new(uidlist->record_pool, struct maildir_uidlist_rec, 1); + rec->uid = uid; + rec->flags = MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; + + while (*line == ' ') line++; + + if (uidlist->version == UIDLIST_VERSION) { + /* read extended fields */ + bool ret; + + T_BEGIN { + ret = maildir_uidlist_read_extended(uidlist, &line, + rec); + } T_END; + if (!ret) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid extended fields: %s", line); + return FALSE; + } + } + + if (strchr(line, '/') != NULL) { + maildir_uidlist_set_corrupted(uidlist, + "%s: Broken filename at line %u: %s", + uidlist->path, uidlist->read_line_count, line); + return FALSE; + } + + old_rec = hash_table_lookup(uidlist->files, line); + if (old_rec == NULL) { + /* no conflicts */ + } else if (old_rec->uid == uid) { + /* most likely this is a record we saved ourself, but couldn't + update last_seen_uid because uidlist wasn't refreshed while + it was locked. + + another possibility is a duplicate file record. currently + it would be a bug, but not that big of a deal. also perhaps + in future such duplicate lines could be used to update + extended fields. so just let it through anyway. + + we'll waste a bit of memory here by allocating the record + twice, but that's not really a problem. */ + rec->filename = old_rec->filename; + hash_table_update(uidlist->files, rec->filename, rec); + uidlist->unsorted = TRUE; + return TRUE; + } else { + /* This can happen if expunged file is moved back and the file + was appended to uidlist. */ + i_warning("%s: Duplicate file entry at line %u: " + "%s (uid %u -> %u)%s", + uidlist->path, uidlist->read_line_count, line, + old_rec->uid, uid, uidlist->retry_rewind ? + " - retrying by re-reading from beginning" : ""); + if (uidlist->retry_rewind) + return FALSE; + /* Delete the old UID */ + (void)maildir_uidlist_records_array_delete(uidlist, old_rec); + /* Replace the old record with this new one */ + *old_rec = *rec; + rec = old_rec; + uidlist->recreate = TRUE; + } + + recs = array_get(&uidlist->records, &count); + if (count > 0 && recs[count-1]->uid > uid) { + /* we most likely have some records in the array that we saved + ourself without refreshing uidlist */ + uidlist->unsorted = TRUE; + } + + rec->filename = p_strdup(uidlist->record_pool, line); + hash_table_update(uidlist->files, rec->filename, rec); + array_push_back(&uidlist->records, &rec); + return TRUE; +} + +static int +maildir_uidlist_read_v3_header(struct maildir_uidlist *uidlist, + const char *line, + unsigned int *uid_validity_r, + unsigned int *next_uid_r) +{ + char key; + + str_truncate(uidlist->hdr_extensions, 0); + while (*line != '\0') { + const char *value; + + key = *line; + value = ++line; + while (*line != '\0' && *line != ' ') line++; + value = t_strdup_until(value, line); + + switch (key) { + case MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY: + if (str_to_uint(value, uid_validity_r) < 0) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid mailbox UID_VALIDITY: %s", value); + return -1; + } + break; + case MAILDIR_UIDLIST_HDR_EXT_NEXT_UID: + if (str_to_uint(value, next_uid_r) < 0) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid mailbox NEXT_UID: %s", value); + return -1; + } + break; + case MAILDIR_UIDLIST_HDR_EXT_GUID: + if (guid_128_from_string(value, + uidlist->mailbox_guid) < 0) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid mailbox GUID: %s", value); + return -1; + } + uidlist->have_mailbox_guid = TRUE; + break; + default: + if (str_len(uidlist->hdr_extensions) > 0) + str_append_c(uidlist->hdr_extensions, ' '); + str_printfa(uidlist->hdr_extensions, + "%c%s", key, value); + break; + } + + while (*line == ' ') line++; + } + return 0; +} + +static int maildir_uidlist_read_header(struct maildir_uidlist *uidlist, + struct istream *input) +{ + unsigned int uid_validity = 0, next_uid = 0; + const char *line; + int ret; + + line = i_stream_read_next_line(input); + if (line == NULL) { + /* I/O error / empty file */ + return input->stream_errno == 0 ? 0 : -1; + } + uidlist->read_line_count = 1; + + if (*line < '0' || *line > '9' || line[1] != ' ') { + maildir_uidlist_set_corrupted(uidlist, + "Corrupted header (invalid version number)"); + return 0; + } + + uidlist->version = *line - '0'; + line += 2; + + switch (uidlist->version) { + case 1: + if (sscanf(line, "%u %u", &uid_validity, &next_uid) != 2) { + maildir_uidlist_set_corrupted(uidlist, + "Corrupted header (version 1)"); + return 0; + } + break; + case UIDLIST_VERSION: + T_BEGIN { + ret = maildir_uidlist_read_v3_header(uidlist, line, + &uid_validity, + &next_uid); + } T_END; + if (ret < 0) + return 0; + break; + default: + maildir_uidlist_set_corrupted(uidlist, "Unsupported version %u", + uidlist->version); + return 0; + } + + if (uid_validity == 0 || next_uid == 0) { + maildir_uidlist_set_corrupted(uidlist, + "Broken header (uidvalidity = %u, next_uid=%u)", + uid_validity, next_uid); + return 0; + } + + if (uid_validity == uidlist->uid_validity && + next_uid < uidlist->hdr_next_uid) { + maildir_uidlist_set_corrupted(uidlist, + "next_uid header was lowered (%u -> %u)", + uidlist->hdr_next_uid, next_uid); + return 0; + } + + uidlist->uid_validity = uid_validity; + uidlist->next_uid = next_uid; + uidlist->hdr_next_uid = next_uid; + return 1; +} + +static void maildir_uidlist_records_sort_by_uid(struct maildir_uidlist *uidlist) +{ + array_sort(&uidlist->records, maildir_uid_cmp); + uidlist->unsorted = FALSE; +} + +static int +maildir_uidlist_update_read(struct maildir_uidlist *uidlist, + bool *retry_r, bool try_retry) +{ + const char *line; + uint32_t orig_next_uid, orig_uid_validity; + struct istream *input; + struct stat st; + uoff_t last_read_offset; + int fd, ret; + bool readonly = FALSE; + + *retry_r = FALSE; + + if (uidlist->fd == -1) { + fd = nfs_safe_open(uidlist->path, O_RDWR); + if (fd == -1 && errno == EACCES) { + fd = nfs_safe_open(uidlist->path, O_RDONLY); + readonly = TRUE; + } + if (fd == -1) { + if (errno != ENOENT) { + mailbox_set_critical(uidlist->box, + "open(%s) failed: %m", uidlist->path); + return -1; + } + return 0; + } + last_read_offset = 0; + } else { + /* the file was updated */ + fd = uidlist->fd; + if (lseek(fd, 0, SEEK_SET) < 0) { + if (errno == ESTALE && try_retry) { + *retry_r = TRUE; + return -1; + } + mailbox_set_critical(uidlist->box, + "lseek(%s) failed: %m", uidlist->path); + return -1; + } + uidlist->fd = -1; + uidlist->fd_ino = 0; + last_read_offset = uidlist->last_read_offset; + uidlist->last_read_offset = 0; + } + + if (fstat(fd, &st) < 0) { + i_close_fd(&fd); + if (errno == ESTALE && try_retry) { + *retry_r = TRUE; + return -1; + } + mailbox_set_critical(uidlist->box, + "fstat(%s) failed: %m", uidlist->path); + return -1; + } + + if (uidlist->record_pool == NULL) { + uidlist->record_pool = + pool_alloconly_create(MEMPOOL_GROWING + "uidlist record_pool", + nearest_power(st.st_size - + st.st_size/8)); + } + + input = i_stream_create_fd(fd, SIZE_MAX); + i_stream_seek(input, last_read_offset); + + orig_uid_validity = uidlist->uid_validity; + orig_next_uid = uidlist->next_uid; + ret = input->v_offset != 0 ? 1 : + maildir_uidlist_read_header(uidlist, input); + if (ret > 0) { + uidlist->prev_read_uid = 0; + uidlist->change_counter++; + uidlist->retry_rewind = last_read_offset != 0 && try_retry; + + ret = 1; + while ((line = i_stream_read_next_line(input)) != NULL) { + uidlist->read_records_count++; + uidlist->read_line_count++; + if (!maildir_uidlist_next(uidlist, line)) { + if (!uidlist->retry_rewind) + ret = 0; + else { + ret = -1; + *retry_r = TRUE; + } + break; + } + } + uidlist->retry_rewind = FALSE; + if (input->stream_errno != 0) + ret = -1; + + if (uidlist->unsorted) { + uidlist->recreate_on_change = TRUE; + maildir_uidlist_records_sort_by_uid(uidlist); + } + if (uidlist->next_uid <= uidlist->prev_read_uid) + uidlist->next_uid = uidlist->prev_read_uid + 1; + if (ret > 0 && uidlist->uid_validity != orig_uid_validity && + orig_uid_validity != 0) { + uidlist->recreate = TRUE; + } else if (ret > 0 && uidlist->next_uid < orig_next_uid) { + mailbox_set_critical(uidlist->box, + "%s: next_uid was lowered (%u -> %u, hdr=%u)", + uidlist->path, orig_next_uid, + uidlist->next_uid, uidlist->hdr_next_uid); + uidlist->recreate = TRUE; + uidlist->next_uid = orig_next_uid; + } + } + + if (ret == 0) { + /* file is broken */ + i_unlink(uidlist->path); + } else if (ret > 0) { + /* success */ + if (readonly) + uidlist->recreate_on_change = TRUE; + uidlist->fd = fd; + uidlist->fd_dev = st.st_dev; + uidlist->fd_ino = st.st_ino; + uidlist->fd_size = st.st_size; + uidlist->last_read_offset = input->v_offset; + maildir_uidlist_update_hdr(uidlist, &st); + } else if (!*retry_r) { + /* I/O error */ + if (input->stream_errno == ESTALE && try_retry) + *retry_r = TRUE; + else { + mailbox_set_critical(uidlist->box, + "read(%s) failed: %s", uidlist->path, + i_stream_get_error(input)); + } + uidlist->last_read_offset = 0; + } + + i_stream_destroy(&input); + if (ret <= 0) { + if (close(fd) < 0) { + mailbox_set_critical(uidlist->box, + "close(%s) failed: %m", uidlist->path); + } + } + return ret; +} + +static int +maildir_uidlist_stat(struct maildir_uidlist *uidlist, struct stat *st_r) +{ + struct mail_storage *storage = uidlist->box->storage; + + if (storage->set->mail_nfs_storage) { + nfs_flush_file_handle_cache(uidlist->path); + nfs_flush_attr_cache_unlocked(uidlist->path); + } + if (nfs_safe_stat(uidlist->path, st_r) < 0) { + if (errno != ENOENT) { + mailbox_set_critical(uidlist->box, + "stat(%s) failed: %m", uidlist->path); + return -1; + } + return 0; + } + return 1; +} + +static int +maildir_uidlist_has_changed(struct maildir_uidlist *uidlist, bool *recreated_r) +{ + struct mail_storage *storage = uidlist->box->storage; + struct stat st; + int ret; + + *recreated_r = FALSE; + + if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0) + return -1; + if (ret == 0) { + *recreated_r = TRUE; + return 1; + } + + if (st.st_ino != uidlist->fd_ino || + !CMP_DEV_T(st.st_dev, uidlist->fd_dev)) { + /* file recreated */ + *recreated_r = TRUE; + return 1; + } + + if (storage->set->mail_nfs_storage) { + /* NFS: either the file hasn't been changed, or it has already + been deleted and the inodes just happen to be the same. + check if the fd is still valid. */ + if (fstat(uidlist->fd, &st) < 0) { + if (errno == ESTALE) { + *recreated_r = TRUE; + return 1; + } + mailbox_set_critical(uidlist->box, + "fstat(%s) failed: %m", uidlist->path); + return -1; + } + } + + if (st.st_size != uidlist->fd_size) { + /* file modified but not recreated */ + return 1; + } else { + /* unchanged */ + return 0; + } +} + +static int maildir_uidlist_open_latest(struct maildir_uidlist *uidlist) +{ + bool recreated; + int ret; + + if (uidlist->fd != -1) { + ret = maildir_uidlist_has_changed(uidlist, &recreated); + if (ret <= 0) { + if (UIDLIST_IS_LOCKED(uidlist)) + uidlist->locked_refresh = TRUE; + return ret < 0 ? -1 : 1; + } + + if (!recreated) + return 0; + maildir_uidlist_reset(uidlist); + } + + uidlist->fd = nfs_safe_open(uidlist->path, O_RDWR); + if (uidlist->fd == -1 && errno == EACCES) { + uidlist->fd = nfs_safe_open(uidlist->path, O_RDONLY); + uidlist->recreate_on_change = TRUE; + } + if (uidlist->fd == -1 && errno != ENOENT) { + mailbox_set_critical(uidlist->box, + "open(%s) failed: %m", uidlist->path); + return -1; + } + return 0; +} + +int maildir_uidlist_refresh(struct maildir_uidlist *uidlist) +{ + unsigned int i; + bool retry; + int ret; + + if (maildir_uidlist_open_latest(uidlist) < 0) + return -1; + + for (i = 0; ; i++) { + ret = maildir_uidlist_update_read(uidlist, &retry, + i < UIDLIST_ESTALE_RETRY_COUNT); + if (!retry) + break; + /* ESTALE - try reopening and rereading */ + maildir_uidlist_close(uidlist); + } + if (ret >= 0) { + uidlist->initial_read = TRUE; + uidlist->initial_hdr_read = TRUE; + if (UIDLIST_IS_LOCKED(uidlist)) + uidlist->locked_refresh = TRUE; + if (!uidlist->have_mailbox_guid) { + uidlist->recreate = TRUE; + (void)maildir_uidlist_update(uidlist); + } + } + return ret; +} + +int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist) +{ + const struct maildir_index_header *mhdr = uidlist->mhdr; + struct mail_index *index = uidlist->box->index; + struct mail_index_view *view; + const struct mail_index_header *hdr; + struct stat st; + int ret; + + i_assert(UIDLIST_IS_LOCKED(uidlist)); + + if (uidlist->fd != -1) + return maildir_uidlist_refresh(uidlist); + + if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0) + return ret; + + if (ret > 0 && st.st_size == mhdr->uidlist_size && + st.st_mtime == (time_t)mhdr->uidlist_mtime && + ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), mhdr->uidlist_mtime_nsecs) && + (!mail_index_is_in_memory(index) || st.st_mtime < ioloop_time-1)) { + /* index is up-to-date. look up the uidvalidity and next-uid + from it. we'll need to create a new view temporarily to + make sure we get the latest values. */ + view = mail_index_view_open(index); + hdr = mail_index_get_header(view); + uidlist->uid_validity = hdr->uid_validity; + uidlist->next_uid = hdr->next_uid; + uidlist->initial_hdr_read = TRUE; + mail_index_view_close(&view); + + if (UIDLIST_IS_LOCKED(uidlist)) + uidlist->locked_refresh = TRUE; + return 1; + } else { + return maildir_uidlist_refresh(uidlist); + } +} + +static int +maildir_uid_bsearch_cmp(const uint32_t *uidp, + struct maildir_uidlist_rec *const *recp) +{ + return *uidp < (*recp)->uid ? -1 : + *uidp > (*recp)->uid ? 1 : 0; +} + +static int +maildir_uidlist_lookup_rec(struct maildir_uidlist *uidlist, uint32_t uid, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist_rec *const *pos; + + if (!uidlist->initial_read) { + /* first time we need to read uidlist */ + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } + + pos = array_bsearch(&uidlist->records, &uid, + maildir_uid_bsearch_cmp); + if (pos == NULL) { + *rec_r = NULL; + return 0; + } + *rec_r = *pos; + return 1; +} + +int maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r) +{ + struct maildir_uidlist_rec *rec; + int ret; + + if ((ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec)) <= 0) + return ret; + + *flags_r = rec->flags; + *fname_r = rec->filename; + return 1; +} + +const char * +maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key) +{ + struct maildir_uidlist_rec *rec; + const unsigned char *p; + int ret; + + ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec); + if (ret <= 0 || rec->extensions == NULL) + return NULL; + + p = rec->extensions; + while (*p != '\0') { + /* <key><value>\0 */ + if (*p == (unsigned char)key) + return (const char *)p + 1; + + p += strlen((const char *)p) + 1; + } + return NULL; +} + +uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist) +{ + return uidlist->uid_validity; +} + +uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist) +{ + return !uidlist->initial_hdr_read ? 0 : uidlist->next_uid; +} + +int maildir_uidlist_get_mailbox_guid(struct maildir_uidlist *uidlist, + guid_128_t mailbox_guid) +{ + if (!uidlist->initial_hdr_read) { + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } + if (!uidlist->have_mailbox_guid) { + uidlist->recreate = TRUE; + if (maildir_uidlist_update(uidlist) < 0) + return -1; + } + memcpy(mailbox_guid, uidlist->mailbox_guid, GUID_128_SIZE); + return 0; +} + +void maildir_uidlist_set_mailbox_guid(struct maildir_uidlist *uidlist, + const guid_128_t mailbox_guid) +{ + if (memcmp(uidlist->mailbox_guid, mailbox_guid, + sizeof(uidlist->mailbox_guid)) != 0) { + memcpy(uidlist->mailbox_guid, mailbox_guid, + sizeof(uidlist->mailbox_guid)); + uidlist->recreate = TRUE; + } +} + +void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist, + uint32_t uid_validity) +{ + i_assert(uid_validity != 0); + + if (uid_validity != uidlist->uid_validity) { + uidlist->uid_validity = uid_validity; + uidlist->recreate = TRUE; + } +} + +void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist, + uint32_t next_uid, bool force) +{ + if (uidlist->next_uid < next_uid || force) { + i_assert(next_uid != 0); + uidlist->next_uid = next_uid; + uidlist->recreate = TRUE; + } +} + +static void +maildir_uidlist_rec_set_ext(struct maildir_uidlist_rec *rec, pool_t pool, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + const unsigned char *p; + buffer_t *buf; + size_t len; + + /* copy existing extensions, except for the one we're updating */ + buf = t_buffer_create(128); + if (rec->extensions != NULL) { + p = rec->extensions; + while (*p != '\0') { + /* <key><value>\0 */ + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*p)); + + len = strlen((const char *)p) + 1; + if (*p != (unsigned char)key) + buffer_append(buf, p, len); + p += len; + } + } + if (value != NULL) { + buffer_append_c(buf, key); + buffer_append(buf, value, strlen(value) + 1); + } + buffer_append_c(buf, '\0'); + + rec->extensions = p_malloc(pool, buf->used); + memcpy(rec->extensions, buf->data, buf->used); +} + +static void ATTR_NULL(4) +maildir_uidlist_set_ext_internal(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + struct maildir_uidlist_rec *rec; + int ret; + + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(key)); + + ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec); + if (ret <= 0) { + if (ret < 0) + return; + + /* maybe it's a new message */ + if (maildir_uidlist_refresh(uidlist) < 0) + return; + if (maildir_uidlist_lookup_rec(uidlist, uid, &rec) <= 0) { + /* message is already expunged, ignore */ + return; + } + } + + T_BEGIN { + maildir_uidlist_rec_set_ext(rec, uidlist->record_pool, + key, value); + } T_END; + + if (rec->uid != (uint32_t)-1) { + /* message already exists in uidlist, need to recreate it */ + uidlist->recreate = TRUE; + } +} + +void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + maildir_uidlist_set_ext_internal(uidlist, uid, key, value); +} + +void maildir_uidlist_unset_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key) +{ + maildir_uidlist_set_ext_internal(uidlist, uid, key, NULL); +} + +static void +maildir_uidlist_generate_uid_validity(struct maildir_uidlist *uidlist) +{ + const struct mail_index_header *hdr; + + if (uidlist->box->opened) { + hdr = mail_index_get_header(uidlist->box->view); + if (hdr->uid_validity != 0) { + uidlist->uid_validity = hdr->uid_validity; + return; + } + } + uidlist->uid_validity = + maildir_get_uidvalidity_next(uidlist->box->list); +} + +static int maildir_uidlist_write_fd(struct maildir_uidlist *uidlist, int fd, + const char *path, unsigned int first_idx, + uoff_t *file_size_r) +{ + struct mail_storage *storage = uidlist->box->storage; + struct maildir_uidlist_iter_ctx *iter; + struct ostream *output; + struct maildir_uidlist_rec *rec; + string_t *str; + const unsigned char *p; + const char *strp; + size_t len; + + i_assert(fd != -1); + + output = o_stream_create_fd_file(fd, UOFF_T_MAX, FALSE); + o_stream_cork(output); + str = t_str_new(512); + + if (output->offset == 0) { + i_assert(first_idx == 0); + uidlist->version = UIDLIST_VERSION; + + if (uidlist->uid_validity == 0) + maildir_uidlist_generate_uid_validity(uidlist); + if (!uidlist->have_mailbox_guid) + guid_128_generate(uidlist->mailbox_guid); + + i_assert(uidlist->next_uid > 0); + str_printfa(str, "%u %c%u %c%u %c%s", uidlist->version, + MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY, + uidlist->uid_validity, + MAILDIR_UIDLIST_HDR_EXT_NEXT_UID, + uidlist->next_uid, + MAILDIR_UIDLIST_HDR_EXT_GUID, + guid_128_to_string(uidlist->mailbox_guid)); + if (str_len(uidlist->hdr_extensions) > 0) { + str_append_c(str, ' '); + str_append_str(str, uidlist->hdr_extensions); + } + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); + } + + iter = maildir_uidlist_iter_init(uidlist); + i_assert(first_idx <= array_count(&uidlist->records)); + iter->next += first_idx; + + while (maildir_uidlist_iter_next_rec(iter, &rec)) { + uidlist->read_records_count++; + str_truncate(str, 0); + str_printfa(str, "%u", rec->uid); + if (rec->extensions != NULL) { + for (p = rec->extensions; *p != '\0'; ) { + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*p)); + len = strlen((const char *)p); + str_append_c(str, ' '); + str_append_data(str, p, len); + p += len + 1; + } + } + str_append(str, " :"); + strp = strchr(rec->filename, *MAILDIR_INFO_SEP_S); + if (strp == NULL) + str_append(str, rec->filename); + else + str_append_data(str, rec->filename, strp - rec->filename); + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); + } + maildir_uidlist_iter_deinit(&iter); + + if (o_stream_finish(output) < 0) { + mailbox_set_critical(uidlist->box, "write(%s) failed: %s", + path, o_stream_get_error(output)); + o_stream_unref(&output); + return -1; + } + + *file_size_r = output->offset; + o_stream_unref(&output); + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync(fd) < 0) { + mailbox_set_critical(uidlist->box, + "fdatasync(%s) failed: %m", path); + return -1; + } + } + return 0; +} + +static void +maildir_uidlist_records_drop_expunges(struct maildir_uidlist *uidlist) +{ + struct mail_index_view *view; + struct maildir_uidlist_rec *const *recs; + ARRAY_TYPE(maildir_uidlist_rec_p) new_records; + const struct mail_index_header *hdr; + const struct mail_index_record *rec; + unsigned int i, count; + uint32_t seq; + + /* we could get here when opening and locking mailbox, + before index files have been opened. */ + if (!uidlist->box->opened) + return; + + mail_index_refresh(uidlist->box->index); + view = mail_index_view_open(uidlist->box->index); + count = array_count(&uidlist->records); + hdr = mail_index_get_header(view); + if (count * UIDLIST_COMPRESS_PERCENTAGE / 100 <= hdr->messages_count) { + /* too much trouble to be worth it */ + mail_index_view_close(&view); + return; + } + + i_array_init(&new_records, hdr->messages_count + 64); + recs = array_get(&uidlist->records, &count); + for (i = 0, seq = 1; i < count && seq <= hdr->messages_count; ) { + rec = mail_index_lookup(view, seq); + if (recs[i]->uid < rec->uid) { + /* expunged entry */ + hash_table_remove(uidlist->files, recs[i]->filename); + i++; + } else if (recs[i]->uid > rec->uid) { + /* index isn't up to date. we're probably just + syncing it here. ignore this entry. */ + seq++; + } else { + array_push_back(&new_records, &recs[i]); + seq++; i++; + } + } + + /* drop messages expunged at the end of index */ + while (i < count && recs[i]->uid < hdr->next_uid) { + hash_table_remove(uidlist->files, recs[i]->filename); + i++; + } + /* view might not be completely up-to-date, so preserve any + messages left */ + for (; i < count; i++) + array_push_back(&new_records, &recs[i]); + + array_free(&uidlist->records); + uidlist->records = new_records; + + mail_index_view_close(&view); +} + +static int maildir_uidlist_recreate(struct maildir_uidlist *uidlist) +{ + struct mailbox *box = uidlist->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *control_dir, *temp_path; + struct stat st; + mode_t old_mask; + uoff_t file_size; + int i, fd, ret; + + i_assert(uidlist->initial_read); + + maildir_uidlist_records_drop_expunges(uidlist); + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, + &control_dir) <= 0) + i_unreached(); + temp_path = t_strconcat(control_dir, + "/" MAILDIR_UIDLIST_NAME ".tmp", NULL); + + for (i = 0;; i++) { + old_mask = umask(0777 & ~perm->file_create_mode); + fd = open(temp_path, O_RDWR | O_CREAT | O_TRUNC, 0777); + umask(old_mask); + if (fd != -1) + break; + + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + mailbox_set_critical(box, + "open(%s, O_CREAT) failed: %m", temp_path); + return -1; + } + /* the control dir doesn't exist. create it unless the whole + mailbox was just deleted. */ + if (!maildir_set_deleted(uidlist->box)) + return -1; + } + + if (perm->file_create_gid != (gid_t)-1 && + fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", temp_path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, + "fchown(%s) failed: %m", temp_path); + } + } + + uidlist->read_records_count = 0; + ret = maildir_uidlist_write_fd(uidlist, fd, temp_path, 0, &file_size); + if (ret == 0) { + if (rename(temp_path, uidlist->path) < 0) { + mailbox_set_critical(box, + "rename(%s, %s) failed: %m", + temp_path, uidlist->path); + ret = -1; + } + } + + if (ret < 0) + i_unlink(temp_path); + else if (fstat(fd, &st) < 0) { + mailbox_set_critical(box, + "fstat(%s) failed: %m", temp_path); + ret = -1; + } else if (file_size != (uoff_t)st.st_size) { + i_assert(!file_dotlock_is_locked(uidlist->dotlock)); + mailbox_set_critical(box, + "Maildir uidlist dotlock overridden: %s", + uidlist->path); + ret = -1; + } else { + maildir_uidlist_close(uidlist); + uidlist->fd = fd; + uidlist->fd_dev = st.st_dev; + uidlist->fd_ino = st.st_ino; + uidlist->fd_size = st.st_size; + uidlist->last_read_offset = st.st_size; + uidlist->recreate = FALSE; + uidlist->recreate_on_change = FALSE; + uidlist->have_mailbox_guid = TRUE; + maildir_uidlist_update_hdr(uidlist, &st); + } + if (ret < 0) + i_close_fd(&fd); + return ret; +} + +int maildir_uidlist_update(struct maildir_uidlist *uidlist) +{ + int ret; + + if (!uidlist->recreate) + return 0; + + if (maildir_uidlist_lock(uidlist) <= 0) + return -1; + ret = maildir_uidlist_recreate(uidlist); + maildir_uidlist_unlock(uidlist); + return ret; +} + +static bool maildir_uidlist_want_compress(struct maildir_uidlist_sync_ctx *ctx) +{ + unsigned int min_rewrite_count; + + if (!ctx->uidlist->locked_refresh) + return FALSE; + if (ctx->uidlist->recreate) + return TRUE; + + min_rewrite_count = + (ctx->uidlist->read_records_count + ctx->new_files_count) * + UIDLIST_COMPRESS_PERCENTAGE / 100; + return min_rewrite_count >= array_count(&ctx->uidlist->records); +} + +static bool maildir_uidlist_want_recreate(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + + if (!uidlist->locked_refresh || !uidlist->initial_read) + return FALSE; + + if (ctx->finish_change_counter != uidlist->change_counter) + return TRUE; + if (uidlist->fd == -1 || uidlist->version != UIDLIST_VERSION || + !uidlist->have_mailbox_guid) + return TRUE; + return maildir_uidlist_want_compress(ctx); +} + +static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct stat st; + uoff_t file_size; + + if (maildir_uidlist_want_recreate(ctx) || uidlist->recreate_on_change) + return maildir_uidlist_recreate(uidlist); + + if (!uidlist->locked_refresh || uidlist->fd == -1) { + /* make sure we have the latest file (e.g. NOREFRESH used) */ + i_assert(uidlist->initial_hdr_read); + if (maildir_uidlist_open_latest(uidlist) < 0) + return -1; + if (uidlist->recreate_on_change) + return maildir_uidlist_recreate(uidlist); + } + i_assert(ctx->first_unwritten_pos != UINT_MAX); + + if (lseek(uidlist->fd, 0, SEEK_END) < 0) { + mailbox_set_critical(uidlist->box, + "lseek(%s) failed: %m", uidlist->path); + return -1; + } + + if (maildir_uidlist_write_fd(uidlist, uidlist->fd, uidlist->path, + ctx->first_unwritten_pos, &file_size) < 0) + return -1; + + if (fstat(uidlist->fd, &st) < 0) { + mailbox_set_critical(uidlist->box, + "fstat(%s) failed: %m", uidlist->path); + return -1; + } + if ((uoff_t)st.st_size != file_size) { + i_warning("%s: file size changed unexpectedly after write", + uidlist->path); + } else if (uidlist->locked_refresh) { + uidlist->fd_size = st.st_size; + uidlist->last_read_offset = st.st_size; + maildir_uidlist_update_hdr(uidlist, &st); + } + return 0; +} + +static void maildir_uidlist_mark_all(struct maildir_uidlist *uidlist, + bool nonsynced) +{ + struct maildir_uidlist_rec **recs; + unsigned int i, count; + + recs = array_get_modifiable(&uidlist->records, &count); + if (nonsynced) { + for (i = 0; i < count; i++) + recs[i]->flags |= MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; + } else { + for (i = 0; i < count; i++) + recs[i]->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + } +} + +static int maildir_uidlist_sync_lock(struct maildir_uidlist *uidlist, + enum maildir_uidlist_sync_flags sync_flags, + bool *locked_r) +{ + bool nonblock, refresh; + int ret; + + *locked_r = FALSE; + + if ((sync_flags & MAILDIR_UIDLIST_SYNC_NOLOCK) != 0) { + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + return 1; + } + + nonblock = (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0; + refresh = (sync_flags & MAILDIR_UIDLIST_SYNC_NOREFRESH) == 0; + + ret = maildir_uidlist_lock_timeout(uidlist, nonblock, refresh, refresh); + if (ret <= 0) { + if (ret < 0 || !nonblock) + return ret; + + /* couldn't lock it */ + if ((sync_flags & MAILDIR_UIDLIST_SYNC_FORCE) == 0) + return 0; + /* forcing the sync anyway */ + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } else { + *locked_r = TRUE; + } + return 1; +} + +void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist) +{ + maildir_uidlist_mark_all(uidlist, TRUE); +} + +int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, + enum maildir_uidlist_sync_flags sync_flags, + struct maildir_uidlist_sync_ctx **sync_ctx_r) +{ + struct maildir_uidlist_sync_ctx *ctx; + bool locked; + int ret; + + ret = maildir_uidlist_sync_lock(uidlist, sync_flags, &locked); + if (ret <= 0) + return ret; + + *sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1); + ctx->uidlist = uidlist; + ctx->sync_flags = sync_flags; + ctx->partial = !locked || + (sync_flags & MAILDIR_UIDLIST_SYNC_PARTIAL) != 0; + ctx->locked = locked; + ctx->first_unwritten_pos = UINT_MAX; + ctx->first_new_pos = UINT_MAX; + + if (ctx->partial) { + if ((sync_flags & MAILDIR_UIDLIST_SYNC_KEEP_STATE) == 0) { + /* initially mark all nonsynced */ + maildir_uidlist_mark_all(uidlist, TRUE); + } + return 1; + } + i_assert(uidlist->locked_refresh); + + ctx->record_pool = pool_alloconly_create(MEMPOOL_GROWING + "maildir_uidlist_sync", 16384); + hash_table_create(&ctx->files, ctx->record_pool, 4096, + maildir_filename_base_hash, + maildir_filename_base_cmp); + + i_array_init(&ctx->records, array_count(&uidlist->records)); + return 1; +} + +static int +maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, uint32_t uid, + enum maildir_uidlist_rec_flag flags, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct maildir_uidlist_rec *rec, *const *recs; + unsigned int count; + + /* we'll update uidlist directly */ + rec = hash_table_lookup(uidlist->files, filename); + if (rec == NULL) { + /* doesn't exist in uidlist */ + if (!ctx->locked) { + /* we can't add it, so just ignore it */ + return 1; + } + if (ctx->first_new_pos == UINT_MAX) + ctx->first_new_pos = array_count(&uidlist->records); + ctx->new_files_count++; + ctx->changed = TRUE; + + if (uidlist->record_pool == NULL) { + uidlist->record_pool = + pool_alloconly_create(MEMPOOL_GROWING + "uidlist record_pool", + 1024); + } + + rec = p_new(uidlist->record_pool, + struct maildir_uidlist_rec, 1); + rec->uid = (uint32_t)-1; + rec->filename = p_strdup(uidlist->record_pool, filename); + array_push_back(&uidlist->records, &rec); + uidlist->change_counter++; + + hash_table_insert(uidlist->files, rec->filename, rec); + } else if (strcmp(rec->filename, filename) != 0) { + rec->filename = p_strdup(uidlist->record_pool, filename); + } + if (uid != 0) { + if (rec->uid != uid && rec->uid != (uint32_t)-1) { + mailbox_set_critical(uidlist->box, + "Maildir: %s changed UID %u -> %u", + filename, rec->uid, uid); + return -1; + } + rec->uid = uid; + if (uidlist->next_uid <= uid) + uidlist->next_uid = uid + 1; + else { + recs = array_get(&uidlist->records, &count); + if (count > 1 && uid < recs[count-1]->uid) + uidlist->unsorted = TRUE; + } + } + + rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR); + rec->flags = (rec->flags | flags) & + ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + + ctx->finished = FALSE; + *rec_r = rec; + return 1; +} + +static unsigned char *ext_dup(pool_t pool, const unsigned char *extensions) +{ + unsigned char *ret; + + if (extensions == NULL) + return NULL; + + T_BEGIN { + unsigned int len; + + for (len = 0; extensions[len] != '\0'; len++) { + while (extensions[len] != '\0') len++; + } + ret = p_malloc(pool, len + 1); + memcpy(ret, extensions, len); + } T_END; + return ret; +} + +int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, + enum maildir_uidlist_rec_flag flags) +{ + struct maildir_uidlist_rec *rec; + + return maildir_uidlist_sync_next_uid(ctx, filename, 0, flags, &rec); +} + +int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, uint32_t uid, + enum maildir_uidlist_rec_flag flags, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct maildir_uidlist_rec *rec, *old_rec; + const char *p; + + *rec_r = NULL; + + if (ctx->failed) + return -1; + for (p = filename; *p != '\0'; p++) { + if (*p == 13 || *p == 10) { + i_warning("Maildir %s: Ignoring a file with #0x%x: %s", + mailbox_get_path(uidlist->box), *p, filename); + return 1; + } + } + + if (ctx->partial) { + return maildir_uidlist_sync_next_partial(ctx, filename, + uid, flags, rec_r); + } + + rec = hash_table_lookup(ctx->files, filename); + if (rec != NULL) { + if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | + MAILDIR_UIDLIST_REC_FLAG_MOVED)) == 0) { + /* possibly duplicate */ + return 0; + } + + /* probably was in new/ and now we're seeing it in cur/. + remove new/moved flags so if this happens again we'll know + to check for duplicates. */ + rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | + MAILDIR_UIDLIST_REC_FLAG_MOVED); + if (strcmp(rec->filename, filename) != 0) + rec->filename = p_strdup(ctx->record_pool, filename); + } else { + old_rec = hash_table_lookup(uidlist->files, filename); + i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist)); + + rec = p_new(ctx->record_pool, struct maildir_uidlist_rec, 1); + + if (old_rec != NULL) { + *rec = *old_rec; + rec->extensions = + ext_dup(ctx->record_pool, rec->extensions); + } else { + rec->uid = (uint32_t)-1; + ctx->new_files_count++; + ctx->changed = TRUE; + /* didn't exist in uidlist, it's recent */ + flags |= MAILDIR_UIDLIST_REC_FLAG_RECENT; + } + rec->filename = p_strdup(ctx->record_pool, filename); + hash_table_insert(ctx->files, rec->filename, rec); + + array_push_back(&ctx->records, &rec); + } + if (uid != 0) { + rec->uid = uid; + if (uidlist->next_uid <= uid) + uidlist->next_uid = uid + 1; + } + + rec->flags = (rec->flags | flags) & ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + *rec_r = rec; + return 1; +} + +void maildir_uidlist_sync_remove(struct maildir_uidlist_sync_ctx *ctx, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + unsigned int idx; + + i_assert(ctx->partial); + i_assert(ctx->uidlist->locked_refresh); + + rec = hash_table_lookup(ctx->uidlist->files, filename); + i_assert(rec != NULL); + i_assert(rec->uid != (uint32_t)-1); + + hash_table_remove(ctx->uidlist->files, filename); + idx = maildir_uidlist_records_array_delete(ctx->uidlist, rec); + + if (ctx->first_unwritten_pos != UINT_MAX) { + i_assert(ctx->first_unwritten_pos > idx); + ctx->first_unwritten_pos--; + } + if (ctx->first_new_pos != UINT_MAX) { + i_assert(ctx->first_new_pos > idx); + ctx->first_new_pos--; + } + + ctx->changed = TRUE; + ctx->uidlist->recreate = TRUE; +} + +void maildir_uidlist_sync_set_ext(struct maildir_uidlist_sync_ctx *ctx, + struct maildir_uidlist_rec *rec, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + pool_t pool = ctx->partial ? + ctx->uidlist->record_pool : ctx->record_pool; + + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(key)); + + maildir_uidlist_rec_set_ext(rec, pool, key, value); +} + +const char * +maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(ctx->files, filename); + return rec == NULL ? NULL : rec->filename; +} + +bool maildir_uidlist_get_uid(struct maildir_uidlist *uidlist, + const char *filename, uint32_t *uid_r) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + if (rec == NULL) + return FALSE; + + *uid_r = rec->uid; + return TRUE; +} + +void maildir_uidlist_update_fname(struct maildir_uidlist *uidlist, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + if (rec == NULL) + return; + + rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + if (strcmp(rec->filename, filename) != 0) + rec->filename = p_strdup(uidlist->record_pool, filename); +} + +const char * +maildir_uidlist_get_full_filename(struct maildir_uidlist *uidlist, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + return rec == NULL ? NULL : rec->filename; +} + +static int maildir_assign_uid_cmp(const void *p1, const void *p2) +{ + const struct maildir_uidlist_rec *const *rec1 = p1, *const *rec2 = p2; + + if ((*rec1)->uid != (*rec2)->uid) { + if ((*rec1)->uid < (*rec2)->uid) + return -1; + else + return 1; + } + return maildir_filename_sort_cmp((*rec1)->filename, (*rec2)->filename); +} + +static void maildir_uidlist_assign_uids(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist_rec **recs; + unsigned int dest, count; + + i_assert(UIDLIST_IS_LOCKED(ctx->uidlist)); + i_assert(ctx->first_new_pos != UINT_MAX); + + if (ctx->first_unwritten_pos == UINT_MAX) + ctx->first_unwritten_pos = ctx->first_new_pos; + + /* sort new files and assign UIDs for them */ + recs = array_get_modifiable(&ctx->uidlist->records, &count); + qsort(recs + ctx->first_new_pos, count - ctx->first_new_pos, + sizeof(*recs), maildir_assign_uid_cmp); + + for (dest = ctx->first_new_pos; dest < count; dest++) { + if (recs[dest]->uid == (uint32_t)-1) + break; + } + + for (; dest < count; dest++) { + i_assert(recs[dest]->uid == (uint32_t)-1); + i_assert(ctx->uidlist->next_uid < (uint32_t)-1); + recs[dest]->uid = ctx->uidlist->next_uid++; + recs[dest]->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_MOVED); + } + + if (ctx->uidlist->locked_refresh && ctx->uidlist->initial_read) + ctx->uidlist->last_seen_uid = ctx->uidlist->next_uid-1; + + ctx->new_files_count = 0; + ctx->first_new_pos = UINT_MAX; + ctx->uidlist->change_counter++; + ctx->finish_change_counter = ctx->uidlist->change_counter; +} + +static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + + /* buffer is unsorted, sort it by UID */ + array_sort(&ctx->records, maildir_uid_cmp); + + array_free(&uidlist->records); + uidlist->records = ctx->records; + ctx->records.arr.buffer = NULL; + i_assert(array_is_created(&uidlist->records)); + + hash_table_destroy(&uidlist->files); + uidlist->files = ctx->files; + i_zero(&ctx->files); + + pool_unref(&uidlist->record_pool); + uidlist->record_pool = ctx->record_pool; + ctx->record_pool = NULL; + + if (ctx->new_files_count != 0) { + ctx->first_new_pos = array_count(&uidlist->records) - + ctx->new_files_count; + maildir_uidlist_assign_uids(ctx); + } else { + ctx->uidlist->change_counter++; + } +} + +void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx) +{ + ctx->uidlist->recreate = TRUE; +} + +void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx) +{ + if (!ctx->partial) { + if (!ctx->failed) + maildir_uidlist_swap(ctx); + } else { + if (ctx->new_files_count != 0 && !ctx->failed) { + i_assert(ctx->changed); + i_assert(ctx->locked); + maildir_uidlist_assign_uids(ctx); + } + } + + ctx->finished = TRUE; + + /* mbox=NULL means we're coming from dbox rebuilding code. + the dbox is already locked, so allow uidlist recreation */ + i_assert(ctx->locked || !ctx->changed); + if ((ctx->changed || maildir_uidlist_want_compress(ctx)) && + !ctx->failed && ctx->locked) { + T_BEGIN { + if (maildir_uidlist_sync_update(ctx) < 0) { + /* we couldn't write everything we wanted. make + sure we don't continue using those UIDs */ + maildir_uidlist_reset(ctx->uidlist); + ctx->failed = TRUE; + } + } T_END; + } +} + +int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **_ctx, + bool success) +{ + struct maildir_uidlist_sync_ctx *ctx = *_ctx; + int ret; + + *_ctx = NULL; + + if (!success) + ctx->failed = TRUE; + ret = ctx->failed ? -1 : 0; + + if (!ctx->finished) + maildir_uidlist_sync_finish(ctx); + if (ctx->partial) + maildir_uidlist_mark_all(ctx->uidlist, FALSE); + if (ctx->locked) + maildir_uidlist_unlock(ctx->uidlist); + + hash_table_destroy(&ctx->files); + pool_unref(&ctx->record_pool); + if (array_is_created(&ctx->records)) + array_free(&ctx->records); + i_free(ctx); + return ret; +} + +void maildir_uidlist_add_flags(struct maildir_uidlist *uidlist, + const char *filename, + enum maildir_uidlist_rec_flag flags) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + i_assert(rec != NULL); + + rec->flags |= flags; +} + +struct maildir_uidlist_iter_ctx * +maildir_uidlist_iter_init(struct maildir_uidlist *uidlist) +{ + struct maildir_uidlist_iter_ctx *ctx; + unsigned int count; + + ctx = i_new(struct maildir_uidlist_iter_ctx, 1); + ctx->uidlist = uidlist; + ctx->next = array_get(&uidlist->records, &count); + ctx->end = ctx->next + count; + ctx->change_counter = ctx->uidlist->change_counter; + return ctx; +} + +static void +maildir_uidlist_iter_update_idx(struct maildir_uidlist_iter_ctx *ctx) +{ + unsigned int old_rev_idx, idx, count; + + old_rev_idx = ctx->end - ctx->next; + ctx->next = array_get(&ctx->uidlist->records, &count); + ctx->end = ctx->next + count; + + idx = old_rev_idx >= count ? 0 : + count - old_rev_idx; + while (idx < count && ctx->next[idx]->uid <= ctx->prev_uid) + idx++; + while (idx > 0 && ctx->next[idx-1]->uid > ctx->prev_uid) + idx--; + + ctx->next += idx; +} + +static bool maildir_uidlist_iter_next_rec(struct maildir_uidlist_iter_ctx *ctx, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist_rec *rec; + + if (ctx->change_counter != ctx->uidlist->change_counter) + maildir_uidlist_iter_update_idx(ctx); + + if (ctx->next == ctx->end) + return FALSE; + + rec = *ctx->next; + i_assert(rec->uid != (uint32_t)-1); + + ctx->prev_uid = rec->uid; + ctx->next++; + + *rec_r = rec; + return TRUE; +} + +bool maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, + uint32_t *uid_r, + enum maildir_uidlist_rec_flag *flags_r, + const char **filename_r) +{ + struct maildir_uidlist_rec *rec; + + if (!maildir_uidlist_iter_next_rec(ctx, &rec)) + return FALSE; + + *uid_r = rec->uid; + *flags_r = rec->flags; + *filename_r = rec->filename; + return TRUE; +} + +void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx **_ctx) +{ + i_free(*_ctx); + *_ctx = NULL; +} diff --git a/src/lib-storage/index/maildir/maildir-uidlist.h b/src/lib-storage/index/maildir/maildir-uidlist.h new file mode 100644 index 0000000..8b45723 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-uidlist.h @@ -0,0 +1,161 @@ +#ifndef MAILDIR_UIDLIST_H +#define MAILDIR_UIDLIST_H + +#include "mail-storage.h" + +#define MAILDIR_UIDLIST_NAME "dovecot-uidlist" +/* how many seconds to wait before overriding uidlist.lock */ +#define MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT (60*2) + +struct maildir_mailbox; +struct maildir_uidlist; +struct maildir_uidlist_sync_ctx; +struct maildir_uidlist_rec; + +enum maildir_uidlist_sync_flags { + MAILDIR_UIDLIST_SYNC_PARTIAL = 0x01, + MAILDIR_UIDLIST_SYNC_KEEP_STATE = 0x02, + MAILDIR_UIDLIST_SYNC_FORCE = 0x04, + MAILDIR_UIDLIST_SYNC_TRYLOCK = 0x08, + MAILDIR_UIDLIST_SYNC_NOREFRESH = 0x10, + MAILDIR_UIDLIST_SYNC_NOLOCK = 0x20 +}; + +enum maildir_uidlist_rec_flag { + MAILDIR_UIDLIST_REC_FLAG_NEW_DIR = 0x01, + MAILDIR_UIDLIST_REC_FLAG_MOVED = 0x02, + MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04, + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED = 0x08, + MAILDIR_UIDLIST_REC_FLAG_RACING = 0x10 +}; + +enum maildir_uidlist_hdr_ext_key { + MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY = 'V', + MAILDIR_UIDLIST_HDR_EXT_NEXT_UID = 'N', + MAILDIR_UIDLIST_HDR_EXT_GUID = 'G', + /* POP3 UIDL format unless overridden by records */ + MAILDIR_UIDLIST_HDR_EXT_POP3_UIDL_FORMAT = 'P' +}; + +#define MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(c) \ + ((c) >= 'A' && (c) <= 'Z') +enum maildir_uidlist_rec_ext_key { + /* Physical message size. If filename also contains ,S=<vsize> this + isn't written to uidlist. */ + MAILDIR_UIDLIST_REC_EXT_PSIZE = 'S', + /* Virtual message size. If filename also contains ,W=<vsize> this + isn't written to uidlist. */ + MAILDIR_UIDLIST_REC_EXT_VSIZE = 'W', + /* POP3 UIDL overriding the default format */ + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL = 'P', + /* POP3 message ordering number. Lower numbered messages are listed + first. Messages without ordering number are listed after them. + The idea is to be able to preserve POP3 UIDL list and IMAP UIDs + perfectly when migrating from other servers. */ + MAILDIR_UIDLIST_REC_EXT_POP3_ORDER = 'O', + /* Message GUID (default is the base filename) */ + MAILDIR_UIDLIST_REC_EXT_GUID = 'G' +}; + +int maildir_uidlist_lock(struct maildir_uidlist *uidlist); +int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist); +int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist); +void maildir_uidlist_unlock(struct maildir_uidlist *uidlist); +bool maildir_uidlist_is_locked(struct maildir_uidlist *uidlist); +bool maildir_uidlist_is_read(struct maildir_uidlist *uidlist); +/* Returns TRUE if uidlist file is currently open */ +bool maildir_uidlist_is_open(struct maildir_uidlist *uidlist); + +struct maildir_uidlist *maildir_uidlist_init(struct maildir_mailbox *mbox); +void maildir_uidlist_deinit(struct maildir_uidlist **uidlist); + +/* Returns -1 if error, 0 if file is broken or lost, 1 if ok. If nfs_flush=TRUE + and storage has NFS_FLUSH flag set, the NFS attribute cache is flushed to + make sure that we see the latest uidlist file. */ +int maildir_uidlist_refresh(struct maildir_uidlist *uidlist); +/* Like maildir_uidlist_refresh(), but if uidlist isn't opened yet, try to + fill in the uidvalidity/nextuid from index file instead. */ +int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist); + +/* Look up uidlist record for given filename. Returns 1 if found, + 0 if not found, -1 if error */ +int maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r); +/* Returns extension's value or NULL if it doesn't exist. */ +const char * +maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key); + +uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist); +uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist); +int maildir_uidlist_get_mailbox_guid(struct maildir_uidlist *uidlist, + guid_128_t mailbox_guid); +void maildir_uidlist_set_mailbox_guid(struct maildir_uidlist *uidlist, + const guid_128_t mailbox_guid); + +void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist, + uint32_t uid_validity); +void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist, + uint32_t next_uid, bool force); + +/* Update extended record. */ +void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key, + const char *value); +void maildir_uidlist_unset_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key); + +/* If uidlist has changed, update it. This is mostly meant to be used with + maildir_uidlist_set_ext() */ +int maildir_uidlist_update(struct maildir_uidlist *uidlist); + +void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist); +/* Sync uidlist with what's actually on maildir. Returns same as + maildir_uidlist_lock(). */ +int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, + enum maildir_uidlist_sync_flags sync_flags, + struct maildir_uidlist_sync_ctx **sync_ctx_r); +int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, + enum maildir_uidlist_rec_flag flags); +int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, uint32_t uid, + enum maildir_uidlist_rec_flag flags, + struct maildir_uidlist_rec **rec_r); +void maildir_uidlist_sync_remove(struct maildir_uidlist_sync_ctx *ctx, + const char *filename); +void maildir_uidlist_sync_set_ext(struct maildir_uidlist_sync_ctx *ctx, + struct maildir_uidlist_rec *rec, + enum maildir_uidlist_rec_ext_key key, + const char *value); +void maildir_uidlist_update_fname(struct maildir_uidlist *uidlist, + const char *filename); +const char * +maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx, + const char *filename); +void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx); +void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx); +int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **ctx, + bool success); + +bool maildir_uidlist_get_uid(struct maildir_uidlist *uidlist, + const char *filename, uint32_t *uid_r); +const char * +maildir_uidlist_get_full_filename(struct maildir_uidlist *uidlist, + const char *filename); + +void maildir_uidlist_add_flags(struct maildir_uidlist *uidlist, + const char *filename, + enum maildir_uidlist_rec_flag flags); + +/* List all maildir files. */ +struct maildir_uidlist_iter_ctx * +maildir_uidlist_iter_init(struct maildir_uidlist *uidlist); +bool maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, + uint32_t *uid_r, + enum maildir_uidlist_rec_flag *flags_r, + const char **filename_r); +void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx **ctx); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-util.c b/src/lib-storage/index/maildir/maildir-util.c new file mode 100644 index 0000000..c03546e --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-util.c @@ -0,0 +1,323 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "str.h" +#include "mkdir-parents.h" +#include "mailbox-list-private.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename-flags.h" +#include "maildir-sync.h" +#include "mailbox-recent-flags.h" + +#include <stdio.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <utime.h> +#include <sys/stat.h> + +#define MAILDIR_RESYNC_RETRY_COUNT 10 + +static const char * +maildir_filename_guess(struct maildir_mailbox *mbox, uint32_t uid, + const char *fname, + enum maildir_uidlist_rec_flag *uidlist_flags, + bool *have_flags_r) + +{ + struct mail_index_view *view = mbox->flags_view; + struct maildir_keywords_sync_ctx *kw_ctx; + enum mail_flags flags; + ARRAY_TYPE(keyword_indexes) keywords; + const char *p; + uint32_t seq; + + if (view == NULL || !mail_index_lookup_seq(view, uid, &seq)) { + *have_flags_r = FALSE; + return fname; + } + + t_array_init(&keywords, 32); + mail_index_lookup_view_flags(view, seq, &flags, &keywords); + if (array_count(&keywords) == 0) { + *have_flags_r = (flags & MAIL_FLAGS_NONRECENT) != 0; + fname = maildir_filename_flags_set(fname, flags); + } else { + *have_flags_r = TRUE; + kw_ctx = maildir_keywords_sync_init_readonly(mbox->keywords, + mbox->box.index); + fname = maildir_filename_flags_kw_set(kw_ctx, fname, + flags, &keywords); + maildir_keywords_sync_deinit(&kw_ctx); + } + + if (*have_flags_r) { + /* don't even bother looking into new/ dir */ + *uidlist_flags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + } else if ((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0 && + ((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 || + mailbox_recent_flags_have_uid(&mbox->box, uid))) { + /* probably in new/ dir, drop ":2," from fname */ + *uidlist_flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + p = strrchr(fname, MAILDIR_INFO_SEP); + if (p != NULL) + fname = t_strdup_until(fname, p); + } + + return fname; +} + +static int maildir_file_do_try(struct maildir_mailbox *mbox, uint32_t uid, + maildir_file_do_func *callback, void *context) +{ + const char *path, *fname; + enum maildir_uidlist_rec_flag flags; + bool have_flags; + int ret; + + ret = maildir_sync_lookup(mbox, uid, &flags, &fname); + if (ret <= 0) + return ret == 0 ? -2 : -1; + + if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* let's see if we can guess the filename based on index */ + fname = maildir_filename_guess(mbox, uid, fname, + &flags, &have_flags); + } + /* make a copy, just in case callback refreshes uidlist and + the pointer becomes invalid. */ + fname = t_strdup(fname); + + ret = 0; + if ((flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) { + /* probably in new/ dir */ + path = t_strconcat(mailbox_get_path(&mbox->box), + "/new/", fname, NULL); + ret = callback(mbox, path, context); + } + if (ret == 0) { + path = t_strconcat(mailbox_get_path(&mbox->box), "/cur/", + fname, NULL); + ret = callback(mbox, path, context); + } + if (ret > 0 && (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* file was found. make sure we remember its latest name. */ + maildir_uidlist_update_fname(mbox->uidlist, fname); + } else if (ret == 0 && + (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) { + /* file wasn't found. mark this message nonsynced, so we can + retry the lookup by guessing the flags */ + maildir_uidlist_add_flags(mbox->uidlist, fname, + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + } + return ret; +} + +static int do_racecheck(struct maildir_mailbox *mbox, const char *path, + void *context) +{ + const uint32_t *uidp = context; + struct stat st; + int ret; + + ret = lstat(path, &st); + if (ret == 0 && (st.st_mode & S_IFMT) == S_IFLNK) { + /* most likely a symlink pointing to a nonexistent file */ + mailbox_set_critical(&mbox->box, + "Maildir: Symlink destination doesn't exist for UID=%u: %s", *uidp, path); + return -2; + } else if (ret < 0 && errno != ENOENT) { + mailbox_set_critical(&mbox->box, "lstat(%s) failed: %m", path); + return -1; + } else { + /* success or ENOENT, either way we're done */ + mailbox_set_critical(&mbox->box, + "maildir_file_do(%s): Filename keeps changing for UID=%u", path, *uidp); + return -1; + } +} + +#undef maildir_file_do +int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid, + maildir_file_do_func *callback, void *context) +{ + int i, ret; + + T_BEGIN { + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; + if (ret == 0 && mbox->storage->set->maildir_very_dirty_syncs) T_BEGIN { + /* try guessing again with refreshed flags */ + if (maildir_sync_refresh_flags_view(mbox) == 0) + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; + for (i = 0; i < MAILDIR_RESYNC_RETRY_COUNT && ret == 0; i++) { + /* file is either renamed or deleted. sync the maildir and + see which one. if file appears to be renamed constantly, + don't try to open it more than 10 times. */ + if (maildir_storage_sync_force(mbox, uid) < 0) + return -1; + + T_BEGIN { + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; + } + + if (i == MAILDIR_RESYNC_RETRY_COUNT) T_BEGIN { + ret = maildir_file_do_try(mbox, uid, do_racecheck, &uid); + } T_END; + + return ret == -2 ? 0 : ret; +} + +static int maildir_create_path(struct mailbox *box, const char *path, + enum mailbox_list_path_type type, bool retry) +{ + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *p, *parent; + + if (mkdir_chgrp(path, perm->dir_create_mode, perm->file_create_gid, + perm->file_create_gid_origin) == 0) + return 0; + + switch (errno) { + case EEXIST: + return 0; + case ENOENT: + p = strrchr(path, '/'); + if (type == MAILBOX_LIST_PATH_TYPE_MAILBOX || + p == NULL || !retry) { + /* mailbox was being deleted just now */ + mailbox_set_deleted(box); + return -1; + } + /* create index/control root directory */ + parent = t_strdup_until(path, p); + if (mailbox_list_mkdir_root(box->list, parent, type) < 0) { + mail_storage_copy_list_error(box->storage, box->list); + return -1; + } + /* should work now, try again */ + return maildir_create_path(box, path, type, FALSE); + default: + mailbox_set_critical(box, "mkdir(%s) failed: %m", path); + return -1; + } +} + +static int maildir_create_subdirs(struct mailbox *box) +{ + static const char *subdirs[] = { "cur", "new", "tmp" }; + const char *dirs[N_ELEMENTS(subdirs) + 2]; + enum mailbox_list_path_type types[N_ELEMENTS(subdirs) + 2]; + struct stat st; + const char *path; + unsigned int i, count; + + /* @UNSAFE: get a list of directories we want to create */ + for (i = 0; i < N_ELEMENTS(subdirs); i++) { + types[i] = MAILBOX_LIST_PATH_TYPE_MAILBOX; + dirs[i] = t_strconcat(mailbox_get_path(box), + "/", subdirs[i], NULL); + } + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, &path) > 0) { + types[i] = MAILBOX_LIST_PATH_TYPE_CONTROL; + dirs[i++] = path; + } + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) > 0) { + types[i] = MAILBOX_LIST_PATH_TYPE_INDEX; + dirs[i++] = path; + } + count = i; + i_assert(count <= N_ELEMENTS(dirs)); + + for (i = 0; i < count; i++) { + path = dirs[i]; + if (stat(path, &st) == 0) + continue; + if (errno != ENOENT) { + mailbox_set_critical(box, "stat(%s) failed: %m", path); + break; + } + if (maildir_create_path(box, path, types[i], TRUE) < 0) + break; + } + return i == N_ELEMENTS(dirs) ? 0 : -1; +} + +bool maildir_set_deleted(struct mailbox *box) +{ + struct stat st; + int ret; + + if (stat(mailbox_get_path(box), &st) < 0) { + if (errno == ENOENT) + mailbox_set_deleted(box); + else { + mailbox_set_critical(box, + "stat(%s) failed: %m", mailbox_get_path(box)); + } + return FALSE; + } + /* maildir itself exists. create all of its subdirectories in case + they got lost. */ + T_BEGIN { + ret = maildir_create_subdirs(box); + } T_END; + return ret < 0 ? FALSE : TRUE; +} + +int maildir_lose_unexpected_dir(struct mail_storage *storage, const char *path) +{ + const char *dest, *fname, *p; + + /* There's a directory in maildir, get rid of it. + + In some installations this was caused by a messed up configuration + where e.g. mails was initially delivered to new/new/ directory. + Also Dovecot v2.0.0 - v2.0.4 sometimes may have renamed tmp/ + directory under new/ or cur/. */ + if (rmdir(path) == 0) { + mail_storage_set_critical(storage, + "Maildir: rmdir()ed unwanted empty directory: %s", + path); + return 1; + } else if (errno == ENOENT) { + /* someone else rmdired or renamed it */ + return 0; + } else if (errno != ENOTEMPTY) { + mail_storage_set_critical(storage, + "Maildir: Found unwanted directory %s, " + "but rmdir() failed: %m", path); + return -1; + } + + /* It's not safe to delete this directory since it has some files in it, + but it's also not helpful to log this message over and over again. + Get rid of this error by renaming the directory elsewhere */ + p = strrchr(path, '/'); + i_assert(p != NULL); + fname = p + 1; + while (p != path && p[-1] != '/') p--; + i_assert(p != NULL); + + dest = t_strconcat(t_strdup_until(path, p), "extra-", fname, NULL); + if (rename(path, dest) == 0) { + mail_storage_set_critical(storage, + "Maildir: renamed unwanted directory %s to %s", + path, dest); + return 1; + } else if (errno == ENOENT) { + /* someone else renamed it (could have been flag change) */ + return 0; + } else { + mail_storage_set_critical(storage, + "Maildir: Found unwanted directory, " + "but rename(%s, %s) failed: %m", path, dest); + return -1; + } +} diff --git a/src/lib-storage/index/mbox/Makefile.am b/src/lib-storage/index/mbox/Makefile.am new file mode 100644 index 0000000..4165a20 --- /dev/null +++ b/src/lib-storage/index/mbox/Makefile.am @@ -0,0 +1,39 @@ +noinst_LTLIBRARIES = libstorage_mbox.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_mbox_la_SOURCES = \ + istream-raw-mbox.c \ + mbox-file.c \ + mbox-lock.c \ + mbox-mail.c \ + mbox-md5-apop3d.c \ + mbox-md5-all.c \ + mbox-save.c \ + mbox-settings.c \ + mbox-sync-list-index.c \ + mbox-sync-parse.c \ + mbox-sync-rewrite.c \ + mbox-sync-update.c \ + mbox-sync.c \ + mbox-storage.c + +headers = \ + istream-raw-mbox.h \ + mbox-file.h \ + mbox-lock.h \ + mbox-md5.h \ + mbox-settings.h \ + mbox-storage.h \ + mbox-sync-private.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/mbox/Makefile.in b/src/lib-storage/index/mbox/Makefile.in new file mode 100644 index 0000000..c0b98cb --- /dev/null +++ b/src/lib-storage/index/mbox/Makefile.in @@ -0,0 +1,884 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/mbox +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_mbox_la_LIBADD = +am_libstorage_mbox_la_OBJECTS = istream-raw-mbox.lo mbox-file.lo \ + mbox-lock.lo mbox-mail.lo mbox-md5-apop3d.lo mbox-md5-all.lo \ + mbox-save.lo mbox-settings.lo mbox-sync-list-index.lo \ + mbox-sync-parse.lo mbox-sync-rewrite.lo mbox-sync-update.lo \ + mbox-sync.lo mbox-storage.lo +libstorage_mbox_la_OBJECTS = $(am_libstorage_mbox_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/istream-raw-mbox.Plo \ + ./$(DEPDIR)/mbox-file.Plo ./$(DEPDIR)/mbox-lock.Plo \ + ./$(DEPDIR)/mbox-mail.Plo ./$(DEPDIR)/mbox-md5-all.Plo \ + ./$(DEPDIR)/mbox-md5-apop3d.Plo ./$(DEPDIR)/mbox-save.Plo \ + ./$(DEPDIR)/mbox-settings.Plo ./$(DEPDIR)/mbox-storage.Plo \ + ./$(DEPDIR)/mbox-sync-list-index.Plo \ + ./$(DEPDIR)/mbox-sync-parse.Plo \ + ./$(DEPDIR)/mbox-sync-rewrite.Plo \ + ./$(DEPDIR)/mbox-sync-update.Plo ./$(DEPDIR)/mbox-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_mbox_la_SOURCES) +DIST_SOURCES = $(libstorage_mbox_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_mbox.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_mbox_la_SOURCES = \ + istream-raw-mbox.c \ + mbox-file.c \ + mbox-lock.c \ + mbox-mail.c \ + mbox-md5-apop3d.c \ + mbox-md5-all.c \ + mbox-save.c \ + mbox-settings.c \ + mbox-sync-list-index.c \ + mbox-sync-parse.c \ + mbox-sync-rewrite.c \ + mbox-sync-update.c \ + mbox-sync.c \ + mbox-storage.c + +headers = \ + istream-raw-mbox.h \ + mbox-file.h \ + mbox-lock.h \ + mbox-md5.h \ + mbox-settings.h \ + mbox-storage.h \ + mbox-sync-private.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/mbox/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/mbox/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_mbox.la: $(libstorage_mbox_la_OBJECTS) $(libstorage_mbox_la_DEPENDENCIES) $(EXTRA_libstorage_mbox_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_mbox_la_OBJECTS) $(libstorage_mbox_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-raw-mbox.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-lock.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-md5-all.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-md5-apop3d.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-list-index.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-parse.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-rewrite.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-update.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/istream-raw-mbox.Plo + -rm -f ./$(DEPDIR)/mbox-file.Plo + -rm -f ./$(DEPDIR)/mbox-lock.Plo + -rm -f ./$(DEPDIR)/mbox-mail.Plo + -rm -f ./$(DEPDIR)/mbox-md5-all.Plo + -rm -f ./$(DEPDIR)/mbox-md5-apop3d.Plo + -rm -f ./$(DEPDIR)/mbox-save.Plo + -rm -f ./$(DEPDIR)/mbox-settings.Plo + -rm -f ./$(DEPDIR)/mbox-storage.Plo + -rm -f ./$(DEPDIR)/mbox-sync-list-index.Plo + -rm -f ./$(DEPDIR)/mbox-sync-parse.Plo + -rm -f ./$(DEPDIR)/mbox-sync-rewrite.Plo + -rm -f ./$(DEPDIR)/mbox-sync-update.Plo + -rm -f ./$(DEPDIR)/mbox-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/istream-raw-mbox.Plo + -rm -f ./$(DEPDIR)/mbox-file.Plo + -rm -f ./$(DEPDIR)/mbox-lock.Plo + -rm -f ./$(DEPDIR)/mbox-mail.Plo + -rm -f ./$(DEPDIR)/mbox-md5-all.Plo + -rm -f ./$(DEPDIR)/mbox-md5-apop3d.Plo + -rm -f ./$(DEPDIR)/mbox-save.Plo + -rm -f ./$(DEPDIR)/mbox-settings.Plo + -rm -f ./$(DEPDIR)/mbox-storage.Plo + -rm -f ./$(DEPDIR)/mbox-sync-list-index.Plo + -rm -f ./$(DEPDIR)/mbox-sync-parse.Plo + -rm -f ./$(DEPDIR)/mbox-sync-rewrite.Plo + -rm -f ./$(DEPDIR)/mbox-sync-update.Plo + -rm -f ./$(DEPDIR)/mbox-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.c b/src/lib-storage/index/mbox/istream-raw-mbox.c new file mode 100644 index 0000000..e91e581 --- /dev/null +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c @@ -0,0 +1,821 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "istream-private.h" +#include "istream-raw-mbox.h" +#include "mbox-from.h" + +struct raw_mbox_istream { + struct istream_private istream; + + time_t received_time, next_received_time; + char *sender, *next_sender; + + uoff_t from_offset, hdr_offset, body_offset, mail_size; + uoff_t input_peak_offset; + + bool locked:1; + bool seeked:1; + bool crlf_ending:1; + bool corrupted:1; + bool mail_size_forced:1; + bool eof:1; + bool header_missing_eoh:1; +}; + +static void mbox_istream_log_read_error(struct raw_mbox_istream *rstream) +{ + if (rstream->istream.parent->stream_errno != 0) { + /* Log e.g. compression istream error */ + i_error("Failed to read mbox file %s: %s", + i_stream_get_name(&rstream->istream.istream), + i_stream_get_error(rstream->istream.parent)); + } +} + +static void i_stream_raw_mbox_destroy(struct iostream_private *stream) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + i_free(rstream->sender); + i_free(rstream->next_sender); + + i_stream_seek(rstream->istream.parent, + rstream->istream.istream.v_offset); +} + +static int mbox_read_from_line(struct raw_mbox_istream *rstream) +{ + const unsigned char *buf, *p; + char *sender; + time_t received_time; + size_t pos, line_pos; + ssize_t ret; + unsigned int skip; + int tz; + + buf = i_stream_get_data(rstream->istream.parent, &pos); + i_assert(pos > 0); + + /* from_offset points to "\nFrom ", so unless we're at the beginning + of the file, skip the initial \n */ + if (rstream->from_offset == 0) + skip = 0; + else { + skip = 1; + if (*buf == '\r') + skip++; + } + + while ((p = memchr(buf+skip, '\n', pos-skip)) == NULL) { + ret = i_stream_read_memarea(rstream->istream.parent); + buf = i_stream_get_data(rstream->istream.parent, &pos); + if (ret < 0) { + if (ret == -2) { + /* From_-line is too long, but we should be + able to parse what we have so far. */ + break; + } + /* EOF shouldn't happen */ + rstream->istream.istream.eof = + rstream->istream.parent->eof; + rstream->istream.istream.stream_errno = + rstream->istream.parent->stream_errno; + return -1; + } + i_assert(pos > 0); + } + line_pos = p == NULL ? 0 : (size_t)(p - buf); + + /* beginning of mbox */ + if (memcmp(buf+skip, "From ", 5) != 0 || + mbox_from_parse((buf+skip)+5, (pos-skip)-5, + &received_time, &tz, &sender) < 0) { + /* broken From - should happen only at beginning of + file if this isn't a mbox.. */ + io_stream_set_error(&rstream->istream.iostream, + "mbox file doesn't begin with 'From ' line"); + rstream->istream.istream.stream_errno = EINVAL; + return -1; + } + + if (rstream->istream.istream.v_offset == rstream->from_offset) { + rstream->received_time = received_time; + i_free(rstream->sender); + rstream->sender = sender; + } else { + rstream->next_received_time = received_time; + i_free(rstream->next_sender); + rstream->next_sender = sender; + } + + /* skip over From-line */ + if (line_pos == 0) { + /* line was too long. skip the input until we find LF. */ + rstream->istream.istream.v_offset += pos; + i_stream_skip(rstream->istream.parent, pos); + + while ((ret = i_stream_read_memarea(rstream->istream.parent)) > 0) { + p = memchr(buf, '\n', pos); + if (p != NULL) + break; + rstream->istream.istream.v_offset += pos; + i_stream_skip(rstream->istream.parent, pos); + } + if (ret <= 0) { + i_assert(ret == -1); + /* EOF shouldn't happen */ + rstream->istream.istream.eof = + rstream->istream.parent->eof; + rstream->istream.istream.stream_errno = + rstream->istream.parent->stream_errno; + return -1; + } + line_pos = (size_t)(p - buf); + } + rstream->istream.istream.v_offset += line_pos+1; + i_stream_skip(rstream->istream.parent, line_pos+1); + + rstream->hdr_offset = rstream->istream.istream.v_offset; + return 0; +} + +static void handle_end_of_mail(struct raw_mbox_istream *rstream, size_t pos) +{ + rstream->mail_size = rstream->istream.istream.v_offset + pos - + rstream->hdr_offset; + + if (rstream->hdr_offset + rstream->mail_size < rstream->body_offset) { + uoff_t new_body_offset = + rstream->hdr_offset + rstream->mail_size; + + if (rstream->body_offset != UOFF_T_MAX) { + /* Header didn't have ending \n */ + rstream->header_missing_eoh = TRUE; + } else { + /* "headers\n\nFrom ..", the second \n belongs to next + message which we didn't know at the time yet. */ + } + + /* The +2 check is for CR+LF linefeeds */ + i_assert(rstream->body_offset == UOFF_T_MAX || + rstream->body_offset == new_body_offset + 1 || + rstream->body_offset == new_body_offset + 2); + rstream->body_offset = new_body_offset; + } +} + +static ssize_t i_stream_raw_mbox_read(struct istream_private *stream) +{ + static const char *mbox_from = "\nFrom "; + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + const unsigned char *buf; + const char *fromp; + char *sender; + time_t received_time; + size_t i, pos, new_pos, from_start_pos, from_after_pos; + ssize_t ret = 0; + int eoh_char, tz; + bool crlf_ending = FALSE; + + i_assert(rstream->seeked); + i_assert(stream->istream.v_offset >= rstream->from_offset); + + if (stream->istream.eof) + return -1; + if (rstream->corrupted) { + rstream->istream.istream.stream_errno = EINVAL; + return -1; + } + + i_stream_seek(stream->parent, stream->istream.v_offset); + + stream->pos -= stream->skip; + stream->skip = 0; + stream->buffer = NULL; + + do { + buf = i_stream_get_data(stream->parent, &pos); + if (pos > 1 && stream->istream.v_offset + pos > + rstream->input_peak_offset) { + /* fake our read count. needed because if in the end + we have only one character in buffer and we skip it + (as potential CR), we want to get back to this + i_stream_raw_mbox_read() to read more data. */ + ret = pos; + break; + } + ret = i_stream_read_memarea(stream->parent); + } while (ret > 0); + stream->istream.stream_errno = stream->parent->stream_errno; + + if (ret < 0) { + if (ret == -1) + mbox_istream_log_read_error(rstream); + if (ret == -2) { + if (stream->skip == stream->pos) { + /* From_-line is longer than our input buffer. + finish the check without seeing the LF. */ + } else if (stream->istream.v_offset + pos == + rstream->input_peak_offset) { + /* we've read everything our parent stream + has to offer. */ + stream->buffer = buf; + return -2; + } + /* parent stream is full, but we haven't returned + all its bytes to our caller yet. */ + } else if (stream->istream.v_offset != 0 || pos == 0) { + /* we've read the whole file, final byte should be + the \n trailer */ + if (pos > 0 && buf[pos-1] == '\n') { + pos--; + if (pos > 0 && buf[pos-1] == '\r') { + crlf_ending = TRUE; + pos--; + } + } + + i_assert(pos >= stream->pos); + ret = pos == stream->pos ? -1 : + (ssize_t)(pos - stream->pos); + + stream->buffer = buf; + stream->pos = pos; + + if (stream->istream.v_offset == rstream->from_offset) { + /* haven't seen From-line yet, so this mbox + stream is now at EOF */ + rstream->eof = TRUE; + } + stream->istream.eof = TRUE; + rstream->crlf_ending = crlf_ending; + handle_end_of_mail(rstream, pos); + return ret < 0 ? i_stream_raw_mbox_read(stream) : ret; + } + } + + if (stream->istream.v_offset == rstream->from_offset) { + /* beginning of message, we haven't yet read our From-line */ + if (pos == 2 && ret > 0) { + /* we're at the end of file with CR+LF linefeeds? + need more data to verify it. */ + rstream->input_peak_offset = + stream->istream.v_offset + pos; + return i_stream_raw_mbox_read(stream); + } + if (mbox_read_from_line(rstream) < 0) { + io_stream_set_error(&stream->iostream, + "Next message unexpectedly corrupted in mbox file " + "%s at %"PRIuUOFF_T, + i_stream_get_name(&stream->istream), + stream->istream.v_offset); + if (stream->istream.v_offset != 0) + i_error("%s", stream->iostream.error); + stream->pos = 0; + rstream->eof = TRUE; + rstream->corrupted = TRUE; + return -1; + } + + /* got it. we don't want to return it however, + so start again from headers */ + buf = i_stream_get_data(stream->parent, &pos); + if (pos == 0) + return i_stream_raw_mbox_read(stream); + } + + /* See if we have From-line here - note that it works right only + because all characters are different in mbox_from. */ + fromp = mbox_from; from_start_pos = from_after_pos = SIZE_MAX; + eoh_char = rstream->body_offset == UOFF_T_MAX ? '\n' : -1; + for (i = stream->pos; i < pos; i++) { + if (buf[i] == eoh_char && + ((i > 0 && buf[i-1] == '\n') || + (i > 1 && buf[i-1] == '\r' && buf[i-2] == '\n') || + stream->istream.v_offset + i == rstream->hdr_offset)) { + rstream->body_offset = stream->istream.v_offset + i + 1; + eoh_char = -1; + } + if ((char)buf[i] == *fromp) { + if (*++fromp == '\0') { + /* potential From-line, see if we have the + rest of the line buffered. */ + i++; + if (i >= 7 && buf[i-7] == '\r') { + /* CR also belongs to it. */ + crlf_ending = TRUE; + from_start_pos = i - 7; + } else { + crlf_ending = FALSE; + from_start_pos = i - 6; + } + + if (rstream->mail_size == UOFF_T_MAX || + rstream->hdr_offset + rstream->mail_size == + stream->istream.v_offset + from_start_pos) { + from_after_pos = i; + if (ret == -2) { + /* even if we don't have the + whole line, we need to + finish this check now. */ + goto mbox_verify; + } + } + fromp = mbox_from; + } else if (from_after_pos != SIZE_MAX) { + /* we have the whole From-line here now. + See if it's a valid one. */ + mbox_verify: + if (mbox_from_parse(buf + from_after_pos, + pos - from_after_pos, + &received_time, &tz, + &sender) == 0) { + /* yep, we stop here. */ + rstream->next_received_time = + received_time; + i_free(rstream->next_sender); + rstream->next_sender = sender; + stream->istream.eof = TRUE; + + rstream->crlf_ending = crlf_ending; + handle_end_of_mail(rstream, + from_start_pos); + break; + } + from_after_pos = SIZE_MAX; + } + } else { + fromp = mbox_from; + if ((char)buf[i] == *fromp) + fromp++; + } + } + + /* we want to go at least one byte further next time */ + rstream->input_peak_offset = stream->istream.v_offset + i; + + if (from_after_pos != SIZE_MAX) { + /* we're waiting for the \n at the end of From-line */ + new_pos = from_start_pos; + } else { + /* leave out the beginnings of potential From-line + CR */ + new_pos = i - (fromp - mbox_from); + if (new_pos > 0) + new_pos--; + } + + if (stream->istream.v_offset - + rstream->hdr_offset + new_pos > rstream->mail_size) { + /* istream_raw_mbox_set_next_offset() used invalid + cached next_offset? */ + io_stream_set_error(&stream->iostream, + "Next message unexpectedly lost from mbox file " + "%s at %"PRIuUOFF_T" (%s)", + i_stream_get_name(&stream->istream), + rstream->hdr_offset + rstream->mail_size, + rstream->mail_size_forced ? "cached" : "noncached"); + i_error("%s", stream->iostream.error); + rstream->eof = TRUE; + rstream->corrupted = TRUE; + rstream->istream.istream.stream_errno = EINVAL; + stream->pos = 0; + return -1; + } + + stream->buffer = buf; + if (new_pos == stream->pos) { + if (stream->istream.eof || ret > 0) + return i_stream_raw_mbox_read(stream); + i_assert(new_pos > 0); + ret = -2; + } else { + i_assert(new_pos > stream->pos); + ret = new_pos - stream->pos; + stream->pos = new_pos; + } + return ret; +} + +static void i_stream_raw_mbox_seek(struct istream_private *stream, + uoff_t v_offset, bool mark ATTR_UNUSED) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + stream->istream.v_offset = v_offset; + stream->skip = stream->pos = 0; + stream->buffer = NULL; + + rstream->input_peak_offset = 0; + rstream->eof = FALSE; +} + +static void i_stream_raw_mbox_sync(struct istream_private *stream) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + i_stream_sync(stream->parent); + + rstream->istream.skip = 0; + rstream->istream.pos = 0; + rstream->input_peak_offset = 0; +} + +static int +i_stream_raw_mbox_stat(struct istream_private *stream, bool exact) +{ + const struct stat *st; + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; + return -1; + } + + stream->statbuf = *st; + stream->statbuf.st_size = + !exact && rstream->seeked && rstream->mail_size != UOFF_T_MAX ? + (off_t)rstream->mail_size : -1; + return 0; +} + +struct istream *i_stream_create_raw_mbox(struct istream *input) +{ + struct raw_mbox_istream *rstream; + + i_assert(input->v_offset == 0); + + rstream = i_new(struct raw_mbox_istream, 1); + + rstream->body_offset = UOFF_T_MAX; + rstream->mail_size = UOFF_T_MAX; + rstream->received_time = (time_t)-1; + rstream->next_received_time = (time_t)-1; + + rstream->istream.iostream.destroy = i_stream_raw_mbox_destroy; + rstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + rstream->istream.read = i_stream_raw_mbox_read; + rstream->istream.seek = i_stream_raw_mbox_seek; + rstream->istream.sync = i_stream_raw_mbox_sync; + rstream->istream.stat = i_stream_raw_mbox_stat; + + rstream->istream.istream.readable_fd = input->readable_fd; + rstream->istream.istream.blocking = input->blocking; + rstream->istream.istream.seekable = input->seekable; + + return i_stream_create(&rstream->istream, input, -1, 0); +} + +static int istream_raw_mbox_is_valid_from(struct raw_mbox_istream *rstream) +{ + const unsigned char *data; + size_t size = 0; + time_t received_time; + char *sender; + int tz; + ssize_t ret = 0; + + /* minimal: "From x Thu Nov 29 22:33:52 2001" = 31 chars */ + do { + data = i_stream_get_data(rstream->istream.parent, &size); + if (size >= 31) + break; + } while ((ret = i_stream_read_memarea(rstream->istream.parent)) > 0); + if (ret == -1) + mbox_istream_log_read_error(rstream); + + if ((size == 1 && data[0] == '\n') || + (size == 2 && data[0] == '\r' && data[1] == '\n')) { + /* EOF */ + return 1; + } + + if (size > 31 && memcmp(data, "\nFrom ", 6) == 0) { + data += 6; + size -= 6; + } else if (size > 32 && memcmp(data, "\r\nFrom ", 7) == 0) { + data += 7; + size -= 7; + } else { + return 0; + } + + while (memchr(data, '\n', size) == NULL) { + ret = i_stream_read_bytes(rstream->istream.parent, + &data, &size, size+1); + if (ret < 0) { + if (ret == -1) + mbox_istream_log_read_error(rstream); + break; + } + } + + if (mbox_from_parse(data, size, &received_time, &tz, &sender) < 0) + return 0; + + rstream->next_received_time = received_time; + i_free(rstream->next_sender); + rstream->next_sender = sender; + return 1; +} + +uoff_t istream_raw_mbox_get_start_offset(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + return rstream->from_offset; +} + +int istream_raw_mbox_get_header_offset(struct istream *stream, + uoff_t *hdr_offset_r) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + if (rstream->hdr_offset == rstream->from_offset) + (void)i_stream_read(stream); + + if (rstream->corrupted) { + i_error("Unexpectedly lost From-line from mbox file %s at " + "%"PRIuUOFF_T, i_stream_get_name(stream), + rstream->from_offset); + return -1; + } + if (stream->stream_errno != 0) + return -1; + + *hdr_offset_r = rstream->hdr_offset; + return 0; +} + +int istream_raw_mbox_get_body_offset(struct istream *stream, + uoff_t *body_offset_r) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + uoff_t offset; + + i_assert(rstream->seeked); + + if (rstream->body_offset != UOFF_T_MAX) { + *body_offset_r = rstream->body_offset; + return 0; + } + + offset = stream->v_offset; + i_stream_seek(stream, rstream->hdr_offset); + while (rstream->body_offset == UOFF_T_MAX) { + i_stream_skip(stream, i_stream_get_data_size(stream)); + + if (i_stream_read(stream) < 0) { + if (rstream->corrupted) { + i_error("Unexpectedly lost From-line from mbox file " + "%s at %"PRIuUOFF_T, + i_stream_get_name(stream), + rstream->from_offset); + } else { + i_assert(rstream->body_offset != UOFF_T_MAX); + } + return -1; + } + } + + i_stream_seek(stream, offset); + *body_offset_r = rstream->body_offset; + return 0; +} + +int istream_raw_mbox_get_body_size(struct istream *stream, + uoff_t expected_body_size, + uoff_t *body_size_r) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + const unsigned char *data; + size_t size; + uoff_t old_offset, body_offset, body_size, next_body_offset; + + i_assert(rstream->seeked); + i_assert(rstream->hdr_offset != UOFF_T_MAX); + + if (istream_raw_mbox_get_body_offset(stream, &body_offset) < 0) + return -1; + body_size = rstream->mail_size == UOFF_T_MAX ? UOFF_T_MAX : + rstream->mail_size - (rstream->body_offset - + rstream->hdr_offset); + old_offset = stream->v_offset; + if (expected_body_size != UOFF_T_MAX) { + /* if we already have the existing body size, use it as long as + it's >= expected body_size. otherwise the previous parsing + may have stopped at a From_-line that belongs to the body. */ + if (body_size != UOFF_T_MAX && body_size >= expected_body_size) { + *body_size_r = body_size; + return 0; + } + + next_body_offset = rstream->body_offset + expected_body_size; + /* If header_missing_eoh is set, the message body begins with + a From_-line and the body_offset is pointing to the line + *before* the first line of the body, i.e. the empty line + separating the headers from the body. If that is the case, + we'll have to skip over the empty line to get the correct + next_body_offset. */ + if (rstream->header_missing_eoh) { + i_assert(body_size == 0); + next_body_offset += rstream->crlf_ending ? 2 : 1; + } + + i_stream_seek(rstream->istream.parent, next_body_offset); + if (istream_raw_mbox_is_valid_from(rstream) > 0) { + rstream->mail_size = + next_body_offset - rstream->hdr_offset; + i_stream_seek(stream, old_offset); + *body_size_r = expected_body_size; + return 0; + } + /* invalid expected_body_size */ + } + if (body_size != UOFF_T_MAX) { + *body_size_r = body_size; + return 0; + } + + /* have to read through the message body */ + while (i_stream_read_more(stream, &data, &size) > 0) + i_stream_skip(stream, size); + i_stream_seek(stream, old_offset); + if (stream->stream_errno != 0) + return -1; + + i_assert(rstream->mail_size != UOFF_T_MAX); + *body_size_r = rstream->mail_size - + (rstream->body_offset - rstream->hdr_offset); + return 0; +} + +time_t istream_raw_mbox_get_received_time(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + if (rstream->received_time == (time_t)-1) + (void)i_stream_read(stream); + return rstream->received_time; +} + +const char *istream_raw_mbox_get_sender(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + if (rstream->sender == NULL) + (void)i_stream_read(stream); + return rstream->sender == NULL ? "" : rstream->sender; +} + +bool istream_raw_mbox_has_crlf_ending(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + return rstream->crlf_ending; +} + +int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + uoff_t body_size; + + if (istream_raw_mbox_get_body_size(stream, expected_body_size, + &body_size) < 0) + return -1; + rstream->mail_size = UOFF_T_MAX; + + rstream->received_time = rstream->next_received_time; + rstream->next_received_time = (time_t)-1; + + i_free(rstream->sender); + rstream->sender = rstream->next_sender; + rstream->next_sender = NULL; + + rstream->from_offset = rstream->body_offset + body_size; + rstream->hdr_offset = rstream->from_offset; + rstream->body_offset = UOFF_T_MAX; + rstream->header_missing_eoh = FALSE; + + if (stream->v_offset != rstream->from_offset) + i_stream_seek_mark(stream, rstream->from_offset); + i_stream_seek_mark(rstream->istream.parent, rstream->from_offset); + + rstream->eof = FALSE; + rstream->istream.istream.eof = FALSE; + return 0; +} + +int istream_raw_mbox_seek(struct istream *stream, uoff_t offset) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + bool check; + + i_assert(rstream->locked); + + /* reset any (corruption) errors */ + stream->stream_errno = 0; + i_free_and_null(stream->real_stream->iostream.error); + rstream->corrupted = FALSE; + rstream->eof = FALSE; + rstream->istream.istream.eof = FALSE; + + /* if seeked is FALSE, we unlocked in the middle. don't try to use + any cached state then. */ + if (rstream->mail_size != UOFF_T_MAX && rstream->seeked && + rstream->hdr_offset + rstream->mail_size == offset) + return istream_raw_mbox_next(stream, UOFF_T_MAX); + + if (offset == rstream->from_offset && rstream->seeked) { + /* back to beginning of current message */ + offset = rstream->hdr_offset; + check = offset == 0; + } else { + rstream->body_offset = UOFF_T_MAX; + rstream->mail_size = UOFF_T_MAX; + rstream->received_time = (time_t)-1; + rstream->next_received_time = (time_t)-1; + rstream->header_missing_eoh = FALSE; + + i_free(rstream->sender); + rstream->sender = NULL; + i_free(rstream->next_sender); + rstream->next_sender = NULL; + + rstream->from_offset = offset; + rstream->hdr_offset = offset; + check = TRUE; + } + rstream->seeked = TRUE; + + i_stream_seek_mark(stream, offset); + i_stream_seek_mark(rstream->istream.parent, offset); + + if (check) + (void)i_stream_read(stream); + return rstream->corrupted ? -1 : 0; +} + +void istream_raw_mbox_set_next_offset(struct istream *stream, uoff_t offset) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->hdr_offset != UOFF_T_MAX); + + rstream->mail_size_forced = TRUE; + rstream->mail_size = offset - rstream->hdr_offset; +} + +bool istream_raw_mbox_is_eof(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + return rstream->eof; +} + +bool istream_raw_mbox_is_corrupted(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + return rstream->corrupted; +} + +void istream_raw_mbox_set_locked(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + rstream->locked = TRUE; +} + +void istream_raw_mbox_set_unlocked(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + rstream->locked = FALSE; + rstream->seeked = FALSE; +} diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.h b/src/lib-storage/index/mbox/istream-raw-mbox.h new file mode 100644 index 0000000..4543841 --- /dev/null +++ b/src/lib-storage/index/mbox/istream-raw-mbox.h @@ -0,0 +1,56 @@ +#ifndef ISTREAM_RAW_MBOX_H +#define ISTREAM_RAW_MBOX_H + +/* Create a mbox stream for parsing mbox. Reading stops before From-line, + you'll have to call istream_raw_mbox_next() to get to next message. + path is used only for logging purposes. */ +struct istream *i_stream_create_raw_mbox(struct istream *input); + +/* Return offset to beginning of the "\nFrom"-line. */ +uoff_t istream_raw_mbox_get_start_offset(struct istream *stream); +/* Return offset to beginning of the headers. */ +int istream_raw_mbox_get_header_offset(struct istream *stream, + uoff_t *hdr_offset_r); +/* Return offset to beginning of the body. */ +int istream_raw_mbox_get_body_offset(struct istream *stream, + uoff_t *body_offset_r); + +/* Return the number of bytes in the body of this message. If + expected_body_size isn't UOFF_T_MAX, we'll use it as potentially valid body + size to avoid actually reading through the whole message. */ +int istream_raw_mbox_get_body_size(struct istream *stream, + uoff_t expected_body_size, + uoff_t *body_size_r); + +/* Return received time of current message, or (time_t)-1 if the timestamp is + broken. */ +time_t istream_raw_mbox_get_received_time(struct istream *stream); + +/* Return sender of current message. */ +const char *istream_raw_mbox_get_sender(struct istream *stream); +/* Return TRUE if the empty line between this and the next mail contains CR. */ +bool istream_raw_mbox_has_crlf_ending(struct istream *stream); + +/* Jump to next message. If expected_body_size isn't UOFF_T_MAX, we'll use it + as potentially valid body size. */ +int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size); + +/* Seek to message at given offset. offset must point to beginning of + "\nFrom ", or 0 for beginning of file. Returns -1 if it offset doesn't + contain a valid From-line. */ +int istream_raw_mbox_seek(struct istream *stream, uoff_t offset); +/* Set next message's start offset. If this isn't set, read stops at the next + valid From_-line, even if it belongs to the current message's body + (Content-Length: header can be used to determine that). */ +void istream_raw_mbox_set_next_offset(struct istream *stream, uoff_t offset); + +/* Returns TRUE if we've read the whole mbox. */ +bool istream_raw_mbox_is_eof(struct istream *stream); +/* Returns TRUE if we've noticed corruption in used offsets/sizes. */ +bool istream_raw_mbox_is_corrupted(struct istream *stream); +/* Change stream's locking state. We'll assert-crash if stream is tried to be + read while it's unlocked. */ +void istream_raw_mbox_set_locked(struct istream *stream); +void istream_raw_mbox_set_unlocked(struct istream *stream); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-file.c b/src/lib-storage/index/mbox/mbox-file.c new file mode 100644 index 0000000..b71abb8 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-file.c @@ -0,0 +1,207 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" +#include "mbox-file.h" +#include "istream-raw-mbox.h" + +#include <sys/stat.h> +#include <utime.h> + +#define MBOX_READ_BLOCK_SIZE IO_BLOCK_SIZE + +int mbox_file_open(struct mbox_mailbox *mbox) +{ + struct stat st; + int fd; + + i_assert(mbox->mbox_fd == -1); + + if (mbox->mbox_file_stream != NULL) { + /* read-only mbox stream */ + i_assert(mbox_is_backend_readonly(mbox)); + return 0; + } + + fd = open(mailbox_get_path(&mbox->box), + mbox_is_backend_readonly(mbox) ? O_RDONLY : O_RDWR); + if (fd == -1 && errno == EACCES && !mbox->backend_readonly) { + mbox->backend_readonly = TRUE; + fd = open(mailbox_get_path(&mbox->box), O_RDONLY); + } + + if (fd == -1) { + mbox_set_syscall_error(mbox, "open()"); + return -1; + } + + if (fstat(fd, &st) < 0) { + mbox_set_syscall_error(mbox, "fstat()"); + i_close_fd(&fd); + return -1; + } + + mbox->mbox_writeonly = S_ISFIFO(st.st_mode); + mbox->mbox_fd = fd; + mbox->mbox_dev = st.st_dev; + mbox->mbox_ino = st.st_ino; + return 0; +} + +void mbox_file_close(struct mbox_mailbox *mbox) +{ + mbox_file_close_stream(mbox); + + if (mbox->mbox_fd != -1) { + if (close(mbox->mbox_fd) < 0) + mbox_set_syscall_error(mbox, "close()"); + mbox->mbox_fd = -1; + } +} + +int mbox_file_open_stream(struct mbox_mailbox *mbox) +{ + if (mbox->mbox_stream != NULL) + return 0; + + if (mbox->mbox_file_stream != NULL) { + /* read-only mbox stream */ + i_assert(mbox->mbox_fd == -1 && mbox_is_backend_readonly(mbox)); + } else { + if (mbox->mbox_fd == -1) { + if (mbox_file_open(mbox) < 0) + return -1; + } + + if (mbox->mbox_writeonly) { + mbox->mbox_file_stream = + i_stream_create_from_data("", 0); + } else { + mbox->mbox_file_stream = + i_stream_create_fd(mbox->mbox_fd, + MBOX_READ_BLOCK_SIZE); + i_stream_set_init_buffer_size(mbox->mbox_file_stream, + MBOX_READ_BLOCK_SIZE); + } + i_stream_set_name(mbox->mbox_file_stream, + mailbox_get_path(&mbox->box)); + } + + mbox->mbox_stream = i_stream_create_raw_mbox(mbox->mbox_file_stream); + if (mbox->mbox_lock_type != F_UNLCK) + istream_raw_mbox_set_locked(mbox->mbox_stream); + return 0; +} + +static void mbox_file_fix_atime(struct mbox_mailbox *mbox) +{ + struct utimbuf buf; + struct stat st; + + if (mbox->box.recent_flags_count > 0 && + (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0 && + mbox->mbox_fd != -1 && !mbox_is_backend_readonly(mbox)) { + /* we've seen recent messages which we want to keep recent. + keep file's atime lower than mtime so \Marked status + gets shown while listing */ + if (fstat(mbox->mbox_fd, &st) < 0) { + mbox_set_syscall_error(mbox, "fstat()"); + return; + } + if (st.st_atime >= st.st_mtime) { + buf.modtime = st.st_mtime; + buf.actime = buf.modtime - 1; + /* EPERM can happen with shared mailboxes */ + if (utime(mailbox_get_path(&mbox->box), &buf) < 0 && + errno != EPERM) + mbox_set_syscall_error(mbox, "utime()"); + } + } +} +void mbox_file_close_stream(struct mbox_mailbox *mbox) +{ + /* if we read anything, fix the atime if needed */ + mbox_file_fix_atime(mbox); + + i_stream_destroy(&mbox->mbox_stream); + + if (mbox->mbox_file_stream != NULL) { + if (mbox->mbox_fd == -1) { + /* read-only mbox stream */ + i_assert(mbox_is_backend_readonly(mbox)); + i_stream_seek(mbox->mbox_file_stream, 0); + } else { + i_stream_destroy(&mbox->mbox_file_stream); + } + } +} + +int mbox_file_lookup_offset(struct mbox_mailbox *mbox, + struct mail_index_view *view, + uint32_t seq, uoff_t *offset_r) +{ + const void *data; + bool deleted; + + mail_index_lookup_ext(view, seq, mbox->mbox_ext_idx, &data, &deleted); + if (deleted) + return -1; + + if (data == NULL) { + mailbox_set_critical(&mbox->box, + "Cached message offset lost for seq %u in mbox", seq); + mbox->mbox_hdr.dirty_flag = 1; + mbox->mbox_broken_offsets = TRUE; + return 0; + } + + *offset_r = *((const uint64_t *)data); + return 1; +} + +int mbox_file_seek(struct mbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, bool *deleted_r) +{ + uoff_t offset; + int ret; + + ret = mbox_file_lookup_offset(mbox, view, seq, &offset); + if (ret <= 0) { + *deleted_r = ret < 0; + return ret; + } + *deleted_r = FALSE; + + if (istream_raw_mbox_seek(mbox->mbox_stream, offset) < 0) { + if (offset == 0) { + mbox->invalid_mbox_file = TRUE; + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOTPOSSIBLE, + "Mailbox isn't a valid mbox file"); + return -1; + } + + if (mbox->mbox_hdr.dirty_flag != 0) + return 0; + + mailbox_set_critical(&mbox->box, + "Cached message offset %s is invalid for mbox", + dec2str(offset)); + mbox->mbox_hdr.dirty_flag = 1; + mbox->mbox_broken_offsets = TRUE; + return 0; + } + + if (mbox->mbox_hdr.dirty_flag != 0) { + /* we're dirty - make sure this is the correct mail */ + if (!mbox_sync_parse_match_mail(mbox, view, seq)) + return 0; + + ret = istream_raw_mbox_seek(mbox->mbox_stream, offset); + i_assert(ret == 0); + } + + return 1; +} diff --git a/src/lib-storage/index/mbox/mbox-file.h b/src/lib-storage/index/mbox/mbox-file.h new file mode 100644 index 0000000..529a2e0 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-file.h @@ -0,0 +1,16 @@ +#ifndef MBOX_FILE_H +#define MBOX_FILE_H + +int mbox_file_open(struct mbox_mailbox *mbox); +void mbox_file_close(struct mbox_mailbox *mbox); + +int mbox_file_open_stream(struct mbox_mailbox *mbox); +void mbox_file_close_stream(struct mbox_mailbox *mbox); + +int mbox_file_lookup_offset(struct mbox_mailbox *mbox, + struct mail_index_view *view, + uint32_t seq, uoff_t *offset_r); +int mbox_file_seek(struct mbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, bool *deleted_r); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-lock.c b/src/lib-storage/index/mbox/mbox-lock.c new file mode 100644 index 0000000..cebff48 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-lock.c @@ -0,0 +1,900 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "eacces-error.h" +#include "restrict-access.h" +#include "nfs-workarounds.h" +#include "ipwd.h" +#include "mail-index-private.h" +#include "mbox-storage.h" +#include "istream-raw-mbox.h" +#include "mbox-file.h" +#include "mbox-lock.h" + +#include <time.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> + +#ifdef HAVE_FLOCK +# include <sys/file.h> +#endif + +/* 0.1 .. 0.2msec */ +#define LOCK_RANDOM_USLEEP_TIME (100000 + (unsigned int)i_rand() % 100000) + +enum mbox_lock_type { + MBOX_LOCK_DOTLOCK, + MBOX_LOCK_DOTLOCK_TRY, + MBOX_LOCK_FCNTL, + MBOX_LOCK_FLOCK, + MBOX_LOCK_LOCKF, + + MBOX_LOCK_COUNT +}; + +enum mbox_dotlock_op { + MBOX_DOTLOCK_OP_LOCK, + MBOX_DOTLOCK_OP_UNLOCK, + MBOX_DOTLOCK_OP_TOUCH +}; + +struct mbox_lock_context { + struct mbox_mailbox *mbox; + bool locked_status[MBOX_LOCK_COUNT]; + bool checked_file; + + int lock_type; + bool dotlock_last_stale; + bool fcntl_locked; + bool using_privileges; +}; + +struct mbox_lock_data { + enum mbox_lock_type type; + const char *name; + int (*func)(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +}; + +static int mbox_lock_dotlock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +static int mbox_lock_dotlock_try(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +static int mbox_lock_fcntl(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +#ifdef HAVE_FLOCK +static int mbox_lock_flock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +#else +# define mbox_lock_flock NULL +#endif +#ifdef HAVE_LOCKF +static int mbox_lock_lockf(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +#else +# define mbox_lock_lockf NULL +#endif + +static struct mbox_lock_data lock_data[] = { + { MBOX_LOCK_DOTLOCK, "dotlock", mbox_lock_dotlock }, + { MBOX_LOCK_DOTLOCK_TRY, "dotlock_try", mbox_lock_dotlock_try }, + { MBOX_LOCK_FCNTL, "fcntl", mbox_lock_fcntl }, + { MBOX_LOCK_FLOCK, "flock", mbox_lock_flock }, + { MBOX_LOCK_LOCKF, "lockf", mbox_lock_lockf }, + { 0, NULL, NULL } +}; + +static int ATTR_NOWARN_UNUSED_RESULT +mbox_lock_list(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time, int idx); +static int ATTR_NOWARN_UNUSED_RESULT +mbox_unlock_files(struct mbox_lock_context *ctx); + +static void mbox_read_lock_methods(const char *str, const char *env, + enum mbox_lock_type *locks) +{ + enum mbox_lock_type type; + const char *const *lock; + int i, dest; + + for (lock = t_strsplit(str, " "), dest = 0; *lock != NULL; lock++) { + for (type = 0; lock_data[type].name != NULL; type++) { + if (strcasecmp(*lock, lock_data[type].name) == 0) { + type = lock_data[type].type; + break; + } + } + if (lock_data[type].name == NULL) + i_fatal("%s: Invalid value %s", env, *lock); + if (lock_data[type].func == NULL) { + i_fatal("%s: Support for lock type %s " + "not compiled into binary", env, *lock); + } + + for (i = 0; i < dest; i++) { + if (locks[i] == type) + i_fatal("%s: Duplicated value %s", env, *lock); + } + + /* @UNSAFE */ + locks[dest++] = type; + } + locks[dest] = (enum mbox_lock_type)-1; +} + +static void mbox_init_lock_settings(struct mbox_storage *storage) +{ + enum mbox_lock_type read_locks[MBOX_LOCK_COUNT+1]; + enum mbox_lock_type write_locks[MBOX_LOCK_COUNT+1]; + int r, w; + + mbox_read_lock_methods(storage->set->mbox_read_locks, + "mbox_read_locks", read_locks); + mbox_read_lock_methods(storage->set->mbox_write_locks, + "mbox_write_locks", write_locks); + + /* check that read/write list orders match. write_locks must contain + at least read_locks and possibly more. */ + for (r = w = 0; write_locks[w] != (enum mbox_lock_type)-1; w++) { + if (read_locks[r] == (enum mbox_lock_type)-1) + break; + if (read_locks[r] == write_locks[w]) + r++; + } + if (read_locks[r] != (enum mbox_lock_type)-1) { + i_fatal("mbox read/write lock list settings are invalid. " + "Lock ordering must be the same with both, " + "and write locks must contain all read locks " + "(and possibly more)"); + } + + storage->read_locks = p_new(storage->storage.pool, + enum mbox_lock_type, MBOX_LOCK_COUNT+1); + memcpy(storage->read_locks, read_locks, + sizeof(*storage->read_locks) * (MBOX_LOCK_COUNT+1)); + + storage->write_locks = p_new(storage->storage.pool, + enum mbox_lock_type, MBOX_LOCK_COUNT+1); + memcpy(storage->write_locks, write_locks, + sizeof(*storage->write_locks) * (MBOX_LOCK_COUNT+1)); + + storage->lock_settings_initialized = TRUE; +} + +static int mbox_file_open_latest(struct mbox_lock_context *ctx, int lock_type) +{ + struct mbox_mailbox *mbox = ctx->mbox; + struct stat st; + + if (ctx->checked_file || lock_type == F_UNLCK) + return 0; + + if (mbox->mbox_fd != -1) { + /* we could flush NFS file handle cache here if we wanted to + be sure that the file is latest, but mbox files get rarely + deleted and the flushing might cause errors (e.g. EBUSY for + trying to flush a /var/mail mountpoint) */ + if (nfs_safe_stat(mailbox_get_path(&mbox->box), &st) < 0) { + if (errno == ENOENT) + mailbox_set_deleted(&mbox->box); + else + mbox_set_syscall_error(mbox, "stat()"); + return -1; + } + + if (st.st_ino != mbox->mbox_ino || + !CMP_DEV_T(st.st_dev, mbox->mbox_dev)) + mbox_file_close(mbox); + } + + if (mbox->mbox_fd == -1) { + if (mbox_file_open(mbox) < 0) + return -1; + } + + ctx->checked_file = TRUE; + return 0; +} + +static bool dotlock_callback(unsigned int secs_left, bool stale, void *context) +{ + struct mbox_lock_context *ctx = context; + enum mbox_lock_type *lock_types; + int i; + + if (ctx->using_privileges) + restrict_access_drop_priv_gid(); + + if (stale && !ctx->dotlock_last_stale) { + /* get next index we wish to try locking. it's the one after + dotlocking. */ + lock_types = ctx->lock_type == F_WRLCK || + (ctx->lock_type == F_UNLCK && + ctx->mbox->mbox_lock_type == F_WRLCK) ? + ctx->mbox->storage->write_locks : + ctx->mbox->storage->read_locks; + + for (i = 0; lock_types[i] != (enum mbox_lock_type)-1; i++) { + if (lock_types[i] == MBOX_LOCK_DOTLOCK) + break; + } + + if (lock_types[i] != (enum mbox_lock_type)-1 && + lock_types[i+1] != (enum mbox_lock_type)-1) { + i++; + if (mbox_lock_list(ctx, ctx->lock_type, 0, i) <= 0) { + /* we couldn't get fd lock - + it's really locked */ + ctx->dotlock_last_stale = TRUE; + return FALSE; + } + mbox_lock_list(ctx, F_UNLCK, 0, i); + } + } + ctx->dotlock_last_stale = stale; + + index_storage_lock_notify(&ctx->mbox->box, stale ? + MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE : + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + secs_left); + if (ctx->using_privileges) { + if (restrict_access_use_priv_gid() < 0) { + /* shouldn't get here */ + return FALSE; + } + } + return TRUE; +} + +static int ATTR_NULL(2) ATTR_NOWARN_UNUSED_RESULT +mbox_dotlock_privileged_op(struct mbox_mailbox *mbox, + struct dotlock_settings *set, + enum mbox_dotlock_op op) +{ + const char *box_path, *dir, *fname; + int ret = -1, orig_dir_fd, orig_errno; + + orig_dir_fd = open(".", O_RDONLY); + if (orig_dir_fd == -1) { + mailbox_set_critical(&mbox->box, "open(.) failed: %m"); + return -1; + } + + /* allow dotlocks to be created only for files we can read while we're + unprivileged. to make sure there are no race conditions we first + have to chdir to the mbox file's directory and then use relative + paths. unless this is done, users could: + - create *.lock files to any directory writable by the + privileged group + - DoS other users by dotlocking their mailboxes infinitely + */ + box_path = mailbox_get_path(&mbox->box); + fname = strrchr(box_path, '/'); + if (fname == NULL) { + /* already relative */ + fname = box_path; + } else { + dir = t_strdup_until(box_path, fname); + if (chdir(dir) < 0) { + mailbox_set_critical(&mbox->box, + "chdir(%s) failed: %m", dir); + i_close_fd(&orig_dir_fd); + return -1; + } + fname++; + } + if (op == MBOX_DOTLOCK_OP_LOCK) { + if (access(fname, R_OK) < 0) { + mailbox_set_critical(&mbox->box, + "access(%s) failed: %m", box_path); + i_close_fd(&orig_dir_fd); + return -1; + } + } + + if (restrict_access_use_priv_gid() < 0) { + i_close_fd(&orig_dir_fd); + return -1; + } + + switch (op) { + case MBOX_DOTLOCK_OP_LOCK: + /* we're now privileged - avoid doing as much as possible */ + ret = file_dotlock_create(set, fname, 0, &mbox->mbox_dotlock); + if (ret > 0) + mbox->mbox_used_privileges = TRUE; + else if (ret < 0 && errno == EACCES) { + const char *errmsg = + eacces_error_get_creating("file_dotlock_create", + fname); + mailbox_set_critical(&mbox->box, "%s", errmsg); + } else { + mbox_set_syscall_error(mbox, "file_dotlock_create()"); + } + break; + case MBOX_DOTLOCK_OP_UNLOCK: + /* we're now privileged - avoid doing as much as possible */ + ret = file_dotlock_delete(&mbox->mbox_dotlock); + if (ret < 0) + mbox_set_syscall_error(mbox, "file_dotlock_delete()"); + mbox->mbox_used_privileges = FALSE; + break; + case MBOX_DOTLOCK_OP_TOUCH: + ret = file_dotlock_touch(mbox->mbox_dotlock); + if (ret < 0) + mbox_set_syscall_error(mbox, "file_dotlock_touch()"); + break; + } + + orig_errno = errno; + restrict_access_drop_priv_gid(); + + if (fchdir(orig_dir_fd) < 0) { + mailbox_set_critical(&mbox->box, "fchdir() failed: %m"); + } + i_close_fd(&orig_dir_fd); + errno = orig_errno; + return ret; +} + +static void +mbox_dotlock_log_eacces_error(struct mbox_mailbox *mbox, const char *path) +{ + const char *dir, *errmsg, *name; + struct stat st; + struct group group; + int orig_errno = errno; + + errmsg = eacces_error_get_creating("file_dotlock_create", path); + dir = strrchr(path, '/'); + dir = dir == NULL ? "." : t_strdup_until(path, dir); + /* allow privileged locking for + a) user's own INBOX, + b) another user's shared INBOX, and + c) anything called INBOX (in inbox=no namespace) */ + if (!mbox->box.inbox_any && strcmp(mbox->box.name, "INBOX") != 0) { + mailbox_set_critical(&mbox->box, + "%s (not INBOX -> no privileged locking)", errmsg); + } else if (!mbox->mbox_privileged_locking) { + dir = mailbox_list_get_root_forced(mbox->box.list, + MAILBOX_LIST_PATH_TYPE_DIR); + mailbox_set_critical(&mbox->box, + "%s (under root dir %s -> no privileged locking)", + errmsg, dir); + } else if (stat(dir, &st) == 0 && + (st.st_mode & 02) == 0 && /* not world-writable */ + (st.st_mode & 020) != 0) { /* group-writable */ + if (i_getgrgid(st.st_gid, &group) <= 0) + name = dec2str(st.st_gid); + else + name = group.gr_name; + mailbox_set_critical(&mbox->box, + "%s (set mail_privileged_group=%s)", errmsg, name); + } else { + mailbox_set_critical(&mbox->box, + "%s (nonstandard permissions in %s)", errmsg, dir); + } + errno = orig_errno; +} + +static int +mbox_lock_dotlock_int(struct mbox_lock_context *ctx, int lock_type, bool try) +{ + struct mbox_mailbox *mbox = ctx->mbox; + struct dotlock_settings set; + int ret; + + if (lock_type == F_UNLCK) { + if (!mbox->mbox_dotlocked) + return 1; + + if (!mbox->mbox_used_privileges) { + if (file_dotlock_delete(&mbox->mbox_dotlock) <= 0) { + mbox_set_syscall_error(mbox, + "file_dotlock_delete()"); + } + } else { + ctx->using_privileges = TRUE; + mbox_dotlock_privileged_op(mbox, NULL, + MBOX_DOTLOCK_OP_UNLOCK); + ctx->using_privileges = FALSE; + } + mbox->mbox_dotlocked = FALSE; + return 1; + } + + if (mbox->mbox_dotlocked) + return 1; + + ctx->dotlock_last_stale = TRUE; + + i_zero(&set); + set.use_excl_lock = mbox->storage->storage.set->dotlock_use_excl; + set.nfs_flush = mbox->storage->storage.set->mail_nfs_storage; + set.timeout = mail_storage_get_lock_timeout(&mbox->storage->storage, + mbox->storage->set->mbox_lock_timeout); + set.stale_timeout = mbox->storage->set->mbox_dotlock_change_timeout; + set.callback = dotlock_callback; + set.context = ctx; + + ret = file_dotlock_create(&set, mailbox_get_path(&mbox->box), 0, + &mbox->mbox_dotlock); + if (ret >= 0) { + /* success / timeout */ + } else if (errno == EACCES && restrict_access_have_priv_gid() && + mbox->mbox_privileged_locking) { + /* try again, this time with extra privileges */ + ret = mbox_dotlock_privileged_op(mbox, &set, + MBOX_DOTLOCK_OP_LOCK); + } else if (errno == EACCES) + mbox_dotlock_log_eacces_error(mbox, mailbox_get_path(&mbox->box)); + else + mbox_set_syscall_error(mbox, "file_dotlock_create()"); + + if (ret < 0) { + if ((ENOSPACE(errno) || errno == EACCES) && try) + return 1; + return -1; + } + if (ret == 0) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT); + return 0; + } + mbox->mbox_dotlocked = TRUE; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + return 1; +} + +static int mbox_lock_dotlock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time ATTR_UNUSED) +{ + return mbox_lock_dotlock_int(ctx, lock_type, FALSE); +} + +static int mbox_lock_dotlock_try(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time ATTR_UNUSED) +{ + return mbox_lock_dotlock_int(ctx, lock_type, TRUE); +} + +#ifdef HAVE_FLOCK +static int mbox_lock_flock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time) +{ + time_t now; + unsigned int next_alarm; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + + if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1) + return 1; + + if (lock_type == F_WRLCK) + lock_type = LOCK_EX; + else if (lock_type == F_RDLCK) + lock_type = LOCK_SH; + else + lock_type = LOCK_UN; + + if (max_wait_time == 0) { + /* usually we're waiting here, but if we came from + mbox_lock_dotlock(), we just want to try locking */ + lock_type |= LOCK_NB; + } else { + now = time(NULL); + if (now >= max_wait_time) + alarm(1); + else + alarm(I_MIN(max_wait_time - now, 5)); + } + + while (flock(ctx->mbox->mbox_fd, lock_type) < 0) { + if (errno != EINTR) { + if (errno == EWOULDBLOCK && max_wait_time == 0) { + /* non-blocking lock trying failed */ + return 0; + } + alarm(0); + mbox_set_syscall_error(ctx->mbox, "flock()"); + return -1; + } + + now = time(NULL); + if (now >= max_wait_time) { + alarm(0); + return 0; + } + + /* notify locks once every 5 seconds. + try to use rounded values. */ + next_alarm = (max_wait_time - now) % 5; + if (next_alarm == 0) + next_alarm = 5; + alarm(next_alarm); + + index_storage_lock_notify(&ctx->mbox->box, + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + max_wait_time - now); + } + + alarm(0); + return 1; +} +#endif + +#ifdef HAVE_LOCKF +static int mbox_lock_lockf(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time) +{ + time_t now; + unsigned int next_alarm; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + + if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1) + return 1; + + if (lock_type == F_UNLCK) + lock_type = F_ULOCK; + else if (max_wait_time == 0) { + /* usually we're waiting here, but if we came from + mbox_lock_dotlock(), we just want to try locking */ + lock_type = F_TLOCK; + } else { + now = time(NULL); + if (now >= max_wait_time) + alarm(1); + else + alarm(I_MIN(max_wait_time - now, 5)); + lock_type = F_LOCK; + } + + while (lockf(ctx->mbox->mbox_fd, lock_type, 0) < 0) { + if (errno != EINTR) { + if ((errno == EACCES || errno == EAGAIN) && + max_wait_time == 0) { + /* non-blocking lock trying failed */ + return 0; + } + alarm(0); + mbox_set_syscall_error(ctx->mbox, "lockf()"); + return -1; + } + + now = time(NULL); + if (now >= max_wait_time) { + alarm(0); + return 0; + } + + /* notify locks once every 5 seconds. + try to use rounded values. */ + next_alarm = (max_wait_time - now) % 5; + if (next_alarm == 0) + next_alarm = 5; + alarm(next_alarm); + + index_storage_lock_notify(&ctx->mbox->box, + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + max_wait_time - now); + } + + alarm(0); + return 1; +} +#endif + +static int mbox_lock_fcntl(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time) +{ + struct flock fl; + time_t now; + unsigned int next_alarm; + int wait_type; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + + if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1) + return 1; + + i_zero(&fl); + fl.l_type = lock_type; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + + if (max_wait_time == 0) { + /* usually we're waiting here, but if we came from + mbox_lock_dotlock(), we just want to try locking */ + wait_type = F_SETLK; + } else { + wait_type = F_SETLKW; + now = time(NULL); + if (now >= max_wait_time) + alarm(1); + else + alarm(I_MIN(max_wait_time - now, 5)); + } + + while (fcntl(ctx->mbox->mbox_fd, wait_type, &fl) < 0) { + if (errno != EINTR) { + if ((errno == EACCES || errno == EAGAIN) && + wait_type == F_SETLK) { + /* non-blocking lock trying failed */ + return 0; + } + alarm(0); + if (errno != EACCES) { + mbox_set_syscall_error(ctx->mbox, "fcntl()"); + return -1; + } + mailbox_set_critical(&ctx->mbox->box, + "fcntl() failed with mbox file %s: " + "File is locked by another process (EACCES)", + mailbox_get_path(&ctx->mbox->box)); + return -1; + } + + now = time(NULL); + if (now >= max_wait_time) { + alarm(0); + return 0; + } + + /* notify locks once every 5 seconds. + try to use rounded values. */ + next_alarm = (max_wait_time - now) % 5; + if (next_alarm == 0) + next_alarm = 5; + alarm(next_alarm); + + index_storage_lock_notify(&ctx->mbox->box, + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + max_wait_time - now); + } + + alarm(0); + ctx->fcntl_locked = TRUE; + return 1; +} + +static int ATTR_NOWARN_UNUSED_RESULT +mbox_lock_list(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time, int idx) +{ + enum mbox_lock_type *lock_types; + enum mbox_lock_type type; + int i, ret = 0; + bool locked_status; + + ctx->lock_type = lock_type; + + lock_types = lock_type == F_WRLCK || + (lock_type == F_UNLCK && ctx->mbox->mbox_lock_type == F_WRLCK) ? + ctx->mbox->storage->write_locks : + ctx->mbox->storage->read_locks; + for (i = idx; lock_types[i] != (enum mbox_lock_type)-1; i++) { + type = lock_types[i]; + locked_status = lock_type != F_UNLCK; + + if (ctx->locked_status[type] == locked_status) + continue; + ctx->locked_status[type] = locked_status; + + ret = lock_data[type].func(ctx, lock_type, max_wait_time); + if (ret <= 0) + break; + } + return ret; +} + +static int mbox_update_locking(struct mbox_mailbox *mbox, int lock_type, + bool *fcntl_locked_r) +{ + struct mbox_lock_context ctx; + time_t max_wait_time; + int ret, i; + bool drop_locks; + + *fcntl_locked_r = FALSE; + + index_storage_lock_notify_reset(&mbox->box); + + if (!mbox->storage->lock_settings_initialized) + mbox_init_lock_settings(mbox->storage); + + if (mbox->mbox_fd == -1 && mbox->mbox_file_stream != NULL) { + /* read-only mbox stream. no need to lock. */ + i_assert(mbox_is_backend_readonly(mbox)); + mbox->mbox_lock_type = lock_type; + return 1; + } + + max_wait_time = time(NULL) + + mail_storage_get_lock_timeout(&mbox->storage->storage, + mbox->storage->set->mbox_lock_timeout); + + i_zero(&ctx); + ctx.mbox = mbox; + + if (mbox->mbox_lock_type == F_WRLCK) { + /* dropping to shared lock. first drop those that we + don't remove completely. */ + const enum mbox_lock_type *read_locks = + mbox->storage->read_locks; + + for (i = 0; i < MBOX_LOCK_COUNT; i++) + ctx.locked_status[i] = TRUE; + for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++) + ctx.locked_status[read_locks[i]] = FALSE; + drop_locks = TRUE; + } else { + drop_locks = FALSE; + } + + mbox->mbox_lock_type = lock_type; + ret = mbox_lock_list(&ctx, lock_type, max_wait_time, 0); + if (ret <= 0) { + if (!drop_locks) + mbox_unlock_files(&ctx); + if (ret == 0) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT); + } + return ret; + } + + if (drop_locks) { + /* dropping to shared lock: drop the locks that are only + in write list */ + const enum mbox_lock_type *read_locks = + mbox->storage->read_locks; + const enum mbox_lock_type *write_locks = + mbox->storage->write_locks; + + memset(ctx.locked_status, 0, sizeof(ctx.locked_status)); + for (i = 0; write_locks[i] != (enum mbox_lock_type)-1; i++) + ctx.locked_status[write_locks[i]] = TRUE; + for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++) + ctx.locked_status[read_locks[i]] = FALSE; + + mbox->mbox_lock_type = F_WRLCK; + mbox_lock_list(&ctx, F_UNLCK, 0, 0); + mbox->mbox_lock_type = F_RDLCK; + } + + *fcntl_locked_r = ctx.fcntl_locked; + return 1; +} + +int mbox_lock(struct mbox_mailbox *mbox, int lock_type, + unsigned int *lock_id_r) +{ + const char *path = mailbox_get_path(&mbox->box); + int mbox_fd = mbox->mbox_fd; + bool fcntl_locked; + int ret; + + if (lock_type == F_RDLCK && mbox->external_transactions > 0 && + mbox->mbox_lock_type != F_RDLCK) { + /* we have a transaction open that is going to save mails + and apparently also wants to read from the same mailbox + (copy, move, catenate). we need to write lock the mailbox, + since we can't later upgrade a read lock to write lock. */ + lock_type = F_WRLCK; + } + + /* allow only unlock -> shared/exclusive or exclusive -> shared */ + i_assert(lock_type == F_RDLCK || lock_type == F_WRLCK); + i_assert(lock_type == F_RDLCK || mbox->mbox_lock_type != F_RDLCK); + + if (mbox->mbox_lock_type == F_UNLCK) { + ret = mbox_update_locking(mbox, lock_type, &fcntl_locked); + if (ret <= 0) + return ret; + + if (mbox->storage->storage.set->mail_nfs_storage) { + if (fcntl_locked) { + nfs_flush_attr_cache_fd_locked(path, mbox_fd); + nfs_flush_read_cache_locked(path, mbox_fd); + } else { + nfs_flush_attr_cache_unlocked(path); + nfs_flush_read_cache_unlocked(path, mbox_fd); + } + } + + mbox->mbox_lock_id += 2; + } + + if (lock_type == F_RDLCK) { + mbox->mbox_shared_locks++; + *lock_id_r = mbox->mbox_lock_id; + } else { + mbox->mbox_excl_locks++; + *lock_id_r = mbox->mbox_lock_id + 1; + } + if (mbox->mbox_stream != NULL) + istream_raw_mbox_set_locked(mbox->mbox_stream); + return 1; +} + +static int mbox_unlock_files(struct mbox_lock_context *ctx) +{ + int ret = 0; + + if (mbox_lock_list(ctx, F_UNLCK, 0, 0) < 0) + ret = -1; + + ctx->mbox->mbox_lock_id += 2; + ctx->mbox->mbox_lock_type = F_UNLCK; + return ret; +} + +int mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id) +{ + struct mbox_lock_context ctx; + bool fcntl_locked; + int i; + + i_assert(mbox->mbox_lock_id == (lock_id & ~1U)); + + if ((lock_id & 1) != 0) { + /* dropping exclusive lock */ + i_assert(mbox->mbox_excl_locks > 0); + if (--mbox->mbox_excl_locks > 0) + return 0; + if (mbox->mbox_shared_locks > 0) { + /* drop to shared lock */ + if (mbox_update_locking(mbox, F_RDLCK, + &fcntl_locked) < 0) + return -1; + return 0; + } + } else { + /* dropping shared lock */ + i_assert(mbox->mbox_shared_locks > 0); + if (--mbox->mbox_shared_locks > 0) + return 0; + if (mbox->mbox_excl_locks > 0) + return 0; + } + /* all locks gone */ + + /* make sure we don't read the stream while unlocked */ + if (mbox->mbox_stream != NULL) + istream_raw_mbox_set_unlocked(mbox->mbox_stream); + + i_zero(&ctx); + ctx.mbox = mbox; + + for (i = 0; i < MBOX_LOCK_COUNT; i++) + ctx.locked_status[i] = TRUE; + + return mbox_unlock_files(&ctx); +} + +unsigned int mbox_get_cur_lock_id(struct mbox_mailbox *mbox) +{ + return mbox->mbox_lock_id + + (mbox->mbox_excl_locks > 0 ? 1 : 0); +} + +void mbox_dotlock_touch(struct mbox_mailbox *mbox) +{ + if (mbox->mbox_dotlock == NULL) + return; + + if (!mbox->mbox_used_privileges) + (void)file_dotlock_touch(mbox->mbox_dotlock); + else { + mbox_dotlock_privileged_op(mbox, NULL, + MBOX_DOTLOCK_OP_TOUCH); + } +} diff --git a/src/lib-storage/index/mbox/mbox-lock.h b/src/lib-storage/index/mbox/mbox-lock.h new file mode 100644 index 0000000..2175908 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-lock.h @@ -0,0 +1,15 @@ +#ifndef MBOX_LOCK_H +#define MBOX_LOCK_H + +/* NOTE: if mbox file is not open, it's opened. if it is open but file has + been overwritten (ie. inode has changed), it's reopened. */ +int mbox_lock(struct mbox_mailbox *mbox, int lock_type, + unsigned int *lock_id_r); +int ATTR_NOWARN_UNUSED_RESULT +mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id); + +unsigned int mbox_get_cur_lock_id(struct mbox_mailbox *mbox); + +void mbox_dotlock_touch(struct mbox_mailbox *mbox); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-mail.c b/src/lib-storage/index/mbox/mbox-mail.c new file mode 100644 index 0000000..09223b0 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-mail.c @@ -0,0 +1,439 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "hex-binary.h" +#include "index-mail.h" +#include "mbox-storage.h" +#include "mbox-file.h" +#include "mbox-lock.h" +#include "mbox-sync-private.h" +#include "istream-raw-mbox.h" +#include "istream-header-filter.h" + +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> + +static void mbox_prepare_resync(struct mail *mail) +{ + struct mbox_transaction_context *t = MBOX_TRANSCTX(mail->transaction); + struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->box); + + if (mbox->mbox_lock_type == F_RDLCK) { + if (mbox->mbox_lock_id == t->read_lock_id) + t->read_lock_id = 0; + mbox_unlock(mbox, mbox->mbox_lock_id); + i_assert(mbox->mbox_lock_type == F_UNLCK); + } +} + +static int mbox_mail_seek(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + struct mbox_transaction_context *t = MBOX_TRANSCTX(_mail->transaction); + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + enum mbox_sync_flags sync_flags = 0; + int ret, try; + bool deleted; + + if (_mail->expunged || mbox->syncing) + return -1; + + if (!mail_stream_access_start(_mail)) + return -1; + + if (mbox->mbox_stream != NULL && + istream_raw_mbox_is_corrupted(mbox->mbox_stream)) { + /* clear the corruption by forcing a full resync */ + sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC; + } + + for (try = 0; try < 2; try++) { + if ((sync_flags & MBOX_SYNC_FORCE_SYNC) != 0) { + /* dirty offsets are broken. make sure we can sync. */ + mbox_prepare_resync(_mail); + } + if (mbox->mbox_lock_type == F_UNLCK) { + i_assert(t->read_lock_id == 0); + sync_flags |= MBOX_SYNC_LOCK_READING; + if (mbox_sync(mbox, sync_flags) < 0) + return -1; + t->read_lock_id = mbox_get_cur_lock_id(mbox); + i_assert(t->read_lock_id != 0); + + /* refresh index file after mbox has been locked to + make sure we get only up-to-date mbox offsets. */ + if (mail_index_refresh(mbox->box.index) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + i_assert(mbox->mbox_lock_type != F_UNLCK); + } else if (t->read_lock_id == 0) { + /* file is already locked by another transaction, but + we must keep it locked for the entire transaction, + so increase the lock counter. */ + if (mbox_lock(mbox, mbox->mbox_lock_type, + &t->read_lock_id) < 0) + i_unreached(); + } + + if (mbox_file_open_stream(mbox) < 0) + return -1; + + ret = mbox_file_seek(mbox, _mail->transaction->view, + _mail->seq, &deleted); + if (ret > 0) { + /* success */ + break; + } + if (ret < 0) { + if (deleted) + mail_set_expunged(_mail); + return -1; + } + + /* we'll need to re-sync it completely */ + sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC; + } + if (ret == 0) { + mail_set_critical(_mail, "mbox: Losing sync"); + } + return 0; +} + +static int mbox_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (mbox_mail_seek(mail) < 0) + return -1; + data->received_date = + istream_raw_mbox_get_received_time(mbox->mbox_stream); + if (data->received_date == (time_t)-1) { + /* it's broken and conflicts with our "not found" + return value. change it. */ + data->received_date = 0; + } + + *date_r = data->received_date; + return 0; +} + +static int mbox_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + if (index_mail_get_save_date(_mail, date_r) > 0) + return 0; + + /* no way to know this. save the current time into cache and use + that from now on. this works only as long as the index files + are permanent */ + data->save_date = ioloop_time; + *date_r = data->save_date; + return 0; +} + +static int +mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r) +{ + struct mail *_mail = &mail->mail.mail; + static uint8_t empty_md5[16] = + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + const void *ext_data; + + if (mail->data.guid != NULL) { + *value_r = mail->data.guid; + return 1; + } + + mail_index_lookup_ext(_mail->transaction->view, _mail->seq, + mbox->md5hdr_ext_idx, &ext_data, NULL); + if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) { + mail->data.guid = p_strdup(mail->mail.data_pool, + binary_to_hex(ext_data, 16)); + *value_r = mail->data.guid; + return 1; + } else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) { + mail_set_expunged(_mail); + return -1; + } else { + return 0; + } +} + +static int +mbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + uoff_t offset; + bool move_offset; + int ret; + + switch (field) { + case MAIL_FETCH_FROM_ENVELOPE: + if (mbox_mail_seek(mail) < 0) + return -1; + + *value_r = istream_raw_mbox_get_sender(mbox->mbox_stream); + return 0; + case MAIL_FETCH_GUID: + case MAIL_FETCH_HEADER_MD5: + if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0) + return ret < 0 ? -1 : 0; + + /* i guess in theory the empty_md5 is valid and can happen, + but it's almost guaranteed that it means the MD5 sum is + missing. recalculate it. */ + if (mbox->mbox_lock_type == F_UNLCK || + mbox->mbox_stream == NULL) { + offset = 0; + move_offset = FALSE; + } else { + offset = istream_raw_mbox_get_start_offset(mbox->mbox_stream); + move_offset = TRUE; + } + mbox->mbox_save_md5 = TRUE; + if (mbox_sync(mbox, MBOX_SYNC_FORCE_SYNC | + MBOX_SYNC_READONLY) < 0) + return -1; + if (move_offset) { + if (istream_raw_mbox_seek(mbox->mbox_stream, + offset) < 0) { + i_error("mbox %s sync lost during MD5 syncing", + _mail->box->name); + return -1; + } + } + + if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) { + i_error("mbox %s resyncing didn't save header MD5 values", + _mail->box->name); + return -1; + } + return ret < 0 ? -1 : 0; + default: + break; + } + + return index_mail_get_special(_mail, field, value_r); +} + +static int +mbox_mail_get_next_offset(struct index_mail *mail, uoff_t *next_offset_r) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->mail.mail.box); + struct mail_index_view *view; + const struct mail_index_header *hdr; + uint32_t seq; + int trailer_size; + int ret = 1; + + *next_offset_r = UOFF_T_MAX; + + hdr = mail_index_get_header(mail->mail.mail.transaction->view); + if (mail->mail.mail.seq > hdr->messages_count) { + /* we're appending a new message */ + return 0; + } + + /* We can't really trust trans_view. The next message may already be + expunged from it. Also hdr.messages_count may be incorrect there. + So refresh the index to get the latest changes and get the next + message's offset using a new view. */ + i_assert(mbox->mbox_lock_type != F_UNLCK); + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + + view = mail_index_view_open(mail->mail.mail.box->index); + hdr = mail_index_get_header(view); + if (!mail_index_lookup_seq(view, mail->mail.mail.uid, &seq)) + i_panic("Message unexpectedly expunged from index"); + + if (seq < hdr->messages_count) { + if (mbox_file_lookup_offset(mbox, view, seq + 1, + next_offset_r) <= 0) + ret = -1; + } else if (mail->mail.mail.box->input != NULL) { + /* opened the mailbox as input stream. we can't trust the + sync_size, since it's wrong with compressed mailboxes */ + ret = 0; + } else { + /* last message, use the synced mbox size */ + trailer_size = + mbox->storage->storage.set->mail_save_crlf ? 2 : 1; + *next_offset_r = mbox->mbox_hdr.sync_size - trailer_size; + } + mail_index_view_close(&view); + return ret; +} + +static int mbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + struct istream *input; + struct message_size hdr_size; + uoff_t old_offset, body_offset, body_size, next_offset; + + if (index_mail_get_physical_size(_mail, size_r) == 0) + return 0; + + /* we want to return the header size as seen by mail_get_stream(). */ + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream(_mail, &hdr_size, NULL, &input) < 0) + return -1; + + /* our header size varies, so don't do any caching */ + if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) { + mail_set_critical(_mail, "mbox: Couldn't get body offset"); + return -1; + } + + /* use the next message's offset to avoid reading through the entire + message body to find out its size */ + if (mbox_mail_get_next_offset(mail, &next_offset) > 0) + body_size = next_offset - body_offset; + else + body_size = UOFF_T_MAX; + + /* verify that the calculated body size is correct */ + if (istream_raw_mbox_get_body_size(mbox->mbox_stream, + body_size, &body_size) < 0) { + mail_set_critical(_mail, "mbox: Couldn't get body size"); + return -1; + } + + data->physical_size = hdr_size.physical_size + body_size; + *size_r = data->physical_size; + + i_stream_seek(input, old_offset); + return 0; +} + +static int mbox_mail_init_stream(struct index_mail *mail) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->mail.mail.box); + struct istream *raw_stream; + uoff_t hdr_offset, next_offset; + int ret; + + if (mbox_mail_seek(mail) < 0) + return -1; + + ret = mbox_mail_get_next_offset(mail, &next_offset); + if (ret < 0) { + if (mbox_mail_seek(mail) < 0) + return -1; + ret = mbox_mail_get_next_offset(mail, &next_offset); + if (ret < 0) { + i_warning("mbox %s: Can't find next message offset " + "for uid=%u", mailbox_get_path(&mbox->box), + mail->mail.mail.uid); + } + } + + raw_stream = mbox->mbox_stream; + if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) { + mail_set_critical(&mail->mail.mail, + "mbox: Couldn't get header offset"); + return -1; + } + i_stream_seek(raw_stream, hdr_offset); + + if (next_offset != UOFF_T_MAX) + istream_raw_mbox_set_next_offset(raw_stream, next_offset); + + raw_stream = i_stream_create_limit(raw_stream, UOFF_T_MAX); + mail->data.stream = + i_stream_create_header_filter(raw_stream, + HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, + mbox_hide_headers, mbox_hide_headers_count, + *null_header_filter_callback, NULL); + i_stream_unref(&raw_stream); + return 0; +} + +static int mbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (mail->data.stream == NULL) { + if (mbox_mail_init_stream(mail) < 0) + return -1; + } + + return index_mail_init_stream(mail, hdr_size, body_size, stream_r); +} + +static void mbox_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + index_mail_set_seq(_mail, seq, saving); + mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE; +} + +static bool mbox_mail_set_uid(struct mail *_mail, uint32_t uid) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + bool ret; + + ret = index_mail_set_uid(_mail, uid); + mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE; + return ret; +} + +struct mail_vfuncs mbox_mail_vfuncs = { + index_mail_close, + index_mail_free, + mbox_mail_set_seq, + mbox_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + mbox_mail_get_received_date, + mbox_mail_get_save_date, + index_mail_get_virtual_size, + mbox_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + mbox_mail_get_stream, + index_mail_get_binary_stream, + mbox_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/mbox/mbox-md5-all.c b/src/lib-storage/index/mbox/mbox-md5-all.c new file mode 100644 index 0000000..9a09fb2 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-md5-all.c @@ -0,0 +1,39 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "md5.h" +#include "message-parser.h" +#include "mbox-md5.h" + + +struct mbox_md5_context { + struct md5_context hdr_md5_ctx; +}; + +static struct mbox_md5_context *mbox_md5_all_init(void) +{ + struct mbox_md5_context *ctx; + + ctx = i_new(struct mbox_md5_context, 1); + md5_init(&ctx->hdr_md5_ctx); + return ctx; +} + +static void mbox_md5_all_more(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); +} + +static void mbox_md5_all_finish(struct mbox_md5_context *ctx, + unsigned char result[STATIC_ARRAY 16]) +{ + md5_final(&ctx->hdr_md5_ctx, result); + i_free(ctx); +} + +struct mbox_md5_vfuncs mbox_md5_all = { + mbox_md5_all_init, + mbox_md5_all_more, + mbox_md5_all_finish +}; diff --git a/src/lib-storage/index/mbox/mbox-md5-apop3d.c b/src/lib-storage/index/mbox/mbox-md5-apop3d.c new file mode 100644 index 0000000..56f6f91 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-md5-apop3d.c @@ -0,0 +1,119 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "md5.h" +#include "message-parser.h" +#include "mbox-md5.h" + + +struct mbox_md5_context { + struct md5_context hdr_md5_ctx; + bool seen_received_hdr; +}; + +struct mbox_md5_header_func { + const char *header; + bool (*func)(struct mbox_md5_context *ctx, + struct message_header_line *hdr); +}; + +static bool parse_date(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + if (!ctx->seen_received_hdr) { + /* Received-header contains date too, and more trusted one */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + } + return TRUE; +} + +static bool parse_delivered_to(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + return TRUE; +} + +static bool parse_message_id(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + if (!ctx->seen_received_hdr) { + /* Received-header contains unique ID too, + and more trusted one */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + } + return TRUE; +} + +static bool parse_received(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + if (!ctx->seen_received_hdr) { + /* get only the first received-header */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + if (!hdr->continues) + ctx->seen_received_hdr = TRUE; + } + return TRUE; +} + +static bool parse_x_delivery_id(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + /* Let the local delivery agent help generate unique ID's but don't + blindly trust this header alone as it could just as easily come from + the remote. */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + return TRUE; +} + + +static struct mbox_md5_header_func md5_header_funcs[] = { + { "Date", parse_date }, + { "Delivered-To", parse_delivered_to }, + { "Message-ID", parse_message_id }, + { "Received", parse_received }, + { "X-Delivery-ID", parse_x_delivery_id } +}; + +static int bsearch_header_func_cmp(const void *p1, const void *p2) +{ + const char *key = p1; + const struct mbox_md5_header_func *func = p2; + + return strcasecmp(key, func->header); +} + +static struct mbox_md5_context *mbox_md5_apop3d_init(void) +{ + struct mbox_md5_context *ctx; + + ctx = i_new(struct mbox_md5_context, 1); + md5_init(&ctx->hdr_md5_ctx); + return ctx; +} + +static void mbox_md5_apop3d_more(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + struct mbox_md5_header_func *func; + + func = bsearch(hdr->name, md5_header_funcs, + N_ELEMENTS(md5_header_funcs), sizeof(*md5_header_funcs), + bsearch_header_func_cmp); + if (func != NULL) + (void)func->func(ctx, hdr); +} + +static void mbox_md5_apop3d_finish(struct mbox_md5_context *ctx, + unsigned char result[STATIC_ARRAY 16]) +{ + md5_final(&ctx->hdr_md5_ctx, result); + i_free(ctx); +} + +struct mbox_md5_vfuncs mbox_md5_apop3d = { + mbox_md5_apop3d_init, + mbox_md5_apop3d_more, + mbox_md5_apop3d_finish +}; diff --git a/src/lib-storage/index/mbox/mbox-md5.h b/src/lib-storage/index/mbox/mbox-md5.h new file mode 100644 index 0000000..7584052 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-md5.h @@ -0,0 +1,17 @@ +#ifndef MBOX_MD5_H +#define MBOX_MD5_H + +struct message_header_line; + +struct mbox_md5_vfuncs { + struct mbox_md5_context *(*init)(void); + void (*more)(struct mbox_md5_context *ctx, + struct message_header_line *hdr); + void (*finish)(struct mbox_md5_context *ctx, + unsigned char result[STATIC_ARRAY 16]); +}; + +extern struct mbox_md5_vfuncs mbox_md5_apop3d; +extern struct mbox_md5_vfuncs mbox_md5_all; + +#endif diff --git a/src/lib-storage/index/mbox/mbox-save.c b/src/lib-storage/index/mbox/mbox-save.c new file mode 100644 index 0000000..2fb3e19 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-save.c @@ -0,0 +1,833 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "base64.h" +#include "hostpid.h" +#include "randgen.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "write-full.h" +#include "istream-header-filter.h" +#include "istream-crlf.h" +#include "istream-concat.h" +#include "message-parser.h" +#include "mail-user.h" +#include "index-mail.h" +#include "mbox-storage.h" +#include "mbox-file.h" +#include "mbox-from.h" +#include "mbox-lock.h" +#include "mbox-md5.h" +#include "mbox-sync-private.h" + +#include <stddef.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <utime.h> + +#define MBOX_DELIVERY_ID_RAND_BYTES (64/8) + +struct mbox_save_context { + struct mail_save_context ctx; + + struct mbox_mailbox *mbox; + struct mail_index_transaction *trans; + uoff_t append_offset, mail_offset; + time_t orig_atime; + + string_t *headers; + size_t space_end_idx; + uint32_t seq, next_uid, uid_validity; + + struct istream *input; + struct ostream *output; + uoff_t extra_hdr_offset, eoh_offset; + char last_char; + + struct mbox_md5_context *mbox_md5_ctx; + char *x_delivery_id_header; + + bool synced:1; + bool failed:1; + bool finished:1; +}; + +#define MBOX_SAVECTX(s) container_of(s, struct mbox_save_context, ctx) + +static void ostream_error(struct mbox_save_context *ctx, const char *func) +{ + mbox_ostream_set_syscall_error(ctx->mbox, ctx->output, func); + ctx->failed = TRUE; +} + +static void write_stream_error(struct mbox_save_context *ctx) +{ + ostream_error(ctx, "write()"); +} + +static void lseek_stream_error(struct mbox_save_context *ctx) +{ + ostream_error(ctx, "o_stream_seek()"); +} + +static int mbox_seek_to_end(struct mbox_save_context *ctx, uoff_t *offset) +{ + struct stat st; + char ch; + int fd; + + if (ctx->mbox->mbox_writeonly) { + *offset = 0; + return 0; + } + + fd = ctx->mbox->mbox_fd; + if (fstat(fd, &st) < 0) { + mbox_set_syscall_error(ctx->mbox, "fstat()"); + return -1; + } + + ctx->orig_atime = st.st_atime; + + *offset = (uoff_t)st.st_size; + if (st.st_size == 0) + return 0; + + if (lseek(fd, st.st_size-1, SEEK_SET) < 0) { + mbox_set_syscall_error(ctx->mbox, "lseek()"); + return -1; + } + + if (read(fd, &ch, 1) != 1) { + mbox_set_syscall_error(ctx->mbox, "read()"); + return -1; + } + + if (ch != '\n') { + if (write_full(fd, "\n", 1) < 0) { + mbox_set_syscall_error(ctx->mbox, "write()"); + return -1; + } + *offset += 1; + } + + return 0; +} + +static int mbox_append_lf(struct mbox_save_context *ctx) +{ + if (o_stream_send(ctx->output, "\n", 1) < 0) { + write_stream_error(ctx); + return -1; + } + + return 0; +} + +static int write_from_line(struct mbox_save_context *ctx, time_t received_date, + const char *from_envelope) +{ + int ret; + + T_BEGIN { + const char *line; + + if (from_envelope == NULL) { + struct mail_storage *storage = + &ctx->mbox->storage->storage; + + from_envelope = + strchr(storage->user->username, '@') != NULL ? + storage->user->username : + t_strconcat(storage->user->username, + "@", my_hostdomain(), NULL); + } else if (*from_envelope == '\0') { + /* can't write empty envelope */ + from_envelope = "MAILER-DAEMON"; + } + + /* save in local timezone, no matter what it was given with */ + line = mbox_from_create(from_envelope, received_date); + + if ((ret = o_stream_send_str(ctx->output, line)) < 0) + write_stream_error(ctx); + } T_END; + return ret; +} + +static int mbox_write_content_length(struct mbox_save_context *ctx) +{ + uoff_t end_offset; + const char *str; + size_t len; + + i_assert(ctx->eoh_offset != UOFF_T_MAX); + + if (ctx->mbox->mbox_writeonly) { + /* we can't seek, don't set Content-Length */ + return 0; + } + + end_offset = ctx->output->offset; + + /* write Content-Length headers */ + str = t_strdup_printf("\nContent-Length: %s", + dec2str(end_offset - ctx->eoh_offset)); + len = strlen(str); + + /* flush manually here so that we don't confuse seek() errors with + buffer flushing errors */ + if (o_stream_flush(ctx->output) < 0) { + write_stream_error(ctx); + return -1; + } + if (o_stream_seek(ctx->output, ctx->extra_hdr_offset + + ctx->space_end_idx - len) < 0) { + lseek_stream_error(ctx); + return -1; + } + + if (o_stream_send(ctx->output, str, len) < 0 || + o_stream_flush(ctx->output) < 0) { + write_stream_error(ctx); + return -1; + } + + if (o_stream_seek(ctx->output, end_offset) < 0) { + lseek_stream_error(ctx); + return -1; + } + return 0; +} + +static void mbox_save_init_sync(struct mailbox_transaction_context *t) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(t->box); + struct mbox_save_context *ctx = MBOX_SAVECTX(t->save_ctx); + const struct mail_index_header *hdr; + struct mail_index_view *view; + + /* open a new view to get the header. this is required if we just + synced the mailbox so we can get updated next_uid. */ + mail_index_refresh(mbox->box.index); + view = mail_index_view_open(mbox->box.index); + hdr = mail_index_get_header(view); + + ctx->next_uid = hdr->next_uid; + ctx->uid_validity = hdr->uid_validity; + ctx->synced = TRUE; + + mail_index_view_close(&view); +} + +static void status_flags_append(string_t *str, enum mail_flags flags, + const struct mbox_flag_type *flags_list) +{ + int i; + + flags ^= MBOX_NONRECENT_KLUDGE; + for (i = 0; flags_list[i].chr != 0; i++) { + if ((flags & flags_list[i].flag) != 0) + str_append_c(str, flags_list[i].chr); + } +} + +static void mbox_save_append_flag_headers(string_t *str, enum mail_flags flags) +{ + /* write the Status: header always. It always gets added soon anyway. */ + str_append(str, "Status: "); + status_flags_append(str, flags, mbox_status_flags); + str_append_c(str, '\n'); + + if ((flags & XSTATUS_FLAGS_MASK) != 0) { + str_append(str, "X-Status: "); + status_flags_append(str, flags, mbox_xstatus_flags); + str_append_c(str, '\n'); + } +} + +static void +mbox_save_append_keyword_headers(struct mbox_save_context *ctx, + struct mail_keywords *keywords) +{ + unsigned char space[MBOX_HEADER_PADDING+1 + + sizeof("Content-Length: \n")-1 + MAX_INT_STRLEN]; + const ARRAY_TYPE(keywords) *keyword_names_list; + const char *const *keyword_names; + unsigned int i, count, keyword_names_count; + + keyword_names_list = mail_index_get_keywords(ctx->mbox->box.index); + keyword_names = array_get(keyword_names_list, &keyword_names_count); + + str_append(ctx->headers, "X-Keywords:"); + count = keywords == NULL ? 0 : keywords->count; + for (i = 0; i < count; i++) { + i_assert(keywords->idx[i] < keyword_names_count); + + str_append_c(ctx->headers, ' '); + str_append(ctx->headers, keyword_names[keywords->idx[i]]); + } + + memset(space, ' ', sizeof(space)); + str_append_data(ctx->headers, space, sizeof(space)); + ctx->space_end_idx = str_len(ctx->headers); + str_append_c(ctx->headers, '\n'); +} + +static int +mbox_save_init_file(struct mbox_save_context *ctx, + struct mbox_transaction_context *t) +{ + struct mailbox_transaction_context *_t = &t->t; + struct mbox_mailbox *mbox = ctx->mbox; + struct mail_storage *storage = &mbox->storage->storage; + int ret; + + if (mbox_is_backend_readonly(ctx->mbox)) { + mail_storage_set_error(storage, MAIL_ERROR_PERM, + "Read-only mbox"); + return -1; + } + + if (ctx->append_offset == UOFF_T_MAX) { + /* first appended mail in this transaction */ + if (t->write_lock_id == 0) { + if (mbox_lock(mbox, F_WRLCK, &t->write_lock_id) <= 0) + return -1; + } + + if (mbox->mbox_fd == -1) { + if (mbox_file_open(mbox) < 0) + return -1; + } + + /* update mbox_sync_dirty state */ + ret = mbox_sync_has_changed(mbox, TRUE); + if (ret < 0) + return -1; + } + + if (!ctx->synced) { + /* we'll need to assign UID for the mail immediately. */ + if (mbox_sync(mbox, 0) < 0) + return -1; + mbox_save_init_sync(_t); + } + + /* the syncing above could have changed the append offset */ + if (ctx->append_offset == UOFF_T_MAX) { + if (mbox_seek_to_end(ctx, &ctx->append_offset) < 0) + return -1; + + i_assert(mbox->mbox_fd != -1); + ctx->output = o_stream_create_fd_file(mbox->mbox_fd, + ctx->append_offset, + FALSE); + o_stream_cork(ctx->output); + } + return 0; +} + +static void +save_header_callback(struct header_filter_istream *input ATTR_UNUSED, + struct message_header_line *hdr, + bool *matched, struct mbox_save_context *ctx) +{ + if (hdr != NULL) { + if (str_begins(hdr->name, "From ")) { + /* we can't allow From_-lines in headers. there's no + legitimate reason for allowing them in any case, + so just drop them. */ + *matched = TRUE; + return; + } + + if (!*matched && ctx->mbox_md5_ctx != NULL) + ctx->mbox->md5_v.more(ctx->mbox_md5_ctx, hdr); + } +} + +static void mbox_save_x_delivery_id(struct mbox_save_context *ctx) +{ + unsigned char md5_result[MD5_RESULTLEN]; + buffer_t *buf; + string_t *str; + void *randbuf; + + buf = t_buffer_create(256); + buffer_append(buf, &ioloop_time, sizeof(ioloop_time)); + buffer_append(buf, &ioloop_timeval.tv_usec, + sizeof(ioloop_timeval.tv_usec)); + + randbuf = buffer_append_space_unsafe(buf, MBOX_DELIVERY_ID_RAND_BYTES); + random_fill(randbuf, MBOX_DELIVERY_ID_RAND_BYTES); + + md5_get_digest(buf->data, buf->used, md5_result); + + str = t_str_new(128); + str_append(str, "X-Delivery-ID: "); + base64_encode(md5_result, sizeof(md5_result), str); + str_append_c(str, '\n'); + + ctx->x_delivery_id_header = i_strdup(str_c(str)); +} + +static struct istream * +mbox_save_get_input_stream(struct mbox_save_context *ctx, struct istream *input) +{ + struct istream *filter, *ret, *cache_input, *streams[3]; + + /* filter out unwanted headers and keep track of headers' MD5 sum */ + filter = i_stream_create_header_filter(input, HEADER_FILTER_EXCLUDE | + HEADER_FILTER_NO_CR | + HEADER_FILTER_ADD_MISSING_EOH | + HEADER_FILTER_END_BODY_WITH_LF, + mbox_save_drop_headers, + mbox_save_drop_headers_count, + save_header_callback, ctx); + + if ((ctx->mbox->storage->storage.flags & + MAIL_STORAGE_FLAG_KEEP_HEADER_MD5) != 0) { + /* we're using MD5 sums to generate POP3 UIDLs. + clients don't like it much if there are duplicates, + so make sure that there can't be any by appending + our own X-Delivery-ID header. */ + const char *hdr; + + T_BEGIN { + mbox_save_x_delivery_id(ctx); + } T_END; + hdr = ctx->x_delivery_id_header; + + streams[0] = i_stream_create_from_data(hdr, strlen(hdr)); + streams[1] = filter; + streams[2] = NULL; + ret = i_stream_create_concat(streams); + i_stream_unref(&filter); + filter = ret; + } + + /* convert linefeeds to wanted format */ + ret = ctx->mbox->storage->storage.set->mail_save_crlf ? + i_stream_create_crlf(filter) : i_stream_create_lf(filter); + i_stream_unref(&filter); + + /* caching creates a tee stream */ + cache_input = index_mail_cache_parse_init(ctx->ctx.dest_mail, ret); + i_stream_unref(&ret); + ret = cache_input; + return ret; +} + +struct mail_save_context * +mbox_save_alloc(struct mailbox_transaction_context *t) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(t->box); + struct mbox_save_context *ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx == NULL) { + ctx = i_new(struct mbox_save_context, 1); + ctx->ctx.transaction = t; + ctx->mbox = mbox; + ctx->trans = t->itrans; + ctx->append_offset = UOFF_T_MAX; + ctx->headers = str_new(default_pool, 512); + ctx->mail_offset = UOFF_T_MAX; + t->save_ctx = &ctx->ctx; + } + return t->save_ctx; +} + +int mbox_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + struct mail_save_data *mdata = &_ctx->data; + struct mbox_transaction_context *t = MBOX_TRANSCTX(_ctx->transaction); + enum mail_flags save_flags; + uint64_t offset; + + /* FIXME: we could write timezone_offset to From-line.. */ + if (mdata->received_date == (time_t)-1) + mdata->received_date = ioloop_time; + + ctx->failed = FALSE; + ctx->seq = 0; + + if (mbox_save_init_file(ctx, t) < 0) { + ctx->failed = TRUE; + return -1; + } + + save_flags = mdata->flags; + if (mdata->uid == 0) + save_flags |= MAIL_RECENT; + str_truncate(ctx->headers, 0); + if (ctx->synced) { + if (ctx->mbox->mbox_save_md5) + ctx->mbox_md5_ctx = ctx->mbox->md5_v.init(); + if (ctx->next_uid < mdata->uid) { + /* we can use the wanted UID */ + ctx->next_uid = mdata->uid; + } + if (ctx->output->offset == 0) { + /* writing the first mail. Insert X-IMAPbase as well. */ + str_printfa(ctx->headers, "X-IMAPbase: %u %010u\n", + ctx->uid_validity, ctx->next_uid); + } + str_printfa(ctx->headers, "X-UID: %u\n", ctx->next_uid); + + mail_index_append(ctx->trans, ctx->next_uid, &ctx->seq); + mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE, + save_flags & ENUM_NEGATE(MAIL_RECENT)); + if (mdata->keywords != NULL) { + mail_index_update_keywords(ctx->trans, ctx->seq, + MODIFY_REPLACE, + mdata->keywords); + } + if (mdata->min_modseq != 0) { + mail_index_update_modseq(ctx->trans, ctx->seq, + mdata->min_modseq); + } + + offset = ctx->output->offset == 0 ? 0 : + ctx->output->offset - 1; + mail_index_update_ext(ctx->trans, ctx->seq, + ctx->mbox->mbox_ext_idx, &offset, NULL); + ctx->next_uid++; + + /* parse and cache the mail headers as we read it */ + mail_set_seq_saving(_ctx->dest_mail, ctx->seq); + } + mbox_save_append_flag_headers(ctx->headers, save_flags); + mbox_save_append_keyword_headers(ctx, mdata->keywords); + str_append_c(ctx->headers, '\n'); + + i_assert(ctx->mbox->mbox_lock_type == F_WRLCK); + + ctx->mail_offset = ctx->output->offset; + ctx->eoh_offset = UOFF_T_MAX; + ctx->last_char = '\n'; + + if (write_from_line(ctx, mdata->received_date, mdata->from_envelope) < 0) + ctx->failed = TRUE; + else + ctx->input = mbox_save_get_input_stream(ctx, input); + return ctx->failed ? -1 : 0; +} + +static int mbox_save_body_input(struct mbox_save_context *ctx) +{ + const unsigned char *data; + size_t size; + + data = i_stream_get_data(ctx->input, &size); + if (size > 0) { + if (o_stream_send(ctx->output, data, size) < 0) { + write_stream_error(ctx); + return -1; + } + ctx->last_char = data[size-1]; + i_stream_skip(ctx->input, size); + } + return 0; +} + +static int mbox_save_body(struct mbox_save_context *ctx) +{ + ssize_t ret; + + while ((ret = i_stream_read(ctx->input)) != -1) { + if (mbox_save_body_input(ctx) < 0) + return -1; + /* i_stream_read() may have returned 0 at EOF + because of this parser */ + index_mail_cache_parse_continue(ctx->ctx.dest_mail); + if (ret == 0) + return 0; + } + + i_assert(ctx->last_char == '\n'); + return 0; +} + +static int mbox_save_finish_headers(struct mbox_save_context *ctx) +{ + i_assert(ctx->eoh_offset == UOFF_T_MAX); + + /* append our own headers and ending empty line */ + ctx->extra_hdr_offset = ctx->output->offset; + if (o_stream_send(ctx->output, str_data(ctx->headers), + str_len(ctx->headers)) < 0) { + write_stream_error(ctx); + return -1; + } + ctx->eoh_offset = ctx->output->offset; + return 0; +} + +int mbox_save_continue(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + const unsigned char *data; + size_t i, size; + ssize_t ret; + + if (ctx->failed) + return -1; + + if (ctx->eoh_offset != UOFF_T_MAX) { + /* writing body */ + return mbox_save_body(ctx); + } + + while ((ret = i_stream_read_more(ctx->input, &data, &size)) > 0) { + for (i = 0; i < size; i++) { + if (data[i] == '\n' && + ((i == 0 && ctx->last_char == '\n') || + (i > 0 && data[i-1] == '\n'))) { + /* end of headers. we don't need to worry about + CRs because they're dropped */ + break; + } + } + if (i != size) { + /* found end of headers. write the rest of them + (not including the finishing empty line) */ + if (o_stream_send(ctx->output, data, i) < 0) { + write_stream_error(ctx); + return -1; + } + ctx->last_char = '\n'; + i_stream_skip(ctx->input, i + 1); + break; + } + + if (o_stream_send(ctx->output, data, size) < 0) { + write_stream_error(ctx); + return -1; + } + i_assert(size > 0); + ctx->last_char = data[size-1]; + i_stream_skip(ctx->input, size); + index_mail_cache_parse_continue(ctx->ctx.dest_mail); + } + if (ret == 0) + return 0; + if (ctx->input->stream_errno != 0) { + i_error("read(%s) failed: %s", i_stream_get_name(ctx->input), + i_stream_get_error(ctx->input)); + ctx->failed = TRUE; + return -1; + } + + i_assert(ctx->last_char == '\n'); + + if (ctx->mbox_md5_ctx != NULL) { + unsigned char hdr_md5_sum[16]; + + if (ctx->x_delivery_id_header != NULL) { + struct message_header_line hdr; + + i_zero(&hdr); + hdr.name = ctx->x_delivery_id_header; + hdr.name_len = sizeof("X-Delivery-ID")-1; + hdr.middle = (const unsigned char *)hdr.name + + hdr.name_len; + hdr.middle_len = 2; + hdr.value = hdr.full_value = + hdr.middle + hdr.middle_len; + hdr.value_len = strlen((const char *)hdr.value); + ctx->mbox->md5_v.more(ctx->mbox_md5_ctx, &hdr); + } + ctx->mbox->md5_v.finish(ctx->mbox_md5_ctx, hdr_md5_sum); + mail_index_update_ext(ctx->trans, ctx->seq, + ctx->mbox->md5hdr_ext_idx, + hdr_md5_sum, NULL); + } + + if (mbox_save_finish_headers(ctx) < 0) + return -1; + + /* write body */ + if (mbox_save_body_input(ctx) < 0) + return -1; + return ctx->input->eof ? 0 : mbox_save_body(ctx); +} + +int mbox_save_finish(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + if (!ctx->failed && ctx->eoh_offset == UOFF_T_MAX) + (void)mbox_save_finish_headers(ctx); + + if (ctx->output != NULL) { + /* make sure everything is written */ + if (o_stream_flush(ctx->output) < 0) + write_stream_error(ctx); + } + + ctx->finished = TRUE; + if (!ctx->failed) { + i_assert(ctx->output != NULL); + T_BEGIN { + if (mbox_write_content_length(ctx) < 0 || + mbox_append_lf(ctx) < 0) + ctx->failed = TRUE; + } T_END; + } + + index_mail_cache_parse_deinit(ctx->ctx.dest_mail, + ctx->ctx.data.received_date, + !ctx->failed); + if (ctx->input != NULL) + i_stream_destroy(&ctx->input); + + if (ctx->failed && ctx->mail_offset != UOFF_T_MAX) { + /* saving this mail failed - truncate back to beginning of it */ + i_assert(ctx->output != NULL); + (void)o_stream_flush(ctx->output); + if (ftruncate(ctx->mbox->mbox_fd, (off_t)ctx->mail_offset) < 0) + mbox_set_syscall_error(ctx->mbox, "ftruncate()"); + (void)o_stream_seek(ctx->output, ctx->mail_offset); + ctx->mail_offset = UOFF_T_MAX; + } + + if (ctx->seq != 0 && ctx->failed) { + index_storage_save_abort_last(&ctx->ctx, ctx->seq); + } + index_save_context_free(_ctx); + return ctx->failed ? -1 : 0; +} + +void mbox_save_cancel(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)mbox_save_finish(_ctx); +} + +static void mbox_transaction_save_deinit(struct mbox_save_context *ctx) +{ + o_stream_destroy(&ctx->output); + str_free(&ctx->headers); +} + +static void mbox_save_truncate(struct mbox_save_context *ctx) +{ + if (ctx->append_offset == UOFF_T_MAX || ctx->mbox->mbox_fd == -1) + return; + + i_assert(ctx->mbox->mbox_lock_type == F_WRLCK); + + /* failed, truncate file back to original size. output stream needs to + be flushed before truncating so unref() won't write anything. */ + if (ctx->output != NULL) + (void)o_stream_flush(ctx->output); + + if (ftruncate(ctx->mbox->mbox_fd, (off_t)ctx->append_offset) < 0) + mbox_set_syscall_error(ctx->mbox, "ftruncate()"); +} + +int mbox_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + struct mbox_mailbox *mbox = ctx->mbox; + struct stat st; + int ret = 0; + + i_assert(ctx->finished); + i_assert(mbox->mbox_fd != -1); + + if (fstat(mbox->mbox_fd, &st) < 0) { + mbox_set_syscall_error(mbox, "fstat()"); + ret = -1; + } + + if (ctx->synced) { + _t->changes->uid_validity = ctx->uid_validity; + mail_index_append_finish_uids(ctx->trans, 0, + &_t->changes->saved_uids); + + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, next_uid), + &ctx->next_uid, sizeof(ctx->next_uid), FALSE); + + if (ret == 0) { + mbox->mbox_hdr.sync_mtime = st.st_mtime; + mbox->mbox_hdr.sync_size = st.st_size; + mail_index_update_header_ext(ctx->trans, + mbox->mbox_ext_idx, + 0, &mbox->mbox_hdr, + sizeof(mbox->mbox_hdr)); + } + } + + if (ret == 0 && ctx->orig_atime != st.st_atime) { + /* try to set atime back to its original value. + (it'll fail with EPERM for shared mailboxes where we aren't + the file's owner) */ + struct utimbuf buf; + + buf.modtime = st.st_mtime; + buf.actime = ctx->orig_atime; + if (utime(mailbox_get_path(&mbox->box), &buf) < 0 && + errno != EPERM) + mbox_set_syscall_error(mbox, "utime()"); + } + + if (ctx->output != NULL) { + /* flush the final LF */ + if (o_stream_flush(ctx->output) < 0) + write_stream_error(ctx); + } + if (mbox->mbox_fd != -1 && !mbox->mbox_writeonly && + mbox->storage->storage.set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync(mbox->mbox_fd) < 0) { + mbox_set_syscall_error(mbox, "fdatasync()"); + mbox_save_truncate(ctx); + ret = -1; + } + } + + mbox_transaction_save_deinit(ctx); + if (ret < 0) + i_free(ctx); + return ret; +} + +void mbox_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result ATTR_UNUSED) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + i_assert(ctx->mbox->mbox_lock_type == F_WRLCK); + + if (ctx->synced) { + /* after saving mails with UIDs we need to update + the last-uid */ + (void)mbox_sync(ctx->mbox, MBOX_SYNC_HEADER | + MBOX_SYNC_REWRITE); + } + i_free(ctx); +} + +void mbox_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + if (!ctx->finished) + mbox_save_cancel(&ctx->ctx); + + mbox_save_truncate(ctx); + mbox_transaction_save_deinit(ctx); + i_free(ctx); +} diff --git a/src/lib-storage/index/mbox/mbox-settings.c b/src/lib-storage/index/mbox/mbox-settings.c new file mode 100644 index 0000000..1df2452 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-settings.c @@ -0,0 +1,55 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "mbox-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct mbox_settings) + +static const struct setting_define mbox_setting_defines[] = { + DEF(STR, mbox_read_locks), + DEF(STR, mbox_write_locks), + DEF(TIME, mbox_lock_timeout), + DEF(TIME, mbox_dotlock_change_timeout), + DEF(SIZE, mbox_min_index_size), + DEF(BOOL, mbox_dirty_syncs), + DEF(BOOL, mbox_very_dirty_syncs), + DEF(BOOL, mbox_lazy_writes), + DEF(ENUM, mbox_md5), + + SETTING_DEFINE_LIST_END +}; + +static const struct mbox_settings mbox_default_settings = { + .mbox_read_locks = "fcntl", + .mbox_write_locks = "dotlock fcntl", + .mbox_lock_timeout = 5*60, + .mbox_dotlock_change_timeout = 2*60, + .mbox_min_index_size = 0, + .mbox_dirty_syncs = TRUE, + .mbox_very_dirty_syncs = FALSE, + .mbox_lazy_writes = TRUE, + .mbox_md5 = "apop3d:all" +}; + +static const struct setting_parser_info mbox_setting_parser_info = { + .module_name = "mbox", + .defines = mbox_setting_defines, + .defaults = &mbox_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct mbox_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info +}; + +const struct setting_parser_info *mbox_get_setting_parser_info(void) +{ + return &mbox_setting_parser_info; +} diff --git a/src/lib-storage/index/mbox/mbox-settings.h b/src/lib-storage/index/mbox/mbox-settings.h new file mode 100644 index 0000000..eb99d82 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-settings.h @@ -0,0 +1,18 @@ +#ifndef MBOX_SETTINGS_H +#define MBOX_SETTINGS_H + +struct mbox_settings { + const char *mbox_read_locks; + const char *mbox_write_locks; + unsigned int mbox_lock_timeout; + unsigned int mbox_dotlock_change_timeout; + uoff_t mbox_min_index_size; + bool mbox_dirty_syncs; + bool mbox_very_dirty_syncs; + bool mbox_lazy_writes; + const char *mbox_md5; +}; + +const struct setting_parser_info *mbox_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-storage.c b/src/lib-storage/index/mbox/mbox-storage.c new file mode 100644 index 0000000..4b2bb35 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-storage.c @@ -0,0 +1,911 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "restrict-access.h" +#include "master-service.h" +#include "mailbox-list-private.h" +#include "mbox-storage.h" +#include "mbox-lock.h" +#include "mbox-file.h" +#include "mbox-sync-private.h" +#include "istream-raw-mbox.h" +#include "mail-copy.h" +#include "index-mail.h" + +#include <sys/stat.h> + +/* How often to touch the dotlock file when using KEEP_LOCKED flag */ +#define MBOX_LOCK_TOUCH_MSECS (10*1000) + +/* Assume that if atime < mtime, there are new mails. If it's good enough for + UW-IMAP, it's good enough for us. */ +#define STAT_GET_MARKED(st) \ + ((st).st_size == 0 ? MAILBOX_UNMARKED : \ + (st).st_atime < (st).st_mtime ? MAILBOX_MARKED : MAILBOX_UNMARKED) + +#define MBOX_LIST_CONTEXT(obj) \ + MODULE_CONTEXT_REQUIRE(obj, mbox_mailbox_list_module) + +struct mbox_mailbox_list { + union mailbox_list_module_context module_ctx; + const struct mbox_settings *set; +}; + +/* NOTE: must be sorted for istream-header-filter. Note that it's not such + a good idea to change this list, as the messages will then change from + client's point of view. So if you do it, change all mailboxes' UIDVALIDITY + so all caches are reset. */ +const char *mbox_hide_headers[] = { + "Content-Length", + "Status", + "X-IMAP", + "X-IMAPbase", + "X-Keywords", + "X-Status", + "X-UID" +}; +unsigned int mbox_hide_headers_count = N_ELEMENTS(mbox_hide_headers); + +/* A bit ugly duplification of the above list. It's safe to modify this list + without bad side effects, just keep the list sorted. */ +const char *mbox_save_drop_headers[] = { + "Content-Length", + "Status", + "X-Delivery-ID", + "X-IMAP", + "X-IMAPbase", + "X-Keywords", + "X-Status", + "X-UID" +}; +unsigned int mbox_save_drop_headers_count = N_ELEMENTS(mbox_save_drop_headers); + +extern struct mail_storage mbox_storage; +extern struct mailbox mbox_mailbox; + +static struct event_category event_category_mbox = { + .name = "mbox", + .parent = &event_category_storage, +}; + +static MODULE_CONTEXT_DEFINE_INIT(mbox_mailbox_list_module, + &mailbox_list_module_register); + +static void +mbox_set_syscall_error_str(struct mbox_mailbox *mbox, const char *function, + const char *error) +{ + i_assert(function != NULL); + + if (ENOQUOTA(errno)) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + } else { + const char *toobig_error = errno != EFBIG ? "" : + " (process was started with ulimit -f limit)"; + mailbox_set_critical(&mbox->box, "%s failed with mbox: %s%s", + function, error, toobig_error); + } +} + +void mbox_set_syscall_error(struct mbox_mailbox *mbox, const char *function) +{ + mbox_set_syscall_error_str(mbox, function, strerror(errno)); +} + +void mbox_istream_set_syscall_error(struct mbox_mailbox *mbox, + struct istream *input, + const char *function) +{ + errno = input->stream_errno; + mbox_set_syscall_error_str(mbox, function, i_stream_get_error(input)); +} + +void mbox_ostream_set_syscall_error(struct mbox_mailbox *mbox, + struct ostream *output, + const char *function) +{ + errno = output->stream_errno; + mbox_set_syscall_error_str(mbox, function, o_stream_get_error(output)); +} + +static int +mbox_list_get_path(struct mailbox_list *list, const char *name, + enum mailbox_list_path_type type, const char **path_r) +{ + struct mbox_mailbox_list *mlist = MBOX_LIST_CONTEXT(list); + const char *path, *p; + int ret; + + *path_r = NULL; + + ret = mlist->module_ctx.super.get_path(list, name, type, &path); + if (ret <= 0) + return ret; + + switch (type) { + case MAILBOX_LIST_PATH_TYPE_CONTROL: + case MAILBOX_LIST_PATH_TYPE_INDEX: + case MAILBOX_LIST_PATH_TYPE_INDEX_CACHE: + case MAILBOX_LIST_PATH_TYPE_LIST_INDEX: + if (name == NULL && type == MAILBOX_LIST_PATH_TYPE_CONTROL && + list->set.control_dir != NULL) { + /* kind of a kludge for backwards compatibility: + the subscriptions file is in the root control_dir + without .imap/ suffix */ + *path_r = path; + return 1; + } + if (name == NULL) { + *path_r = t_strconcat(path, "/"MBOX_INDEX_DIR_NAME, NULL); + return 1; + } + + p = strrchr(path, '/'); + if (p == NULL) + return 0; + + *path_r = t_strconcat(t_strdup_until(path, p), + "/"MBOX_INDEX_DIR_NAME"/", p+1, NULL); + break; + case MAILBOX_LIST_PATH_TYPE_DIR: + case MAILBOX_LIST_PATH_TYPE_ALT_DIR: + case MAILBOX_LIST_PATH_TYPE_MAILBOX: + case MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX: + case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE: + *path_r = path; + break; + } + return 1; +} + +static struct mail_storage *mbox_storage_alloc(void) +{ + struct mbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mbox storage", 512+256); + storage = p_new(pool, struct mbox_storage, 1); + storage->storage = mbox_storage; + storage->storage.pool = pool; + return &storage->storage; +} + +static int +mbox_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, + const char **error_r) +{ + struct mbox_storage *storage = MBOX_STORAGE(_storage); + struct stat st; + const char *dir; + + if (master_service_get_client_limit(master_service) > 1) { + /* we can't handle locking related problems. */ + *error_r = "mbox requires client_limit=1 for service"; + return -1; + } + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + + if (mailbox_list_get_root_path(ns->list, MAILBOX_LIST_PATH_TYPE_INDEX, &dir)) { + _storage->temp_path_prefix = p_strconcat(_storage->pool, dir, + "/", mailbox_list_get_temp_prefix(ns->list), NULL); + } + if (stat(ns->list->set.root_dir, &st) == 0 && !S_ISDIR(st.st_mode)) { + *error_r = t_strdup_printf( + "mbox root directory can't be a file: %s " + "(http://wiki2.dovecot.org/MailLocation/Mbox)", + ns->list->set.root_dir); + return -1; + } + return 0; +} + +static void mbox_storage_get_list_settings(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_FS; + if (set->subscription_fname == NULL) + set->subscription_fname = MBOX_SUBSCRIPTION_FILE_NAME; + + if (set->inbox_path == NULL && + strcasecmp(set->layout, MAILBOX_LIST_NAME_FS) == 0) { + set->inbox_path = t_strconcat(set->root_dir, "/inbox", NULL); + e_debug(ns->user->event, "mbox: INBOX defaulted to %s", set->inbox_path); + } +} + +static bool mbox_is_file(const char *path, const char *name, bool debug) +{ + struct stat st; + + if (stat(path, &st) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: stat(%s) failed: %m", + name, path); + } + return FALSE; + } + if (S_ISDIR(st.st_mode)) { + if (debug) { + i_debug("mbox autodetect: %s: is a directory (%s)", + name, path); + } + return FALSE; + } + if (access(path, R_OK|W_OK) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: no R/W access (%s)", + name, path); + } + return FALSE; + } + + if (debug) + i_debug("mbox autodetect: %s: yes (%s)", name, path); + return TRUE; +} + +static bool mbox_is_dir(const char *path, const char *name, bool debug) +{ + struct stat st; + + if (stat(path, &st) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: stat(%s) failed: %m", + name, path); + } + return FALSE; + } + if (!S_ISDIR(st.st_mode)) { + if (debug) { + i_debug("mbox autodetect: %s: is not a directory (%s)", + name, path); + } + return FALSE; + } + if (access(path, R_OK|W_OK|X_OK) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: no R/W/X access (%s)", + name, path); + } + return FALSE; + } + + if (debug) + i_debug("mbox autodetect: %s: yes (%s)", name, path); + return TRUE; +} + +static bool mbox_storage_is_root_dir(const char *dir, bool debug) +{ + if (mbox_is_dir(t_strconcat(dir, "/"MBOX_INDEX_DIR_NAME, NULL), + "has "MBOX_INDEX_DIR_NAME"/", debug)) + return TRUE; + if (mbox_is_file(t_strconcat(dir, "/inbox", NULL), "has inbox", debug)) + return TRUE; + if (mbox_is_file(t_strconcat(dir, "/mbox", NULL), "has mbox", debug)) + return TRUE; + return FALSE; +} + +static const char *mbox_storage_find_root_dir(const struct mail_namespace *ns) +{ + bool debug = ns->mail_set->mail_debug; + const char *home, *path; + + if (ns->owner == NULL || + mail_user_get_home(ns->owner, &home) <= 0) { + if (debug) + i_debug("maildir: Home directory not set"); + home = ""; + } + + path = t_strconcat(home, "/mail", NULL); + if (mbox_storage_is_root_dir(path, debug)) + return path; + + path = t_strconcat(home, "/Mail", NULL); + if (mbox_storage_is_root_dir(path, debug)) + return path; + return NULL; +} + +static const char * +mbox_storage_find_inbox_file(const char *user, bool debug) +{ + const char *path; + + path = t_strconcat("/var/mail/", user, NULL); + if (access(path, R_OK|W_OK) == 0) { + if (debug) + i_debug("mbox: INBOX exists (%s)", path); + return path; + } + if (debug) + i_debug("mbox: INBOX: access(%s, rw) failed: %m", path); + + path = t_strconcat("/var/spool/mail/", user, NULL); + if (access(path, R_OK|W_OK) == 0) { + if (debug) + i_debug("mbox: INBOX exists (%s)", path); + return path; + } + if (debug) + i_debug("mbox: INBOX: access(%s, rw) failed: %m", path); + return NULL; +} + +static bool mbox_storage_autodetect(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + bool debug = ns->mail_set->mail_debug; + const char *root_dir, *inbox_path; + + root_dir = set->root_dir; + inbox_path = set->inbox_path; + + if (root_dir != NULL) { + if (inbox_path == NULL && + mbox_is_file(root_dir, "INBOX file", debug)) { + /* using location=<INBOX> */ + inbox_path = root_dir; + root_dir = NULL; + } else if (!mbox_storage_is_root_dir(root_dir, debug)) + return FALSE; + } + if (root_dir == NULL) { + root_dir = mbox_storage_find_root_dir(ns); + if (root_dir == NULL) { + if (debug) + i_debug("mbox: couldn't find root dir"); + return FALSE; + } + } + if (inbox_path == NULL) { + inbox_path = mbox_storage_find_inbox_file(ns->user->username, + debug); + } + set->root_dir = root_dir; + set->inbox_path = inbox_path; + + mbox_storage_get_list_settings(ns, set); + return TRUE; +} + +static bool want_memory_indexes(struct mbox_storage *storage, const char *path) +{ + struct stat st; + + if (storage->set->mbox_min_index_size == 0) + return FALSE; + + if (stat(path, &st) < 0) { + if (errno == ENOENT) + st.st_size = 0; + else { + mail_storage_set_critical(&storage->storage, + "stat(%s) failed: %m", path); + return FALSE; + } + } + return (uoff_t)st.st_size < storage->set->mbox_min_index_size; +} + +static struct mailbox * +mbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mbox_mailbox *mbox; + pool_t pool; + + pool = pool_alloconly_create("mbox mailbox", 1024*3); + mbox = p_new(pool, struct mbox_mailbox, 1); + mbox->box = mbox_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = MBOX_STORAGE(storage); + mbox->mbox_fd = -1; + mbox->mbox_lock_type = F_UNLCK; + mbox->mbox_list_index_ext_id = (uint32_t)-1; + + if (strcmp(mbox->storage->set->mbox_md5, "apop3d") == 0) + mbox->md5_v = mbox_md5_apop3d; + else if (strcmp(mbox->storage->set->mbox_md5, "all") == 0) + mbox->md5_v = mbox_md5_all; + else { + i_fatal("Invalid mbox_md5 setting: %s", + mbox->storage->set->mbox_md5); + } + + if ((storage->flags & MAIL_STORAGE_FLAG_KEEP_HEADER_MD5) != 0) + mbox->mbox_save_md5 = TRUE; + return &mbox->box; +} + +static void mbox_lock_touch_timeout(struct mbox_mailbox *mbox) +{ + mbox_dotlock_touch(mbox); +} + +static int +mbox_mailbox_open_finish(struct mbox_mailbox *mbox, bool move_to_memory) +{ + if (index_storage_mailbox_open(&mbox->box, move_to_memory) < 0) + return -1; + + mbox->mbox_ext_idx = + mail_index_ext_register(mbox->box.index, "mbox", + sizeof(mbox->mbox_hdr), + sizeof(uint64_t), sizeof(uint64_t)); + mbox->md5hdr_ext_idx = + mail_index_ext_register(mbox->box.index, "header-md5", + 0, 16, 1); + return 0; +} + +static int mbox_mailbox_open_existing(struct mbox_mailbox *mbox) +{ + struct mailbox *box = &mbox->box; + const char *rootdir, *box_path = mailbox_get_path(box); + bool move_to_memory; + + move_to_memory = want_memory_indexes(mbox->storage, box_path); + + if (box->inbox_any || strcmp(box->name, "INBOX") == 0) { + /* if INBOX isn't under the root directory, it's probably in + /var/mail and we want to allow privileged dotlocking */ + rootdir = mailbox_list_get_root_forced(box->list, + MAILBOX_LIST_PATH_TYPE_DIR); + if (!str_begins(box_path, rootdir)) + mbox->mbox_privileged_locking = TRUE; + } + if ((box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0) { + if (mbox_lock(mbox, F_WRLCK, &mbox->mbox_global_lock_id) <= 0) + return -1; + + if (mbox->mbox_dotlock != NULL) { + mbox->keep_lock_to = + timeout_add(MBOX_LOCK_TOUCH_MSECS, + mbox_lock_touch_timeout, mbox); + } + } + return mbox_mailbox_open_finish(mbox, move_to_memory); +} + +static bool mbox_storage_is_readonly(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (index_storage_is_readonly(box)) + return TRUE; + + if (mbox_is_backend_readonly(mbox)) { + /* return read-only only if there are no private flags + (that are stored in index files) */ + if (mailbox_get_private_flags_mask(box) == 0) + return TRUE; + } + return FALSE; +} + +static int mbox_mailbox_open(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + struct stat st; + int ret; + + if (box->input != NULL) { + i_stream_ref(box->input); + mbox->mbox_file_stream = box->input; + mbox->backend_readonly = TRUE; + mbox->backend_readonly_set = TRUE; + mbox->no_mbox_file = TRUE; + return mbox_mailbox_open_finish(mbox, FALSE); + } + + ret = stat(mailbox_get_path(box), &st); + if (ret == 0) { + if (!S_ISDIR(st.st_mode)) + return mbox_mailbox_open_existing(mbox); + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + "Mailbox isn't selectable"); + return -1; + } else if (ENOTFOUND(errno)) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } else if (mail_storage_set_error_from_errno(box->storage)) { + return -1; + } else { + mailbox_set_critical(box, + "stat(%s) failed: %m", mailbox_get_path(box)); + return -1; + } +} + +static int +mbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + + if (update->uid_validity != 0 || update->min_next_uid != 0 || + !guid_128_is_empty(update->mailbox_guid)) { + mbox->sync_hdr_update = update; + ret = mbox_sync(mbox, MBOX_SYNC_HEADER | MBOX_SYNC_FORCE_SYNC | + MBOX_SYNC_REWRITE); + mbox->sync_hdr_update = NULL; + } + if (ret == 0) + ret = index_storage_mailbox_update(box, update); + return ret; +} + +static int create_inbox(struct mailbox *box) +{ + const char *inbox_path; + int fd; + + inbox_path = mailbox_get_path(box); + + fd = open(inbox_path, O_RDWR | O_CREAT | O_EXCL, 0660); + if (fd == -1 && errno == EACCES) { + /* try again with increased privileges */ + (void)restrict_access_use_priv_gid(); + fd = open(inbox_path, O_RDWR | O_CREAT | O_EXCL, 0660); + restrict_access_drop_priv_gid(); + } + if (fd != -1) { + i_close_fd(&fd); + return 0; + } else if (errno == EACCES) { + mailbox_set_critical(box, "%s", + mail_error_create_eacces_msg("open", inbox_path)); + return -1; + } else if (errno == EEXIST) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } else { + mailbox_set_critical(box, + "open(%s, O_CREAT) failed: %m", inbox_path); + return -1; + } +} + +static int +mbox_mailbox_create(struct mailbox *box, const struct mailbox_update *update, + bool directory) +{ + int fd, ret; + + if ((ret = index_storage_mailbox_create(box, directory)) <= 0) + return ret; + + if (box->inbox_any) { + if (create_inbox(box) < 0) + return -1; + } else { + /* create the mbox file */ + ret = mailbox_create_fd(box, mailbox_get_path(box), + O_RDWR | O_CREAT | O_EXCL, &fd); + if (ret < 0) + return -1; + if (ret == 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } + i_close_fd(&fd); + } + return update == NULL ? 0 : mbox_mailbox_update(box, update); +} + +static void mbox_mailbox_close(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + const struct mail_index_header *hdr; + enum mbox_sync_flags sync_flags = 0; + + if (mbox->mbox_stream != NULL && + istream_raw_mbox_is_corrupted(mbox->mbox_stream)) { + /* clear the corruption by forcing a full resync */ + sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC; + } + + if (box->view != NULL) { + hdr = mail_index_get_header(box->view); + if ((hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 && + !mbox_is_backend_readonly(mbox)) { + /* we've done changes to mbox which haven't been + written yet. do it now. */ + sync_flags |= MBOX_SYNC_REWRITE; + } + } + if (sync_flags != 0 && !mbox->invalid_mbox_file) + (void)mbox_sync(mbox, sync_flags); + + if (mbox->mbox_global_lock_id != 0) + mbox_unlock(mbox, mbox->mbox_global_lock_id); + timeout_remove(&mbox->keep_lock_to); + + mbox_file_close(mbox); + i_stream_destroy(&mbox->mbox_file_stream); + + index_storage_mailbox_close(box); +} + +static int +mbox_mailbox_get_guid(struct mbox_mailbox *mbox, guid_128_t guid_r) +{ + const char *errstr; + + if (mail_index_is_in_memory(mbox->box.index)) { + errstr = "Mailbox GUIDs are not permanent without index files"; + if (mbox->storage->set->mbox_min_index_size != 0) { + errstr = t_strconcat(errstr, + " (mbox_min_index_size is non-zero)", NULL); + } + mail_storage_set_error(mbox->box.storage, + MAIL_ERROR_NOTPOSSIBLE, errstr); + return -1; + } + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + + if (!guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + /* we have the GUID */ + } else if (mbox_file_open(mbox) < 0) + return -1; + else if (mbox->backend_readonly) { + mail_storage_set_error(mbox->box.storage, MAIL_ERROR_PERM, + "Can't set mailbox GUID to a read-only mailbox"); + return -1; + } else { + /* create another mailbox and sync */ + struct mailbox *box2; + struct mbox_mailbox *mbox2; + int ret; + + i_assert(mbox->mbox_lock_type == F_UNLCK); + box2 = mailbox_alloc(mbox->box.list, mbox->box.vname, 0); + ret = mailbox_sync(box2, 0); + mbox2 = MBOX_MAILBOX(box2); + memcpy(guid_r, mbox2->mbox_hdr.mailbox_guid, GUID_128_SIZE); + mailbox_free(&box2); + return ret; + } + memcpy(guid_r, mbox->mbox_hdr.mailbox_guid, GUID_128_SIZE); + return 0; +} + +static int +mbox_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + if ((items & MAILBOX_METADATA_GUID) != 0) { + if (mbox_mailbox_get_guid(mbox, metadata_r->guid) < 0) + return -1; + } + return 0; +} + +static void mbox_notify_changes(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (box->notify_callback == NULL) + mailbox_watch_remove_all(box); + else if (!mbox->no_mbox_file) + mailbox_watch_add(box, mailbox_get_path(box)); +} + +static bool +mbox_is_internal_name(struct mailbox_list *list ATTR_UNUSED, + const char *name) +{ + size_t len; + + /* don't allow *.lock files/dirs */ + len = strlen(name); + if (len > 5 && strcmp(name+len-5, ".lock") == 0) + return TRUE; + + return strcmp(name, MBOX_INDEX_DIR_NAME) == 0; +} + +static void mbox_storage_add_list(struct mail_storage *storage, + struct mailbox_list *list) +{ + struct mbox_mailbox_list *mlist; + + mlist = p_new(list->pool, struct mbox_mailbox_list, 1); + mlist->module_ctx.super = list->v; + mlist->set = mail_namespace_get_driver_settings(list->ns, storage); + + if (*list->set.maildir_name == '\0') { + /* have to use .imap/ directories */ + list->v.get_path = mbox_list_get_path; + } + list->v.is_internal_name = mbox_is_internal_name; + + MODULE_CONTEXT_SET(list, mbox_mailbox_list_module, mlist); +} + +static struct mailbox_transaction_context * +mbox_transaction_begin(struct mailbox *box, + enum mailbox_transaction_flags flags, + const char *reason) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + struct mbox_transaction_context *mt; + + if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) + mbox->external_transactions++; + + mt = i_new(struct mbox_transaction_context, 1); + index_transaction_init(&mt->t, box, flags, reason); + return &mt->t; +} + +static void +mbox_transaction_unlock(struct mailbox *box, unsigned int lock_id1, + unsigned int lock_id2) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (lock_id1 != 0) + mbox_unlock(mbox, lock_id1); + if (lock_id2 != 0) + mbox_unlock(mbox, lock_id2); + if (mbox->mbox_global_lock_id == 0) { + i_assert(mbox->box.transaction_count > 0); + i_assert(mbox->box.transaction_count > 1 || + mbox->external_transactions > 0 || + mbox->mbox_lock_type == F_UNLCK); + } else { + /* mailbox opened with MAILBOX_FLAG_KEEP_LOCKED */ + i_assert(mbox->mbox_lock_type == F_WRLCK); + } +} + +static int +mbox_transaction_commit(struct mailbox_transaction_context *t, + struct mail_transaction_commit_changes *changes_r) +{ + struct mbox_transaction_context *mt = MBOX_TRANSCTX(t); + struct mailbox *box = t->box; + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + unsigned int read_lock_id = mt->read_lock_id; + unsigned int write_lock_id = mt->write_lock_id; + int ret; + + if ((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) { + i_assert(mbox->external_transactions > 0); + mbox->external_transactions--; + } + + ret = index_transaction_commit(t, changes_r); + mbox_transaction_unlock(box, read_lock_id, write_lock_id); + return ret; +} + +static void +mbox_transaction_rollback(struct mailbox_transaction_context *t) +{ + struct mbox_transaction_context *mt = MBOX_TRANSCTX(t); + struct mailbox *box = t->box; + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + unsigned int read_lock_id = mt->read_lock_id; + unsigned int write_lock_id = mt->write_lock_id; + + if ((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) { + i_assert(mbox->external_transactions > 0); + mbox->external_transactions--; + } + + index_transaction_rollback(t); + mbox_transaction_unlock(box, read_lock_id, write_lock_id); +} + +bool mbox_is_backend_readonly(struct mbox_mailbox *mbox) +{ + if (!mbox->backend_readonly_set) { + mbox->backend_readonly_set = TRUE; + if (access(mailbox_get_path(&mbox->box), R_OK|W_OK) < 0 && + errno == EACCES) + mbox->backend_readonly = TRUE; + } + return mbox->backend_readonly; +} + +struct mail_storage mbox_storage = { + .name = MBOX_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_MAILBOX_IS_FILE | + MAIL_STORAGE_CLASS_FLAG_OPEN_STREAMS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS, + .event_category = &event_category_mbox, + + .v = { + mbox_get_setting_parser_info, + mbox_storage_alloc, + mbox_storage_create, + index_storage_destroy, + mbox_storage_add_list, + mbox_storage_get_list_settings, + mbox_storage_autodetect, + mbox_mailbox_alloc, + NULL, + NULL, + } +}; + +struct mailbox mbox_mailbox = { + .v = { + mbox_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mbox_mailbox_open, + mbox_mailbox_close, + index_storage_mailbox_free, + mbox_mailbox_create, + mbox_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mbox_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + mbox_list_index_has_changed, + mbox_list_index_update_sync, + mbox_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + mbox_notify_changes, + mbox_transaction_begin, + mbox_transaction_commit, + mbox_transaction_rollback, + NULL, + index_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + mbox_save_alloc, + mbox_save_begin, + mbox_save_continue, + mbox_save_finish, + mbox_save_cancel, + mail_storage_copy, + mbox_transaction_save_commit_pre, + mbox_transaction_save_commit_post, + mbox_transaction_save_rollback, + index_storage_is_inconsistent + } +}; diff --git a/src/lib-storage/index/mbox/mbox-storage.h b/src/lib-storage/index/mbox/mbox-storage.h new file mode 100644 index 0000000..e35a795 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-storage.h @@ -0,0 +1,115 @@ +#ifndef MBOX_STORAGE_H +#define MBOX_STORAGE_H + +#include "index-storage.h" +#include "mbox-settings.h" +#include "mbox-md5.h" + +/* Padding to leave in X-Keywords header when rewriting mbox */ +#define MBOX_HEADER_PADDING 50 +/* Don't write Content-Length header unless it's value is larger than this. */ +#define MBOX_MIN_CONTENT_LENGTH_SIZE 1024 + +#define MBOX_STORAGE_NAME "mbox" +#define MBOX_SUBSCRIPTION_FILE_NAME ".subscriptions" +#define MBOX_INDEX_DIR_NAME ".imap" +#define MBOX_UIDVALIDITY_FNAME "dovecot-uidvalidity" + +struct mbox_index_header { + uint64_t sync_size; + uint32_t sync_mtime; + uint8_t dirty_flag; + uint8_t unused[3]; + guid_128_t mailbox_guid; +}; + +struct mbox_list_index_record { + uint32_t mtime; + uint32_t size; +}; + +struct mbox_storage { + struct mail_storage storage; + + const struct mbox_settings *set; + enum mbox_lock_type *read_locks; + enum mbox_lock_type *write_locks; + bool lock_settings_initialized:1; +}; + +struct mbox_mailbox { + struct mailbox box; + struct mbox_storage *storage; + + int mbox_fd; + struct istream *mbox_stream, *mbox_file_stream; + int mbox_lock_type; + dev_t mbox_dev; + ino_t mbox_ino; + unsigned int mbox_excl_locks, mbox_shared_locks; + struct dotlock *mbox_dotlock; + unsigned int mbox_lock_id, mbox_global_lock_id; + struct timeout *keep_lock_to; + bool mbox_writeonly; + unsigned int external_transactions; + + uint32_t mbox_ext_idx, md5hdr_ext_idx, mbox_list_index_ext_id; + struct mbox_index_header mbox_hdr; + const struct mailbox_update *sync_hdr_update; + + struct mbox_md5_vfuncs md5_v; + + bool no_mbox_file:1; + bool invalid_mbox_file:1; + bool mbox_broken_offsets:1; + bool mbox_save_md5:1; + bool mbox_dotlocked:1; + bool mbox_used_privileges:1; + bool mbox_privileged_locking:1; + bool syncing:1; + bool backend_readonly:1; + bool backend_readonly_set:1; +}; + +struct mbox_transaction_context { + struct mailbox_transaction_context t; + union mail_index_transaction_module_context module_ctx; + + unsigned int read_lock_id; + unsigned int write_lock_id; +}; + +#define MBOX_STORAGE(s) container_of(s, struct mbox_storage, storage) +#define MBOX_MAILBOX(s) container_of(s, struct mbox_mailbox, box) +#define MBOX_TRANSCTX(s) container_of(s, struct mbox_transaction_context, t) + +extern struct mail_vfuncs mbox_mail_vfuncs; +extern const char *mbox_hide_headers[], *mbox_save_drop_headers[]; +extern unsigned int mbox_hide_headers_count, mbox_save_drop_headers_count; + +void mbox_set_syscall_error(struct mbox_mailbox *mbox, const char *function); +void mbox_istream_set_syscall_error(struct mbox_mailbox *mbox, + struct istream *input, + const char *function); +void mbox_ostream_set_syscall_error(struct mbox_mailbox *mbox, + struct ostream *output, + const char *function); + +struct mailbox_sync_context * +mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +struct mail_save_context * +mbox_save_alloc(struct mailbox_transaction_context *_t); +int mbox_save_begin(struct mail_save_context *ctx, struct istream *input); +int mbox_save_continue(struct mail_save_context *ctx); +int mbox_save_finish(struct mail_save_context *ctx); +void mbox_save_cancel(struct mail_save_context *ctx); + +int mbox_transaction_save_commit_pre(struct mail_save_context *ctx); +void mbox_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void mbox_transaction_save_rollback(struct mail_save_context *ctx); + +bool mbox_is_backend_readonly(struct mbox_mailbox *mbox); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-sync-list-index.c b/src/lib-storage/index/mbox/mbox-sync-list-index.c new file mode 100644 index 0000000..934b7ea --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-list-index.c @@ -0,0 +1,109 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" + +static unsigned int +mbox_list_get_ext_id(struct mbox_mailbox *mbox, + struct mail_index_view *view) +{ + if (mbox->mbox_list_index_ext_id == (uint32_t)-1) { + mbox->mbox_list_index_ext_id = + mail_index_ext_register(mail_index_view_get_index(view), + "mbox", 0, + sizeof(struct mbox_list_index_record), + sizeof(uint32_t)); + } + return mbox->mbox_list_index_ext_id; +} + +int mbox_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, const char **reason_r) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + const struct mbox_list_index_record *rec; + const void *data; + const char *path; + struct stat st; + uint32_t ext_id; + bool expunged; + int ret; + + ret = index_storage_list_index_has_changed(box, list_view, seq, + quick, reason_r); + if (ret != 0 || box->storage->set->mailbox_list_index_very_dirty_syncs) + return ret; + + ext_id = mbox_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + rec = data; + + if (rec == NULL) { + *reason_r = "mbox record is missing"; + return 1; + } else if (expunged) { + *reason_r = "mbox record is expunged"; + return 1; + } else if (rec->mtime == 0) { + /* not synced */ + *reason_r = "mbox record mtime=0"; + return 1; + } + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path); + if (ret < 0) + return ret; + i_assert(ret > 0); + + if (stat(path, &st) < 0) { + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return -1; + } + if ((time_t)rec->mtime != st.st_mtime) { + *reason_r = t_strdup_printf( + "mbox record mtime changed %u != %"PRIdTIME_T, + rec->mtime, st.st_mtime); + return 1; + } + uint32_t new_size = (uint32_t)(st.st_size & 0xffffffffU); + if (rec->size != new_size) { + *reason_r = t_strdup_printf("mbox record size changed %u != %u", + rec->size, new_size); + return 1; + } + return 0; +} + +void mbox_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + struct mail_index_view *list_view; + const struct mbox_index_header *mhdr = &mbox->mbox_hdr; + const struct mbox_list_index_record *old_rec; + struct mbox_list_index_record new_rec; + const void *data; + uint32_t ext_id; + bool expunged; + + index_storage_list_index_update_sync(box, trans, seq); + + /* get the current record */ + list_view = mail_index_transaction_get_view(trans); + ext_id = mbox_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + if (expunged) + return; + old_rec = data; + + i_zero(&new_rec); + new_rec.mtime = mhdr->sync_mtime; + new_rec.size = mhdr->sync_size & 0xffffffffU; + + if (old_rec == NULL || + memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0) + mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL); +} diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c new file mode 100644 index 0000000..bbc2da2 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c @@ -0,0 +1,616 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* MD5 header summing logic was pretty much copy&pasted from popa3d by + Solar Designer */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "istream.h" +#include "str.h" +#include "write-full.h" +#include "message-parser.h" +#include "mail-index.h" +#include "mbox-storage.h" +#include "mbox-md5.h" +#include "mbox-sync-private.h" + + +#define IS_LWSP_LF(c) (IS_LWSP(c) || (c) == '\n') + +struct mbox_sync_header_func { + const char *header; + bool (*func)(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr); +}; + +struct mbox_flag_type mbox_status_flags[] = { + { 'R', MAIL_SEEN }, + { 'O', MBOX_NONRECENT_KLUDGE }, + { 0, 0 } +}; + +struct mbox_flag_type mbox_xstatus_flags[] = { + { 'A', MAIL_ANSWERED }, + { 'F', MAIL_FLAGGED }, + { 'T', MAIL_DRAFT }, + { 'D', MAIL_DELETED }, + { 0, 0 } +}; + +static void parse_trailing_whitespace(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + size_t i, space = 0; + + /* the value may contain newlines. we can't count whitespace before + and after it as a single contiguous whitespace block, as that may + get us into situation where removing whitespace goes eg. + " \n \n" -> " \n\n" which would then be treated as end of headers. + + that could probably be avoided by being careful, but as newlines + should never be there (we don't generate them), it's not worth the + trouble. */ + + for (i = hdr->full_value_len; i > 0; i--) { + if (!IS_LWSP(hdr->full_value[i-1])) + break; + space++; + } + + if ((ssize_t)space > ctx->mail.space) { + i_assert(space != 0); + ctx->mail.offset = ctx->hdr_offset + str_len(ctx->header) + i; + ctx->mail.space = space; + } +} + +static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr) +{ + int i; + + for (i = 0; flags[i].chr != 0; i++) { + if (flags[i].chr == chr) + return flags[i].flag; + } + + return 0; +} + +static bool parse_status_flags(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr, + struct mbox_flag_type *flags_list) +{ + enum mail_flags flag; + size_t i; + bool duplicates = FALSE; + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + for (i = 0; i < hdr->full_value_len; i++) { + flag = mbox_flag_find(flags_list, hdr->full_value[i]); + if ((ctx->mail.flags & flag) != 0) + duplicates = TRUE; + else + ctx->mail.flags |= flag; + } + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + return duplicates; +} + +static bool parse_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + if (parse_status_flags(ctx, hdr, mbox_status_flags)) + ctx->mail.status_broken = TRUE; + ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); + return TRUE; +} + +static bool parse_x_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + if (parse_status_flags(ctx, hdr, mbox_xstatus_flags)) + ctx->mail.xstatus_broken = TRUE; + ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header); + return TRUE; +} + +static void +parse_imap_keywords_list(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr, size_t pos) +{ + struct mailbox *box = &ctx->sync_ctx->mbox->box; + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const char *keyword, *error; + size_t keyword_start; + unsigned int idx, count; + + count = 0; + while (pos < hdr->full_value_len) { + if (IS_LWSP_LF(hdr->full_value[pos])) { + pos++; + continue; + } + + /* read the keyword */ + keyword_start = pos; + for (; pos < hdr->full_value_len; pos++) { + if (IS_LWSP_LF(hdr->full_value[pos])) + break; + } + + /* add it to index's keyword list if it's not there already */ + keyword = t_strndup(hdr->full_value + keyword_start, + pos - keyword_start); + if (mailbox_keyword_is_valid(&ctx->sync_ctx->mbox->box, + keyword, &error)) { + mail_index_keyword_lookup_or_create(box->index, + keyword, &idx); + } + count++; + } + + if (count != array_count(ibox->keyword_names)) { + /* need to update this list */ + ctx->imapbase_rewrite = TRUE; + ctx->need_rewrite = TRUE; + } +} + +static bool parse_x_imap_base(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + size_t i, j, uid_last_pos; + uint32_t uid_validity, uid_last; + + if (ctx->seq != 1 || ctx->seen_imapbase || + ctx->sync_ctx->renumber_uids) { + /* Valid only in first message */ + return FALSE; + } + + /* <uid-validity> 10x<uid-last> */ + for (i = 0, uid_validity = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') { + if (hdr->full_value[i] != ' ') + return FALSE; + break; + } + uid_validity = uid_validity * 10 + (hdr->full_value[i] - '0'); + } + + if (uid_validity == 0) { + /* broken */ + return FALSE; + } + + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) + break; + } + uid_last_pos = i; + + for (uid_last = 0, j = 0; i < hdr->full_value_len; i++, j++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') { + if (!IS_LWSP_LF(hdr->full_value[i])) + return FALSE; + break; + } + uid_last = uid_last * 10 + (hdr->full_value[i] - '0'); + } + + if (j != 10 || + hdr->full_value_offset != ctx->hdr_offset + str_len(ctx->header)) { + /* uid-last field must be exactly 10 characters to make + rewriting it easier. also don't try to do this if some + headers have been removed */ + ctx->imapbase_rewrite = TRUE; + ctx->need_rewrite = TRUE; + } else { + ctx->last_uid_value_start_pos = uid_last_pos; + ctx->sync_ctx->base_uid_last_offset = + hdr->full_value_offset + uid_last_pos; + } + + if (ctx->sync_ctx->base_uid_validity == 0) { + /* first time parsing this (ie. we're not rewriting). + save the values. */ + ctx->sync_ctx->base_uid_validity = uid_validity; + ctx->sync_ctx->base_uid_last = uid_last; + + if (ctx->sync_ctx->next_uid-1 <= uid_last) { + /* new messages have been added since our last sync. + just update our internal next_uid. */ + ctx->sync_ctx->next_uid = uid_last+1; + } else { + /* we need to rewrite the next-uid */ + ctx->need_rewrite = TRUE; + } + i_assert(ctx->sync_ctx->next_uid > ctx->sync_ctx->prev_msg_uid); + } + + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); + ctx->seen_imapbase = TRUE; + + T_BEGIN { + parse_imap_keywords_list(ctx, hdr, i); + } T_END; + parse_trailing_whitespace(ctx, hdr); + return TRUE; +} + +static bool parse_x_imap(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + if (!parse_x_imap_base(ctx, hdr)) + return FALSE; + + /* this is the c-client style "FOLDER INTERNAL DATA" message. + skip it. */ + ctx->mail.pseudo = TRUE; + return TRUE; +} + +static bool parse_x_keywords_real(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + struct mailbox *box = &ctx->sync_ctx->mbox->box; + ARRAY_TYPE(keyword_indexes) keyword_list; + const unsigned int *list; + string_t *keyword; + size_t keyword_start; + unsigned int i, idx, count; + size_t pos; + + if (array_is_created(&ctx->mail.keywords)) + return FALSE; /* duplicate header, delete */ + + /* read keyword indexes to temporary array first */ + keyword = t_str_new(128); + t_array_init(&keyword_list, 16); + + for (pos = 0; pos < hdr->full_value_len; ) { + if (IS_LWSP_LF(hdr->full_value[pos])) { + pos++; + continue; + } + + /* read the keyword string */ + keyword_start = pos; + for (; pos < hdr->full_value_len; pos++) { + if (IS_LWSP_LF(hdr->full_value[pos])) + break; + } + + str_truncate(keyword, 0); + str_append_data(keyword, hdr->full_value + keyword_start, + pos - keyword_start); + if (!mail_index_keyword_lookup(box->index, str_c(keyword), + &idx)) { + /* keyword wasn't found. that means the sent mail + originally contained X-Keywords header. Delete it. */ + return FALSE; + } + + /* check that the keyword isn't already added there. + we don't want duplicates. */ + list = array_get(&keyword_list, &count); + for (i = 0; i < count; i++) { + if (list[i] == idx) + break; + } + + if (i == count) + array_push_back(&keyword_list, &idx); + } + + /* once we know how many keywords there are, we can allocate the array + from mail_keyword_pool without wasting memory. */ + if (array_count(&keyword_list) > 0) { + p_array_init(&ctx->mail.keywords, + ctx->sync_ctx->mail_keyword_pool, + array_count(&keyword_list)); + array_append_array(&ctx->mail.keywords, &keyword_list); + } + + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); + parse_trailing_whitespace(ctx, hdr); + return TRUE; +} + +static bool parse_x_keywords(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + bool ret; + + T_BEGIN { + ret = parse_x_keywords_real(ctx, hdr); + } T_END; + return ret; +} + +static bool parse_x_uid(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uint32_t value = 0; + size_t i; + + if (ctx->mail.uid != 0) { + /* duplicate */ + return FALSE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) { + /* broken value */ + return FALSE; + } + } + + if (ctx->sync_ctx == NULL) { + /* we're in mbox_sync_parse_match_mail(). + don't do any extra checks. */ + ctx->mail.uid = value; + return TRUE; + } + + if (ctx->seq == 1 && !ctx->seen_imapbase) { + /* Don't bother allowing X-UID before X-IMAPbase + header. c-client doesn't allow it either, and this + way the UID doesn't have to be reset if X-IMAPbase + header isn't what we expect it to be. */ + return FALSE; + } + + if (value == ctx->sync_ctx->next_uid) { + /* X-UID is the next expected one. allow it because + we'd just use this UID anyway. X-IMAPbase header + still needs to be updated for this. */ + ctx->sync_ctx->next_uid++; + } else if (value > ctx->sync_ctx->next_uid) { + /* UID is larger than expected. Don't allow it because + incoming mails can contain untrusted X-UID fields, + causing possibly DoS if the UIDs get large enough. */ + ctx->mail.uid_broken = TRUE; + return FALSE; + } + + if (value <= ctx->sync_ctx->prev_msg_uid) { + /* broken - UIDs must be growing */ + ctx->mail.uid_broken = TRUE; + return FALSE; + } + + ctx->mail.uid = value; + /* if we had multiple X-UID headers, we could have + uid_broken=TRUE here. */ + ctx->mail.uid_broken = FALSE; + + if (ctx->sync_ctx->dest_first_mail && ctx->seq != 1) { + /* if we're expunging the first mail, delete this header since + otherwise X-IMAPbase header would be added after this, which + we don't like */ + return FALSE; + } + + ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); + ctx->parsed_uid = value; + parse_trailing_whitespace(ctx, hdr); + return TRUE; +} + +static bool parse_content_length(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uoff_t value = 0; + size_t i; + + if (ctx->content_length != UOFF_T_MAX) { + /* duplicate */ + return FALSE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) { + /* broken value */ + return FALSE; + } + } + + ctx->content_length = value; + return TRUE; +} + +static struct mbox_sync_header_func header_funcs[] = { + { "Content-Length", parse_content_length }, + { "Status", parse_status }, + { "X-IMAP", parse_x_imap }, + { "X-IMAPbase", parse_x_imap_base }, + { "X-Keywords", parse_x_keywords }, + { "X-Status", parse_x_status }, + { "X-UID", parse_x_uid } +}; + +static int mbox_sync_bsearch_header_func_cmp(const void *p1, const void *p2) +{ + const char *key = p1; + const struct mbox_sync_header_func *func = p2; + + return strcasecmp(key, func->header); +} + +int mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx) +{ + struct mbox_sync_context *sync_ctx = ctx->sync_ctx; + struct message_header_parser_ctx *hdr_ctx; + struct message_header_line *hdr; + struct mbox_sync_header_func *func; + struct mbox_md5_context *mbox_md5_ctx; + size_t line_start_pos; + int i, ret; + + ctx->hdr_offset = ctx->mail.offset; + ctx->mail.flags = MAIL_RECENT; /* default to having recent flag */ + + ctx->header_first_change = SIZE_MAX; + ctx->header_last_change = 0; + + for (i = 0; i < MBOX_HDR_COUNT; i++) + ctx->hdr_pos[i] = SIZE_MAX; + + ctx->content_length = UOFF_T_MAX; + str_truncate(ctx->header, 0); + + mbox_md5_ctx = ctx->sync_ctx->mbox->md5_v.init(); + + line_start_pos = 0; + hdr_ctx = message_parse_header_init(input, NULL, 0); + while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) { + if (hdr->eoh) { + ctx->have_eoh = TRUE; + break; + } + + if (!hdr->continued) { + line_start_pos = str_len(ctx->header); + str_append(ctx->header, hdr->name); + str_append_data(ctx->header, hdr->middle, hdr->middle_len); + } + + func = bsearch(hdr->name, header_funcs, + N_ELEMENTS(header_funcs), sizeof(*header_funcs), + mbox_sync_bsearch_header_func_cmp); + + if (func != NULL) { + if (hdr->continues) { + hdr->use_full_value = TRUE; + continue; + } + + if (!func->func(ctx, hdr)) { + /* this header is broken, remove it */ + ctx->need_rewrite = TRUE; + str_truncate(ctx->header, line_start_pos); + if (ctx->header_first_change == SIZE_MAX) { + ctx->header_first_change = + line_start_pos; + } + continue; + } + buffer_append(ctx->header, hdr->full_value, + hdr->full_value_len); + } else { + ctx->sync_ctx->mbox->md5_v.more(mbox_md5_ctx, hdr); + buffer_append(ctx->header, hdr->value, + hdr->value_len); + } + if (!hdr->no_newline) { + if (hdr->crlf_newline) + str_append_c(ctx->header, '\r'); + str_append_c(ctx->header, '\n'); + } + } + i_assert(ret != 0); + message_parse_header_deinit(&hdr_ctx); + + ctx->sync_ctx->mbox->md5_v.finish(mbox_md5_ctx, ctx->hdr_md5_sum); + + if ((ctx->seq == 1 && !ctx->seen_imapbase) || + (ctx->seq > 1 && sync_ctx->dest_first_mail)) { + /* missing X-IMAPbase */ + ctx->need_rewrite = TRUE; + if (sync_ctx->base_uid_validity == 0) { + /* figure out a new UIDVALIDITY for us. */ + sync_ctx->base_uid_validity = + sync_ctx->hdr->uid_validity != 0 && + !sync_ctx->renumber_uids ? + sync_ctx->hdr->uid_validity : + I_MAX((uint32_t)ioloop_time, 1); + } + } + + ctx->body_offset = input->v_offset; + if (input->stream_errno != 0) { + mbox_sync_set_critical(ctx->sync_ctx, "read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); + return -1; + } + return 0; +} + +bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox, + struct mail_index_view *view, uint32_t seq) +{ + struct mbox_sync_mail_context ctx; + struct message_header_parser_ctx *hdr_ctx; + struct message_header_line *hdr; + struct header_func *func; + struct mbox_md5_context *mbox_md5_ctx; + const void *data; + bool expunged; + uint32_t uid; + int ret; + + /* we only wish to be sure that this mail actually is what we expect + it to be. If there's X-UID header and it matches our UID, we use it. + Otherwise it could mean that the X-UID header is invalid and it's + just not yet been rewritten. In that case use MD5 sum, if it + exists. */ + + mail_index_lookup_uid(view, seq, &uid); + i_zero(&ctx); + mbox_md5_ctx = mbox->md5_v.init(); + + hdr_ctx = message_parse_header_init(mbox->mbox_stream, NULL, 0); + while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) { + if (hdr->eoh) + break; + + func = bsearch(hdr->name, header_funcs, + N_ELEMENTS(header_funcs), sizeof(*header_funcs), + mbox_sync_bsearch_header_func_cmp); + if (func != NULL) { + if (strcasecmp(hdr->name, "X-UID") == 0) { + if (hdr->continues) { + hdr->use_full_value = TRUE; + continue; + } + (void)parse_x_uid(&ctx, hdr); + + if (ctx.mail.uid == uid) + break; + } + } else { + mbox->md5_v.more(mbox_md5_ctx, hdr); + } + } + i_assert(ret != 0); + message_parse_header_deinit(&hdr_ctx); + + mbox->md5_v.finish(mbox_md5_ctx, ctx.hdr_md5_sum); + + if (ctx.mail.uid == uid) + return TRUE; + + /* match by MD5 sum */ + mbox->mbox_save_md5 = TRUE; + + mail_index_lookup_ext(view, seq, mbox->md5hdr_ext_idx, + &data, &expunged); + return data == NULL ? 0 : + memcmp(data, ctx.hdr_md5_sum, 16) == 0; +} diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h new file mode 100644 index 0000000..7585e4d --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-private.h @@ -0,0 +1,192 @@ +#ifndef MBOX_SYNC_PRIVATE_H +#define MBOX_SYNC_PRIVATE_H + +#include "md5.h" +#include "mail-index.h" + +#include <sys/stat.h> + +enum mbox_sync_flags { + MBOX_SYNC_HEADER = 0x02, + MBOX_SYNC_LOCK_READING = 0x04, + MBOX_SYNC_UNDIRTY = 0x08, + MBOX_SYNC_REWRITE = 0x10, + MBOX_SYNC_FORCE_SYNC = 0x20, + MBOX_SYNC_READONLY = 0x40 +}; + +struct mbox_flag_type { + char chr; + enum mail_flags flag; +}; + +enum header_position { + MBOX_HDR_STATUS, + MBOX_HDR_X_IMAPBASE, + MBOX_HDR_X_KEYWORDS, + MBOX_HDR_X_STATUS, + MBOX_HDR_X_UID, + + MBOX_HDR_COUNT +}; + +/* kludgy. swap MAIL_RECENT with MBOX_NONRECENT_KLUDGE when writing Status + header, because 'O' flag means non-recent but internally we want to use + recent flag. */ +#define MBOX_NONRECENT_KLUDGE MAIL_RECENT + +#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT_KLUDGE) +#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED) +extern struct mbox_flag_type mbox_status_flags[]; +extern struct mbox_flag_type mbox_xstatus_flags[]; + +struct mbox_sync_mail { + /* uid=0 can mean that this mail describes an expunged area or that + this is a pseudo message */ + uint32_t uid; + uint32_t idx_seq; + + ARRAY_TYPE(keyword_indexes) keywords; + uint8_t flags; + + bool uid_broken:1; + bool expunged:1; + bool pseudo:1; + bool status_broken:1; + bool xstatus_broken:1; + + uoff_t from_offset; + uoff_t body_size; + + /* following variables have a bit overloaded functionality: + + a) space <= 0 : offset points to beginning of headers. space is the + amount of space missing that is required to be able to rewrite + the headers + b) space > 0 : offset points to beginning of whitespace that can + be removed. space is the amount of data that can be removed from + there. note that the message may contain more whitespace + elsewhere. */ + uoff_t offset; + off_t space; +}; + +struct mbox_sync_mail_context { + struct mbox_sync_context *sync_ctx; + struct mbox_sync_mail mail; + + uint32_t seq; + uoff_t hdr_offset, body_offset; + + size_t header_first_change, header_last_change; + string_t *header; + + unsigned char hdr_md5_sum[16]; + + uoff_t content_length; + + size_t hdr_pos[MBOX_HDR_COUNT]; + uint32_t parsed_uid, last_uid_updated_value; + unsigned int last_uid_value_start_pos; + + bool have_eoh:1; + bool need_rewrite:1; + bool seen_imapbase:1; + bool updated:1; + bool recent:1; + bool dirty:1; + bool imapbase_rewrite:1; + bool imapbase_updated:1; +}; + +struct mbox_sync_context { + struct mbox_mailbox *mbox; + enum mbox_sync_flags flags; + struct istream *input, *file_input; + int write_fd; + + time_t orig_mtime, orig_atime; + uoff_t orig_size; + struct stat last_stat; + + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *t; + + struct mail_index_header reset_hdr; + const struct mail_index_header *hdr; + + string_t *header, *from_line; + + /* header state: */ + uint32_t base_uid_validity, base_uid_last; + uoff_t base_uid_last_offset; + + /* mail state: */ + ARRAY(struct mbox_sync_mail) mails; + struct index_sync_changes_context *sync_changes; + + /* per-mail pool */ + pool_t mail_keyword_pool; + /* used for mails[].keywords */ + pool_t saved_keywords_pool; + + uint32_t prev_msg_uid, next_uid, idx_next_uid; + uint32_t seq, idx_seq, need_space_seq; + uint32_t last_nonrecent_uid; + off_t expunged_space, space_diff; + + bool dest_first_mail:1; + bool first_mail_crlf_expunged:1; + + /* global flags: */ + bool keep_recent:1; + bool readonly:1; + bool delay_writes:1; + bool renumber_uids:1; + bool moved_offsets:1; + bool ext_modified:1; + bool index_reset:1; + bool errors:1; +}; + +int mbox_sync_header_refresh(struct mbox_mailbox *mbox); +int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags); +int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty); +void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx, + const char *fmt, ...) ATTR_FORMAT(2, 3); + +int mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx); +bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox, + struct mail_index_view *view, uint32_t seq); + +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx); +void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail); +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff); +int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + uoff_t end_offset, off_t move_diff, uoff_t extra_space, + uint32_t first_seq, uint32_t last_seq); + +int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset); +void mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx); +void mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty); +int mbox_move(struct mbox_sync_context *sync_ctx, + uoff_t dest, uoff_t source, uoff_t size); +void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx, + size_t pos, size_t need, size_t have); +void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size); +int mbox_sync_get_guid(struct mbox_mailbox *mbox); + +int mbox_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, + const char **reason_r); +void mbox_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c new file mode 100644 index 0000000..eeb4878 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c @@ -0,0 +1,615 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "write-full.h" +#include "message-parser.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" +#include "istream-raw-mbox.h" + +int mbox_move(struct mbox_sync_context *sync_ctx, + uoff_t dest, uoff_t source, uoff_t size) +{ + struct mbox_mailbox *mbox = sync_ctx->mbox; + struct istream *input; + struct ostream *output; + int ret; + + i_assert(source > 0 || (dest != 1 && dest != 2)); + i_assert(size < OFF_T_MAX); + + if (size == 0 || source == dest) + return 0; + + i_stream_sync(sync_ctx->input); + + output = o_stream_create_fd_file(sync_ctx->write_fd, UOFF_T_MAX, FALSE); + i_stream_seek(sync_ctx->file_input, source); + if (o_stream_seek(output, dest) < 0) { + mbox_ostream_set_syscall_error(sync_ctx->mbox, output, + "o_stream_seek()"); + o_stream_unref(&output); + return -1; + } + + /* we're moving data within a file. it really shouldn't be failing at + this point or we're corrupted. */ + input = i_stream_create_limit(sync_ctx->file_input, size); + o_stream_nsend_istream(output, input); + if (input->stream_errno != 0) { + mailbox_set_critical(&mbox->box, + "read() failed with mbox: %s", + i_stream_get_error(input)); + ret = -1; + } else if (output->stream_errno != 0) { + mailbox_set_critical(&mbox->box, + "write() failed with mbox: %s", + o_stream_get_error(output)); + ret = -1; + } else if (input->v_offset != size) { + mbox_sync_set_critical(sync_ctx, + "mbox_move(%"PRIuUOFF_T", %"PRIuUOFF_T", %"PRIuUOFF_T + ") moved only %"PRIuUOFF_T" bytes", + dest, source, size, input->v_offset); + ret = -1; + } else { + ret = 0; + } + i_stream_unref(&input); + + mbox_sync_file_updated(sync_ctx, FALSE); + o_stream_destroy(&output); + return ret; +} + +static int mbox_fill_space(struct mbox_sync_context *sync_ctx, + uoff_t offset, uoff_t size) +{ + unsigned char space[1024]; + + memset(space, ' ', sizeof(space)); + while (size > sizeof(space)) { + if (pwrite_full(sync_ctx->write_fd, space, + sizeof(space), offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + size -= sizeof(space); + } + + if (pwrite_full(sync_ctx->write_fd, space, size, offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + mbox_sync_file_updated(sync_ctx, TRUE); + return 0; +} + +void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + size_t data_size, pos, start_pos; + const unsigned char *data; + void *p; + + i_assert(size < SSIZE_T_MAX); + + if (ctx->mail.pseudo) + start_pos = ctx->hdr_pos[MBOX_HDR_X_IMAPBASE]; + else if (ctx->mail.space > 0) { + /* update the header using the existing offset. + otherwise we might chose wrong header and just decrease + the available space */ + start_pos = ctx->mail.offset - ctx->hdr_offset; + } else { + /* Append at the end of X-Keywords header, + or X-UID if it doesn't exist */ + start_pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != SIZE_MAX ? + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] : + ctx->hdr_pos[MBOX_HDR_X_UID]; + } + + data = str_data(ctx->header); + data_size = str_len(ctx->header); + i_assert(start_pos < data_size); + + for (pos = start_pos; pos < data_size; pos++) { + if (data[pos] == '\n') { + /* possibly continues in next line */ + if (pos+1 == data_size || !IS_LWSP(data[pos+1])) + break; + start_pos = pos+1; + } else if (!IS_LWSP(data[pos]) && data[pos] != '\r') { + start_pos = pos+1; + } + } + + /* pos points to end of header now, and start_pos to beginning + of whitespace. */ + mbox_sync_move_buffer(ctx, pos, size, 0); + + p = buffer_get_space_unsafe(ctx->header, pos, size); + memset(p, ' ', size); + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + ctx->header_last_change = SIZE_MAX; + + ctx->mail.space = (pos - start_pos) + size; + ctx->mail.offset = ctx->hdr_offset; + if (ctx->mail.space > 0) + ctx->mail.offset += start_pos; +} + +static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx, + size_t start_pos, size_t *size) +{ + const unsigned char *data; + size_t data_size, pos, last_line_pos; + + /* find the end of the LWSP */ + data = str_data(ctx->header); + data_size = str_len(ctx->header); + + for (pos = last_line_pos = start_pos; pos < data_size; pos++) { + if (data[pos] == '\n') { + /* possibly continues in next line */ + if (pos+1 == data_size || !IS_LWSP(data[pos+1])) { + data_size = pos; + break; + } + last_line_pos = pos+1; + } else if (!IS_LWSP(data[pos]) && data[pos] != '\r') { + start_pos = last_line_pos = pos+1; + } + } + + if (start_pos == data_size) + return; + + /* and remove what we can */ + if (ctx->header_first_change > start_pos) + ctx->header_first_change = start_pos; + ctx->header_last_change = SIZE_MAX; + + if (data_size - start_pos <= *size) { + /* remove it all */ + mbox_sync_move_buffer(ctx, start_pos, 0, data_size - start_pos); + *size -= data_size - start_pos; + return; + } + + /* we have more space than needed. since we're removing from + the beginning of header instead of end, we don't have to + worry about multiline-headers. */ + mbox_sync_move_buffer(ctx, start_pos, 0, *size); + if (last_line_pos <= start_pos + *size) + last_line_pos = start_pos; + else + last_line_pos -= *size; + data_size -= *size; + + *size = 0; + + if (ctx->mail.space < (off_t)(data_size - last_line_pos)) { + ctx->mail.space = data_size - last_line_pos; + ctx->mail.offset = ctx->hdr_offset; + if (ctx->mail.space > 0) + ctx->mail.offset += last_line_pos; + } +} + +static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + static enum header_position space_positions[] = { + MBOX_HDR_X_UID, + MBOX_HDR_X_KEYWORDS, + MBOX_HDR_X_IMAPBASE + }; + enum header_position pos; + int i; + + ctx->mail.space = 0; + ctx->mail.offset = ctx->hdr_offset; + + for (i = 0; i < 3 && size > 0; i++) { + pos = space_positions[i]; + if (ctx->hdr_pos[pos] != SIZE_MAX) { + mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos], + &size); + } + } + + /* FIXME: see if we could remove X-Keywords header completely */ +} + +static void mbox_sync_first_mail_written(struct mbox_sync_mail_context *ctx, + uoff_t hdr_offset) +{ + /* we wrote the first mail. update last-uid offset so we can find + it later */ + i_assert(ctx->last_uid_value_start_pos != 0); + i_assert(ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] != SIZE_MAX); + + ctx->sync_ctx->base_uid_last_offset = hdr_offset + + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] + + ctx->last_uid_value_start_pos; + + if (ctx->imapbase_updated) { + /* update so a) we don't try to update it later needlessly, + b) if we do actually update it, we see the correct value */ + ctx->sync_ctx->base_uid_last = ctx->last_uid_updated_value; + } +} + +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff) +{ + struct mbox_sync_context *sync_ctx = ctx->sync_ctx; + size_t old_hdr_size, new_hdr_size; + + i_assert(sync_ctx->mbox->mbox_lock_type == F_WRLCK); + + old_hdr_size = ctx->body_offset - ctx->hdr_offset; + new_hdr_size = str_len(ctx->header); + + if (new_hdr_size <= old_hdr_size) { + /* add space. note that we must call add_space() even if we're + not adding anything so mail.offset gets fixed. */ + mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size); + } else if (new_hdr_size > old_hdr_size) { + /* try removing the space where we can */ + mbox_sync_headers_remove_space(ctx, + new_hdr_size - old_hdr_size); + new_hdr_size = str_len(ctx->header); + + if (new_hdr_size <= old_hdr_size) { + /* good, we removed enough. */ + i_assert(new_hdr_size == old_hdr_size); + } else if (move_diff < 0 && + new_hdr_size - old_hdr_size <= (uoff_t)-move_diff) { + /* moving backwards - we can use the extra space from + it, just update expunged_space accordingly */ + i_assert(ctx->mail.space == 0); + i_assert(sync_ctx->expunged_space >= + (off_t)(new_hdr_size - old_hdr_size)); + sync_ctx->expunged_space -= new_hdr_size - old_hdr_size; + } else { + /* couldn't get enough space */ + i_assert(ctx->mail.space == 0); + ctx->mail.space = + -(ssize_t)(new_hdr_size - old_hdr_size); + return 0; + } + } + + i_assert(ctx->mail.space >= 0); + + if (ctx->header_first_change == SIZE_MAX && move_diff == 0) { + /* no changes actually. we get here if index sync record told + us to do something that was already there */ + return 1; + } + + if (move_diff != 0) { + /* forget about partial write optimizations */ + ctx->header_first_change = 0; + ctx->header_last_change = 0; + } + + if (ctx->header_last_change != SIZE_MAX && + ctx->header_last_change != 0) + str_truncate(ctx->header, ctx->header_last_change); + + if (pwrite_full(sync_ctx->write_fd, + str_data(ctx->header) + ctx->header_first_change, + str_len(ctx->header) - ctx->header_first_change, + (off_t)ctx->hdr_offset + (off_t)ctx->header_first_change + + move_diff) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + + if (sync_ctx->dest_first_mail && + (ctx->imapbase_updated || ctx->sync_ctx->base_uid_last != 0)) { + /* the position might have moved as a result of moving + whitespace */ + mbox_sync_first_mail_written(ctx, (off_t)ctx->hdr_offset + move_diff); + } + + mbox_sync_file_updated(sync_ctx, FALSE); + return 1; +} + +static int mbox_sync_read_next(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + struct mbox_sync_mail *mails, + uint32_t seq, uint32_t idx, + uoff_t expunged_space) +{ + unsigned int first_mail_expunge_extra; + uint32_t orig_next_uid; + + i_zero(mail_ctx); + mail_ctx->sync_ctx = sync_ctx; + mail_ctx->seq = seq; + mail_ctx->header = sync_ctx->header; + + if (istream_raw_mbox_get_header_offset(sync_ctx->input, + &mail_ctx->mail.offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get header offset for seq=%u", seq); + return -1; + } + mail_ctx->mail.body_size = mails[idx].body_size; + + orig_next_uid = sync_ctx->next_uid; + if (mails[idx].uid != 0) { + /* This will force the UID to be the one that we originally + assigned to it, regardless of whether it's broken or not in + the file. */ + sync_ctx->next_uid = mails[idx].uid; + sync_ctx->prev_msg_uid = mails[idx].uid - 1; + } else { + /* Pseudo mail shouldn't have X-UID header at all */ + i_assert(mails[idx].pseudo); + sync_ctx->prev_msg_uid = 0; + } + + first_mail_expunge_extra = 1 + + (sync_ctx->first_mail_crlf_expunged ? 1 : 0); + if (mails[idx].from_offset + + first_mail_expunge_extra - expunged_space != 0) { + sync_ctx->dest_first_mail = mails[idx].from_offset == 0; + } else { + /* we need to skip over the initial \n (it's already counted in + expunged_space) */ + sync_ctx->dest_first_mail = TRUE; + mails[idx].from_offset += first_mail_expunge_extra; + } + + if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0) + return -1; + i_assert(mail_ctx->mail.pseudo == mails[idx].pseudo); + + /* set next_uid back before updating the headers. this is important + if we're updating the first message to make X-IMAP[base] header + have the correct value. */ + sync_ctx->next_uid = orig_next_uid; + + if (mails[idx].space != 0) { + if (mails[idx].space < 0) { + /* remove all possible spacing before updating */ + mbox_sync_headers_remove_space(mail_ctx, SIZE_MAX); + } + mbox_sync_update_header_from(mail_ctx, &mails[idx]); + } else { + /* updating might just try to add headers and mess up our + calculations completely. so only add the EOH here. */ + if (mail_ctx->have_eoh) + str_append_c(mail_ctx->header, '\n'); + } + return 0; +} + +static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + struct mbox_sync_mail *mails, + uint32_t seq, uint32_t idx, uint32_t padding, + off_t move_diff, uoff_t expunged_space, + uoff_t end_offset, bool first_nonexpunged) +{ + struct mbox_sync_mail_context new_mail_ctx; + uoff_t offset, dest_offset; + size_t need_space; + + if (mail_ctx == NULL) { + if (mbox_sync_seek(sync_ctx, mails[idx].from_offset) < 0) + return -1; + + if (mbox_sync_read_next(sync_ctx, &new_mail_ctx, mails, seq, idx, + expunged_space) < 0) + return -1; + mail_ctx = &new_mail_ctx; + } else { + i_assert(seq == mail_ctx->seq); + if (mail_ctx->mail.space < 0) + mail_ctx->mail.space = 0; + i_stream_seek(sync_ctx->input, mail_ctx->body_offset); + } + + if (mail_ctx->mail.space <= 0) { + need_space = str_len(mail_ctx->header) - mail_ctx->mail.space - + (mail_ctx->body_offset - mail_ctx->hdr_offset); + if (need_space != (uoff_t)-mails[idx].space) { + /* this check works only if we're doing the first + write, or if the file size was changed externally */ + mbox_sync_file_update_ext_modified(sync_ctx); + + mbox_sync_set_critical(sync_ctx, + "seq=%u uid=%u uid_broken=%d " + "originally needed %"PRIuUOFF_T + " bytes, now needs %zu bytes", + seq, mails[idx].uid, mails[idx].uid_broken ? 1 : 0, + (uoff_t)-mails[idx].space, need_space); + return -1; + } + } + + if (first_nonexpunged && expunged_space > 0) { + /* move From-line (after parsing headers so we don't + overwrite them) */ + i_assert(mails[idx].from_offset >= expunged_space); + if (mbox_move(sync_ctx, mails[idx].from_offset - expunged_space, + mails[idx].from_offset, + mails[idx].offset - mails[idx].from_offset) < 0) + return -1; + } + + if (mails[idx].space == 0) { + /* don't touch spacing */ + } else if (padding < (uoff_t)mail_ctx->mail.space) { + mbox_sync_headers_remove_space(mail_ctx, mail_ctx->mail.space - + padding); + } else { + mbox_sync_headers_add_space(mail_ctx, padding - + mail_ctx->mail.space); + } + + /* move the body of this message and headers of next message forward, + then write the headers */ + offset = sync_ctx->input->v_offset; + dest_offset = offset + move_diff; + i_assert(offset <= end_offset); + if (mbox_move(sync_ctx, dest_offset, offset, end_offset - offset) < 0) + return -1; + + /* the header may actually be moved backwards if there was expunged + space which we wanted to remove */ + i_assert(dest_offset >= str_len(mail_ctx->header)); + dest_offset -= str_len(mail_ctx->header); + i_assert(dest_offset >= mails[idx].from_offset - expunged_space); + if (pwrite_full(sync_ctx->write_fd, str_data(mail_ctx->header), + str_len(mail_ctx->header), dest_offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + mbox_sync_file_updated(sync_ctx, TRUE); + + if (sync_ctx->dest_first_mail) { + mbox_sync_first_mail_written(mail_ctx, dest_offset); + sync_ctx->dest_first_mail = FALSE; + } + + mails[idx].offset = dest_offset + + (mail_ctx->mail.offset - mail_ctx->hdr_offset); + mails[idx].space = mail_ctx->mail.space; + return 0; +} + +int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + uoff_t end_offset, off_t move_diff, uoff_t extra_space, + uint32_t first_seq, uint32_t last_seq) +{ + struct mbox_sync_mail *mails; + uoff_t offset, dest_offset, next_end_offset, next_move_diff; + uoff_t start_offset, expunged_space; + uint32_t idx, first_nonexpunged_idx, padding_per_mail; + uint32_t orig_prev_msg_uid; + unsigned int count; + int ret = 0; + + i_assert(extra_space < OFF_T_MAX); + i_assert(sync_ctx->mbox->mbox_lock_type == F_WRLCK); + + mails = array_get_modifiable(&sync_ctx->mails, &count); + i_assert(count == last_seq - first_seq + 1); + + /* if there's expunges in mails[], we would get more correct balancing + by counting only them here. however, that might make us overwrite + data which hasn't yet been copied backwards. to avoid too much + complexity, we just leave all the rest of the extra space to first + mail */ + idx = last_seq - first_seq + 1; + padding_per_mail = extra_space / idx; + + /* after expunge the next mail must have been missing space, or we + would have moved it backwards already */ + expunged_space = 0; + start_offset = mails[0].from_offset; + for (first_nonexpunged_idx = 0;; first_nonexpunged_idx++) { + i_assert(first_nonexpunged_idx != idx); + if (!mails[first_nonexpunged_idx].expunged) + break; + expunged_space += mails[first_nonexpunged_idx].space; + } + i_assert(mails[first_nonexpunged_idx].space < 0); + + orig_prev_msg_uid = sync_ctx->prev_msg_uid; + + /* start moving backwards. */ + while (idx > first_nonexpunged_idx) { + idx--; + if (idx == first_nonexpunged_idx) { + /* give the rest of the extra space to first mail. + we might also have to move the mail backwards to + fill the expunged space */ + padding_per_mail = move_diff + (off_t)expunged_space + + (off_t)mails[idx].space; + } + + next_end_offset = mails[idx].offset; + + if (mails[idx].space <= 0 && !mails[idx].expunged) { + /* give space to this mail. end_offset is left to + contain this message's From-line (ie. below we + move only headers + body). */ + bool first_nonexpunged = idx == first_nonexpunged_idx; + + next_move_diff = -mails[idx].space; + if (mbox_sync_read_and_move(sync_ctx, mail_ctx, mails, + first_seq + idx, idx, + padding_per_mail, + move_diff, expunged_space, + end_offset, + first_nonexpunged) < 0) { + ret = -1; + break; + } + move_diff -= next_move_diff + mails[idx].space; + } else { + /* this mail provides more space. just move it forward + from the extra space offset and set end_offset to + point to beginning of extra space. that way the + header will be moved along with previous mail's + body. + + if this is expunged mail, we're moving following + mail's From-line and maybe headers. */ + offset = mails[idx].offset + mails[idx].space; + dest_offset = offset + move_diff; + i_assert(offset <= end_offset); + if (mbox_move(sync_ctx, dest_offset, offset, + end_offset - offset) < 0) { + ret = -1; + break; + } + + move_diff += mails[idx].space; + if (!mails[idx].expunged) { + move_diff -= padding_per_mail; + mails[idx].space = padding_per_mail; + + if (mbox_fill_space(sync_ctx, move_diff + + mails[idx].offset, + padding_per_mail) < 0) { + ret = -1; + break; + } + } + mails[idx].offset += move_diff; + } + mail_ctx = NULL; + + i_assert(move_diff >= 0 || idx == first_nonexpunged_idx); + i_assert(next_end_offset <= end_offset); + + end_offset = next_end_offset; + mails[idx].from_offset += move_diff; + } + + if (ret == 0) { + i_assert(mails[idx].from_offset == start_offset); + i_assert(move_diff + (off_t)expunged_space >= 0); + } + + mbox_sync_file_updated(sync_ctx, FALSE); + sync_ctx->prev_msg_uid = orig_prev_msg_uid; + return ret; +} diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c new file mode 100644 index 0000000..8442013 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-update.c @@ -0,0 +1,466 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "str.h" +#include "message-parser.h" +#include "index-storage.h" +#include "index-sync-changes.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" + +/* Line length when to wrap X-IMAP, X-IMAPbase and X-Keywords headers to next + line. Keep this pretty long, as after we wrap we lose compatibility with + UW-IMAP */ +#define KEYWORD_WRAP_LINE_LENGTH 1024 + +static void status_flags_append(struct mbox_sync_mail_context *ctx, + const struct mbox_flag_type *flags_list) +{ + int i; + + for (i = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + str_append_c(ctx->header, flags_list[i].chr); + } +} + +void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx, + size_t pos, size_t need, size_t have) +{ + ssize_t diff = (ssize_t)need - (ssize_t)have; + int i; + + i_assert(have < SSIZE_T_MAX); + + if (diff == 0) { + if (ctx->header_last_change < pos + have || + ctx->header_last_change == SIZE_MAX) + ctx->header_last_change = pos + have; + } else { + /* FIXME: if (diff < ctx->space && pos < ctx->offset) then + move the data only up to space offset and give/take the + space from there. update header_last_change accordingly. + (except pos and offset can't be compared directly) */ + ctx->header_last_change = SIZE_MAX; + for (i = 0; i < MBOX_HDR_COUNT; i++) { + if (ctx->hdr_pos[i] > pos && + ctx->hdr_pos[i] != SIZE_MAX) + ctx->hdr_pos[i] = (ssize_t)ctx->hdr_pos[i] + diff; + } + + if (ctx->mail.space > 0) { + i_assert(ctx->mail.offset + ctx->mail.space <= + ctx->hdr_offset + pos || + ctx->mail.offset > ctx->hdr_offset + pos + have); + if (ctx->mail.offset > ctx->hdr_offset + pos) { + /* free space offset moves */ + ctx->mail.offset = (ssize_t)ctx->mail.offset + diff; + } + } + + if (diff < 0) + str_delete(ctx->header, pos, -diff); + else { + ctx->header_last_change = SIZE_MAX; + buffer_copy(ctx->header, pos + diff, + ctx->header, pos, SIZE_MAX); + } + } +} + +static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos, + const struct mbox_flag_type *flags_list) +{ + unsigned char *data; + size_t size; + int i, need, have; + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + + /* how many bytes do we need? */ + for (i = 0, need = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + need++; + } + + /* how many bytes do we have now? */ + data = buffer_get_modifiable_data(ctx->header, &size); + for (have = 0; pos < size; pos++) { + if (data[pos] == '\n' || data[pos] == '\r') + break; + + /* see if this is unknown flag for us */ + for (i = 0; flags_list[i].chr != 0; i++) { + if (flags_list[i].chr == (char)data[pos]) + break; + } + + if (flags_list[i].chr != 0) + have++; + else { + /* save this one */ + data[pos-have] = data[pos]; + } + } + pos -= have; + mbox_sync_move_buffer(ctx, pos, need, have); + + /* @UNSAFE */ + data = buffer_get_space_unsafe(ctx->header, pos, need); + for (i = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + *data++ = flags_list[i].chr; + } + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; +} + +static void +keywords_append(struct mbox_sync_context *sync_ctx, string_t *dest, + const ARRAY_TYPE(keyword_indexes) *keyword_indexes_arr) +{ + struct index_mailbox_context *ibox = + INDEX_STORAGE_CONTEXT(&sync_ctx->mbox->box); + const char *const *keyword_names; + const unsigned int *keyword_indexes; + unsigned int i, idx_count, keywords_count; + size_t last_break; + + keyword_names = array_get(ibox->keyword_names, + &keywords_count); + keyword_indexes = array_get(keyword_indexes_arr, &idx_count); + + for (i = 0, last_break = str_len(dest); i < idx_count; i++) { + i_assert(keyword_indexes[i] < keywords_count); + + /* wrap the line whenever it gets too long */ + if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH) { + if (i > 0) + str_append_c(dest, ' '); + } else { + str_append(dest, "\n\t"); + last_break = str_len(dest); + } + str_append(dest, keyword_names[keyword_indexes[i]]); + } +} + +static void +keywords_append_all(struct mbox_sync_mail_context *ctx, string_t *dest, + size_t startpos) +{ + struct index_mailbox_context *ibox = + INDEX_STORAGE_CONTEXT(&ctx->sync_ctx->mbox->box); + const char *const *names; + const unsigned char *p; + unsigned int i, count; + size_t last_break; + + p = str_data(dest); + if (str_len(dest) - startpos < KEYWORD_WRAP_LINE_LENGTH) + last_break = startpos; + else { + /* set last_break to beginning of line */ + for (last_break = str_len(dest); last_break > 0; last_break--) { + if (p[last_break-1] == '\n') + break; + } + } + + names = array_get(ibox->keyword_names, &count); + for (i = 0; i < count; i++) { + /* wrap the line whenever it gets too long */ + if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH) + str_append_c(dest, ' '); + else { + str_append(dest, "\n\t"); + last_break = str_len(dest); + } + str_append(dest, names[i]); + } +} + +static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) +{ + size_t new_hdr_size, startpos; + + new_hdr_size = str_len(ctx->header); + if (new_hdr_size > 0 && + str_data(ctx->header)[new_hdr_size-1] != '\n') { + /* broken header - doesn't end with \n. fix it. */ + str_append_c(ctx->header, '\n'); + } + + if (ctx->sync_ctx->dest_first_mail && + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX) { + i_assert(ctx->sync_ctx->base_uid_validity != 0); + + str_append(ctx->header, "X-IMAPbase: "); + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); + /* startpos must start from identical position as when + updating */ + startpos = str_len(ctx->header); + str_printfa(ctx->header, "%u ", + ctx->sync_ctx->base_uid_validity); + + ctx->last_uid_updated_value = ctx->sync_ctx->next_uid-1; + ctx->last_uid_value_start_pos = str_len(ctx->header) - + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE]; + ctx->imapbase_updated = TRUE; + str_printfa(ctx->header, "%010u", ctx->last_uid_updated_value); + + keywords_append_all(ctx, ctx->header, startpos); + str_append_c(ctx->header, '\n'); + } + + if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX && !ctx->mail.pseudo) { + str_append(ctx->header, "X-UID: "); + ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); + str_printfa(ctx->header, "%u\n", ctx->mail.uid); + } + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + + if (ctx->hdr_pos[MBOX_HDR_STATUS] == SIZE_MAX && + (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) { + str_append(ctx->header, "Status: "); + ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); + status_flags_append(ctx, mbox_status_flags); + str_append_c(ctx->header, '\n'); + } + + if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == SIZE_MAX && + (ctx->mail.flags & XSTATUS_FLAGS_MASK) != 0) { + str_append(ctx->header, "X-Status: "); + ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header); + status_flags_append(ctx, mbox_xstatus_flags); + str_append_c(ctx->header, '\n'); + } + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + + if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX && + array_is_created(&ctx->mail.keywords) && + array_count(&ctx->mail.keywords) > 0) { + str_append(ctx->header, "X-Keywords: "); + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); + keywords_append(ctx->sync_ctx, ctx->header, + &ctx->mail.keywords); + str_append_c(ctx->header, '\n'); + } + + if (ctx->content_length == UOFF_T_MAX && + ctx->mail.body_size >= MBOX_MIN_CONTENT_LENGTH_SIZE) { + str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n", + ctx->mail.body_size); + } + + if (str_len(ctx->header) != new_hdr_size) { + if (ctx->header_first_change == SIZE_MAX) + ctx->header_first_change = new_hdr_size; + ctx->header_last_change = SIZE_MAX; + } + + if (ctx->have_eoh) + str_append_c(ctx->header, '\n'); +} + +static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx) +{ + if (ctx->hdr_pos[MBOX_HDR_STATUS] != SIZE_MAX) { + status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_STATUS], + mbox_status_flags); + } +} + +static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx) +{ + if (ctx->hdr_pos[MBOX_HDR_X_STATUS] != SIZE_MAX) { + status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_X_STATUS], + mbox_xstatus_flags); + } +} + +static void mbox_sync_update_line(struct mbox_sync_mail_context *ctx, + size_t pos, string_t *new_line) +{ + const char *hdr, *p; + uoff_t file_pos; + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + + /* set p = end of header, handle also wrapped headers */ + hdr = p = str_c(ctx->header) + pos; + for (;;) { + p = strchr(p, '\n'); + if (p == NULL) { + /* shouldn't really happen, but allow anyway.. */ + p = hdr + strlen(hdr); + break; + } + if (p[1] != '\t' && p[1] != ' ') + break; + p += 2; + } + + file_pos = pos + ctx->hdr_offset; + if (ctx->mail.space > 0 && ctx->mail.offset >= file_pos && + ctx->mail.offset < file_pos + (p - hdr)) { + /* extra space points to this line. remove it. */ + ctx->mail.offset = ctx->hdr_offset; + ctx->mail.space = 0; + } + + mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1); + buffer_copy(ctx->header, pos, new_line, 0, SIZE_MAX); +} + +static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx) +{ + string_t *str; + + if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX) + return; + + str = t_str_new(256); + if (array_is_created(&ctx->mail.keywords)) + keywords_append(ctx->sync_ctx, str, &ctx->mail.keywords); + str_append_c(str, '\n'); + mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_KEYWORDS], str); +} + +static void mbox_sync_update_x_imap_base(struct mbox_sync_mail_context *ctx) +{ + struct mbox_sync_context *sync_ctx = ctx->sync_ctx; + string_t *str; + + i_assert(sync_ctx->base_uid_validity != 0); + + if (!sync_ctx->dest_first_mail || + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX) + return; + + if (!ctx->imapbase_rewrite) { + /* uid-last might need updating, but we'll do it later by + writing it directly where needed. */ + return; + } + + /* a) keyword list changed, b) uid-last didn't use 10 digits */ + str = t_str_new(200); + str_printfa(str, "%u ", sync_ctx->base_uid_validity); + + ctx->last_uid_updated_value = sync_ctx->next_uid-1; + ctx->last_uid_value_start_pos = str_len(str); + ctx->imapbase_updated = TRUE; + str_printfa(str, "%010u", ctx->last_uid_updated_value); + + keywords_append_all(ctx, str, 0); + str_append_c(str, '\n'); + + mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_IMAPBASE], str); +} + +static void mbox_sync_update_x_uid(struct mbox_sync_mail_context *ctx) +{ + string_t *str; + + if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX || + ctx->mail.uid == ctx->parsed_uid) + return; + + str = t_str_new(64); + str_printfa(str, "%u\n", ctx->mail.uid); + mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_UID], str); +} + +static void mbox_sync_update_header_real(struct mbox_sync_mail_context *ctx) +{ + i_assert(ctx->mail.uid != 0 || ctx->mail.pseudo); + + if (!ctx->sync_ctx->keep_recent) + ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT); + + mbox_sync_update_status(ctx); + mbox_sync_update_xstatus(ctx); + mbox_sync_update_xkeywords(ctx); + + mbox_sync_update_x_imap_base(ctx); + mbox_sync_update_x_uid(ctx); + + mbox_sync_add_missing_headers(ctx); + ctx->updated = TRUE; +} + +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx) +{ + T_BEGIN { + mbox_sync_update_header_real(ctx); + } T_END; +} + +static void +mbox_sync_update_header_from_real(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail) +{ + if (mail->status_broken || + (ctx->mail.flags & STATUS_FLAGS_MASK) != + (mail->flags & STATUS_FLAGS_MASK) || + (ctx->mail.flags & MAIL_RECENT) != 0) { + ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(STATUS_FLAGS_MASK)) | + (mail->flags & STATUS_FLAGS_MASK); + if (!ctx->sync_ctx->keep_recent) + ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT); + mbox_sync_update_status(ctx); + } + if (mail->xstatus_broken || + (ctx->mail.flags & XSTATUS_FLAGS_MASK) != + (mail->flags & XSTATUS_FLAGS_MASK)) { + ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(XSTATUS_FLAGS_MASK)) | + (mail->flags & XSTATUS_FLAGS_MASK); + mbox_sync_update_xstatus(ctx); + } + if (!array_is_created(&mail->keywords) || + array_count(&mail->keywords) == 0) { + /* no keywords for this mail */ + if (array_is_created(&ctx->mail.keywords)) { + array_clear(&ctx->mail.keywords); + mbox_sync_update_xkeywords(ctx); + } + } else if (!array_is_created(&ctx->mail.keywords)) { + /* adding first keywords */ + p_array_init(&ctx->mail.keywords, + ctx->sync_ctx->mail_keyword_pool, + array_count(&mail->keywords)); + array_append_array(&ctx->mail.keywords, + &mail->keywords); + mbox_sync_update_xkeywords(ctx); + } else if (!array_cmp(&ctx->mail.keywords, &mail->keywords)) { + /* keywords changed. */ + array_clear(&ctx->mail.keywords); + array_append_array(&ctx->mail.keywords, + &mail->keywords); + mbox_sync_update_xkeywords(ctx); + } + + i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid); + ctx->mail.uid = mail->uid; + + mbox_sync_update_x_imap_base(ctx); + mbox_sync_update_x_uid(ctx); + mbox_sync_add_missing_headers(ctx); +} + +void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail) +{ + T_BEGIN { + mbox_sync_update_header_from_real(ctx, mail); + } T_END; +} diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c new file mode 100644 index 0000000..0d2aa7f --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -0,0 +1,2066 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* + Modifying mbox can be slow, so we try to do it all at once minimizing the + required disk I/O. We may need to: + + - Update message flags in Status, X-Status and X-Keywords headers + - Write missing X-UID and X-IMAPbase headers + - Write missing or broken Content-Length header if there's space + - Expunge specified messages + + Here's how we do it: + + - Start reading the mails from the beginning + - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end + of them, remember how much each message has and offset to beginning of the + padding + - If header needs to be rewritten and there's enough space, do it + - If we didn't have enough space, remember how much was missing + - Continue reading and counting the padding in each message. If available + padding is enough to rewrite all the previous messages needing it, do it + - When we encounter expunged message, treat all of it as padding and + rewrite previous messages if needed (and there's enough space). + Afterwards keep moving messages backwards to fill the expunged space. + Moving is done by rewriting each message's headers, with possibly adding + missing Content-Length header and padding. Message bodies are moved + without modifications. + - If we encounter end of file, grow the file and rewrite needed messages + - Rewriting is done by moving message body forward, rewriting message's + header and doing the same for previous message, until all of them are + rewritten. +*/ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "hostpid.h" +#include "istream.h" +#include "file-set-size.h" +#include "str.h" +#include "read-full.h" +#include "write-full.h" +#include "sleep.h" +#include "message-date.h" +#include "istream-raw-mbox.h" +#include "mbox-storage.h" +#include "index-sync-changes.h" +#include "mailbox-uidvalidity.h" +#include "mailbox-recent-flags.h" +#include "mbox-from.h" +#include "mbox-file.h" +#include "mbox-lock.h" +#include "mbox-sync-private.h" + +#include <stddef.h> +#include <utime.h> +#include <sys/stat.h> + +/* The text below was taken exactly as c-client wrote it to my mailbox, + so it's probably copyrighted by University of Washington. */ +#define PSEUDO_MESSAGE_BODY \ +"This text is part of the internal format of your mail folder, and is not\n" \ +"a real message. It is created automatically by the mail system software.\n" \ +"If deleted, important folder data will be lost, and it will be re-created\n" \ +"with the data reset to initial values.\n" + +void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx, + const char *fmt, ...) +{ + va_list va; + + sync_ctx->errors = TRUE; + if (sync_ctx->ext_modified) { + mailbox_set_critical(&sync_ctx->mbox->box, + "mbox was modified while we were syncing, " + "check your locking settings"); + } + + va_start(va, fmt); + mailbox_set_critical(&sync_ctx->mbox->box, + "Sync failed for mbox: %s", + t_strdup_vprintf(fmt, va)); + va_end(va); +} + +int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset) +{ + if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Unexpectedly lost From-line at offset %"PRIuUOFF_T, + from_offset); + return -1; + } + return 0; +} + +void mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx) +{ + struct stat st; + + /* Do this even if ext_modified is already set. Expunging code relies + on last_stat being updated. */ + if (fstat(sync_ctx->write_fd, &st) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "fstat()"); + return; + } + + if (st.st_size != sync_ctx->last_stat.st_size || + (sync_ctx->last_stat.st_mtime != 0 && + !CMP_ST_MTIME(&st, &sync_ctx->last_stat))) + sync_ctx->ext_modified = TRUE; + + sync_ctx->last_stat = st; +} + +void mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty) +{ + if (dirty) { + /* just mark the stat as dirty. */ + sync_ctx->last_stat.st_mtime = 0; + return; + } + if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "fstat()"); + i_stream_sync(sync_ctx->input); +} + +static int +mbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + uoff_t offset; + + /* get EOF */ + (void)istream_raw_mbox_get_header_offset(sync_ctx->input, &offset); + if (istream_raw_mbox_is_eof(sync_ctx->input)) + return 0; + + p_clear(sync_ctx->mail_keyword_pool); + i_zero(mail_ctx); + mail_ctx->sync_ctx = sync_ctx; + mail_ctx->seq = ++sync_ctx->seq; + mail_ctx->header = sync_ctx->header; + + mail_ctx->mail.from_offset = + istream_raw_mbox_get_start_offset(sync_ctx->input); + if (istream_raw_mbox_get_header_offset(sync_ctx->input, &mail_ctx->mail.offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get header offset for seq=%u", mail_ctx->seq); + return -1; + } + + if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0) + return -1; + if (istream_raw_mbox_is_corrupted(sync_ctx->input)) + return -1; + + i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset || + sync_ctx->input->eof); + + if (istream_raw_mbox_get_body_size(sync_ctx->input, + mail_ctx->content_length, + &mail_ctx->mail.body_size) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get body size for seq=%u", mail_ctx->seq); + return -1; + } + i_assert(mail_ctx->mail.body_size < OFF_T_MAX); + + if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 && + !mail_ctx->mail.pseudo) { + if (!sync_ctx->keep_recent) { + /* need to add 'O' flag to Status-header */ + mail_ctx->need_rewrite = TRUE; + } + mail_ctx->recent = TRUE; + } + return 1; +} + +static void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx, + uint32_t uid, bool *sync_expunge_r) +{ + guid_128_t expunged_guid_128; + + if (uid == 0 || sync_ctx->index_reset) { + /* nothing for this or the future ones */ + uid = (uint32_t)-1; + } + + index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r, + expunged_guid_128); + if (sync_ctx->readonly) { + /* we can't expunge anything from read-only mboxes */ + *sync_expunge_r = FALSE; + } +} + +static bool +mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx, + uint32_t uid, const struct mail_index_record **rec_r) +{ + const struct mail_index_record *rec = NULL; + uint32_t messages_count; + bool ret = FALSE; + + if (sync_ctx->index_reset) { + *rec_r = NULL; + return TRUE; + } + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + while (sync_ctx->idx_seq <= messages_count) { + rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq); + if (uid <= rec->uid) + break; + + /* externally expunged message, remove from index */ + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq); + sync_ctx->idx_seq++; + rec = NULL; + } + + if (rec == NULL && uid < sync_ctx->idx_next_uid) { + /* this UID was already in index and it was expunged */ + mbox_sync_set_critical(sync_ctx, + "Expunged message reappeared to mailbox " + "(UID %u < %u, seq=%u, idx_msgs=%u)", + uid, sync_ctx->idx_next_uid, + sync_ctx->seq, messages_count); + ret = FALSE; rec = NULL; + } else if (rec != NULL && rec->uid != uid) { + /* new UID in the middle of the mailbox - shouldn't happen */ + mbox_sync_set_critical(sync_ctx, + "UID inserted in the middle of mailbox " + "(%u > %u, seq=%u, idx_msgs=%u)", + rec->uid, uid, sync_ctx->seq, messages_count); + ret = FALSE; rec = NULL; + } else { + ret = TRUE; + } + + *rec_r = rec; + return ret; +} + +static void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx, + unsigned char hdr_md5_sum[], + const struct mail_index_record **rec_r) +{ + const struct mail_index_record *rec = NULL; + uint32_t messages_count; + const void *data; + + if (sync_ctx->index_reset) { + *rec_r = NULL; + return; + } + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + while (sync_ctx->idx_seq <= messages_count) { + rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq); + mail_index_lookup_ext(sync_ctx->sync_view, + sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + &data, NULL); + if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0) + break; + + /* externally expunged message, remove from index */ + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq); + sync_ctx->idx_seq++; + rec = NULL; + } + + *rec_r = rec; +} + +static void +mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail *mail, + bool nocheck) +{ + const void *data; + uint64_t offset; + + if (!nocheck) { + /* see if from_offset needs updating */ + mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq, + sync_ctx->mbox->mbox_ext_idx, + &data, NULL); + if (data != NULL && + *((const uint64_t *)data) == mail->from_offset) + return; + } + + offset = mail->from_offset; + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->mbox_ext_idx, &offset, NULL); +} + +static void +mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mail_index *index = sync_ctx->mbox->box.index; + struct mail_keywords *keywords; + + keywords = !array_is_created(&mail_ctx->mail.keywords) ? + mail_index_keywords_create(index, NULL) : + mail_index_keywords_create_from_indexes(index, + &mail_ctx->mail.keywords); + mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, keywords); + mail_index_keywords_unref(&keywords); +} + +static void +mbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + const void *ext_data; + + mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, &ext_data, NULL); + if (ext_data == NULL || + memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) { + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + mail_ctx->hdr_md5_sum, NULL); + } +} + +static void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + ARRAY_TYPE(keyword_indexes) idx_keywords; + uint8_t idx_flags, mbox_flags; + + /* default to undirtying the message. it gets added back if + flags/keywords don't match what is in the index. */ + mail_ctx->mail.flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY); + + /* replace flags */ + idx_flags = rec->flags & MAIL_FLAGS_NONRECENT; + mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT; + if (idx_flags != mbox_flags) { + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) | + idx_flags | MAIL_INDEX_MAIL_FLAG_DIRTY; + } + + /* replace keywords */ + t_array_init(&idx_keywords, 32); + mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq, + &idx_keywords); + if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) { + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + + if (!array_is_created(&mail_ctx->mail.keywords)) { + p_array_init(&mail_ctx->mail.keywords, + sync_ctx->mail_keyword_pool, + array_count(&idx_keywords)); + } + array_clear(&mail_ctx->mail.keywords); + array_append_array(&mail_ctx->mail.keywords, &idx_keywords); + } +} + +static void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mailbox *box = &sync_ctx->mbox->box; + struct mbox_sync_mail *mail = &mail_ctx->mail; + enum mail_index_sync_type sync_type; + ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT; + uint8_t flags, orig_flags; + + if (rec != NULL) { + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* flags and keywords are dirty. replace the current + ones from the flags in index file. */ + mbox_sync_get_dirty_flags(mail_ctx, rec); + } + } + + flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT; + if (array_is_created(&mail->keywords)) { + t_array_init(&orig_keywords, 32); + array_append_array(&orig_keywords, &mail->keywords); + } + + /* apply new changes */ + index_sync_changes_apply(sync_ctx->sync_changes, + sync_ctx->mail_keyword_pool, + &flags, &mail->keywords, &sync_type); + if (flags != orig_flags || + !index_keyword_array_cmp(&mail->keywords, &orig_keywords)) { + mail_ctx->need_rewrite = TRUE; + mail->flags = flags | (mail->flags & MAIL_RECENT) | + MAIL_INDEX_MAIL_FLAG_DIRTY; + } + if (sync_type != 0) { + mailbox_sync_notify(box, mail_ctx->mail.uid, + index_sync_type_convert(sync_type)); + } +} + +static void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mbox_sync_mail *mail = &mail_ctx->mail; + ARRAY_TYPE(keyword_indexes) idx_keywords; + uint8_t mbox_flags; + + mbox_flags = mail->flags & ENUM_NEGATE(MAIL_RECENT); + if (!sync_ctx->delay_writes) { + /* changes are written to the mbox file */ + mbox_flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY); + } else if (mail_ctx->need_rewrite) { + /* make sure this message gets written later */ + mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + } + + if (rec == NULL) { + /* new message */ + mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq); + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, mbox_flags); + mbox_sync_update_index_keywords(mail_ctx); + + if (sync_ctx->mbox->mbox_save_md5) { + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + mail_ctx->hdr_md5_sum, NULL); + } + } else { + if ((rec->flags & MAIL_FLAGS_NONRECENT) != + (mbox_flags & MAIL_FLAGS_NONRECENT)) { + /* flags other than recent/dirty have changed */ + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, mbox_flags); + } else if (((rec->flags ^ mbox_flags) & + MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* only dirty flag state changed */ + bool dirty; + + dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0; + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + dirty ? MODIFY_ADD : MODIFY_REMOVE, + (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY); + } + + /* see if keywords changed */ + t_array_init(&idx_keywords, 32); + mail_index_lookup_keywords(sync_ctx->sync_view, + sync_ctx->idx_seq, &idx_keywords); + if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords)) + mbox_sync_update_index_keywords(mail_ctx); + + /* see if we need to update md5 sum. */ + if (sync_ctx->mbox->mbox_save_md5) + mbox_sync_update_md5_if_changed(mail_ctx); + } + + if (!mail_ctx->recent) { + /* Mail has "Status: O" header. No messages before this + can be recent. */ + sync_ctx->last_nonrecent_uid = mail->uid; + } + + /* update from_offsets, but not if we're going to rewrite this message. + rewriting would just move it anyway. */ + if (sync_ctx->need_space_seq == 0) { + bool nocheck = rec == NULL || sync_ctx->expunged_space > 0; + mbox_sync_update_from_offset(sync_ctx, mail, nocheck); + } +} + +static int mbox_read_from_line(struct mbox_sync_mail_context *ctx) +{ + struct istream *input = ctx->sync_ctx->file_input; + const unsigned char *data; + size_t size, from_line_size; + + buffer_set_used_size(ctx->sync_ctx->from_line, 0); + from_line_size = ctx->hdr_offset - ctx->mail.from_offset; + + i_stream_seek(input, ctx->mail.from_offset); + for (;;) { + data = i_stream_get_data(input, &size); + if (size >= from_line_size) + size = from_line_size; + + buffer_append(ctx->sync_ctx->from_line, data, size); + i_stream_skip(input, size); + from_line_size -= size; + + if (from_line_size == 0) + break; + + if (i_stream_read(input) < 0) + return -1; + } + + return 0; +} + +static int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx) +{ + unsigned char buf[10]; + const char *str; + uint32_t uid_last; + unsigned int i; + int ret; + + i_assert(sync_ctx->base_uid_last_offset != 0); + + /* first check that the 10 bytes are there and they're exactly as + expected. just an extra safety check to make sure we never write + to wrong location in the mbox file. */ + ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf), + sync_ctx->base_uid_last_offset); + if (ret < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pread_full()"); + return -1; + } + if (ret == 0) { + mbox_sync_set_critical(sync_ctx, + "X-IMAPbase uid-last offset unexpectedly outside mbox"); + return -1; + } + + for (i = 0, uid_last = 0; i < sizeof(buf); i++) { + if (buf[i] < '0' || buf[i] > '9') { + uid_last = (uint32_t)-1; + break; + } + uid_last = uid_last * 10 + (buf[i] - '0'); + } + + if (uid_last != sync_ctx->base_uid_last) { + mbox_sync_set_critical(sync_ctx, + "X-IMAPbase uid-last unexpectedly lost"); + return -1; + } + + /* and write it */ + str = t_strdup_printf("%010u", sync_ctx->next_uid - 1); + if (pwrite_full(sync_ctx->write_fd, str, 10, + sync_ctx->base_uid_last_offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + mbox_sync_file_updated(sync_ctx, FALSE); + + sync_ctx->base_uid_last = sync_ctx->next_uid - 1; + return 0; +} + +static int +mbox_write_from_line(struct mbox_sync_mail_context *ctx) +{ + string_t *str = ctx->sync_ctx->from_line; + + if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str), + ctx->mail.from_offset) < 0) { + mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()"); + return -1; + } + + mbox_sync_file_updated(ctx->sync_ctx, FALSE); + return 0; +} + +static void update_from_offsets(struct mbox_sync_context *sync_ctx) +{ + const struct mbox_sync_mail *mails; + unsigned int i, count; + uint32_t ext_idx; + uint64_t offset; + + ext_idx = sync_ctx->mbox->mbox_ext_idx; + + mails = array_get(&sync_ctx->mails, &count); + for (i = 0; i < count; i++) { + if (mails[i].idx_seq == 0 || mails[i].expunged) + continue; + + sync_ctx->moved_offsets = TRUE; + offset = mails[i].from_offset; + mail_index_update_ext(sync_ctx->t, mails[i].idx_seq, + ext_idx, &offset, NULL); + } +} + +static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mailbox *box = &sync_ctx->mbox->box; + + mailbox_sync_notify(box, mail_ctx->mail.uid, + MAILBOX_SYNC_TYPE_EXPUNGE); + mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq); + + mail_ctx->mail.expunged = TRUE; + mail_ctx->mail.offset = mail_ctx->mail.from_offset; + mail_ctx->mail.space = + mail_ctx->body_offset - mail_ctx->mail.from_offset + + mail_ctx->mail.body_size; + mail_ctx->mail.body_size = 0; + mail_ctx->mail.uid = 0; + + if (sync_ctx->seq == 1) { + /* expunging first message, fix space to contain next + message's \n header too since it will be removed. */ + mail_ctx->mail.space++; + if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) { + mail_ctx->mail.space++; + sync_ctx->first_mail_crlf_expunged = TRUE; + } + + /* uid-last offset is invalid now */ + sync_ctx->base_uid_last_offset = 0; + } + + sync_ctx->expunged_space += mail_ctx->mail.space; +} + +static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + uoff_t orig_from_offset, postlf_from_offset = UOFF_T_MAX; + off_t move_diff; + int ret; + + if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) { + /* move the header backwards to fill expunged space */ + move_diff = -sync_ctx->expunged_space; + + orig_from_offset = mail_ctx->mail.from_offset; + if (sync_ctx->dest_first_mail) { + /* we're moving this mail to beginning of file. + skip the initial \n (it's already counted in + expunged_space) */ + mail_ctx->mail.from_offset++; + if (sync_ctx->first_mail_crlf_expunged) + mail_ctx->mail.from_offset++; + } + postlf_from_offset = mail_ctx->mail.from_offset; + + /* read the From-line before rewriting overwrites it */ + if (mbox_read_from_line(mail_ctx) < 0) + return -1; + i_assert((off_t)mail_ctx->mail.from_offset + move_diff != 1 && + (off_t)mail_ctx->mail.from_offset + move_diff != 2); + + mbox_sync_update_header(mail_ctx); + ret = mbox_sync_try_rewrite(mail_ctx, move_diff); + if (ret < 0) + return -1; + + if (ret > 0) { + /* rewrite successful, write From-line to + new location */ + i_assert((off_t)mail_ctx->mail.from_offset >= + -move_diff); + mail_ctx->mail.from_offset = (off_t)mail_ctx->mail.from_offset + move_diff; + mail_ctx->mail.offset = (off_t)mail_ctx->mail.offset + move_diff; + if (mbox_write_from_line(mail_ctx) < 0) + return -1; + } else { + if (sync_ctx->dest_first_mail) { + /* didn't have enough space, move the offset + back so seeking into it doesn't fail */ + mail_ctx->mail.from_offset = orig_from_offset; + } + } + } else if (mail_ctx->need_rewrite) { + mbox_sync_update_header(mail_ctx); + if (sync_ctx->delay_writes && sync_ctx->need_space_seq == 0) { + /* mark it dirty and do it later. we can't do this + if we're in the middle of rewriting acquiring more + space. */ + mail_ctx->dirty = TRUE; + return 0; + } + + if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0) + return -1; + } else { + /* nothing to do */ + return 0; + } + + if (ret == 0 && sync_ctx->need_space_seq == 0) { + /* first mail with no space to write it */ + sync_ctx->need_space_seq = sync_ctx->seq; + sync_ctx->space_diff = 0; + + if (sync_ctx->expunged_space > 0) { + /* create dummy message to describe the expunged data */ + struct mbox_sync_mail mail; + + /* if this is going to be the first mail, increase the + from_offset to point to the beginning of the + From-line, because the previous [CR]LF is already + covered by expunged_space. */ + i_assert(postlf_from_offset != UOFF_T_MAX); + mail_ctx->mail.from_offset = postlf_from_offset; + + i_zero(&mail); + mail.expunged = TRUE; + mail.offset = mail.from_offset = + mail_ctx->mail.from_offset - + sync_ctx->expunged_space; + mail.space = sync_ctx->expunged_space; + + sync_ctx->space_diff = sync_ctx->expunged_space; + sync_ctx->expunged_space = 0; + i_assert(sync_ctx->space_diff < -mail_ctx->mail.space); + + sync_ctx->need_space_seq--; + array_push_back(&sync_ctx->mails, &mail); + } + } + return 0; +} + +static int +mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + uoff_t end_offset, move_diff, extra_space, needed_space; + uint32_t last_seq; + ARRAY_TYPE(keyword_indexes) keywords_copy; + + i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 || + mail_ctx->mail.offset == mail_ctx->hdr_offset); + + if (array_is_created(&mail_ctx->mail.keywords)) { + /* mail's keywords are allocated from a pool that's cleared + for each mail. we'll need to copy it to something more + permanent. */ + p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool, + array_count(&mail_ctx->mail.keywords)); + array_append_array(&keywords_copy, &mail_ctx->mail.keywords); + mail_ctx->mail.keywords = keywords_copy; + } + array_push_back(&sync_ctx->mails, &mail_ctx->mail); + + sync_ctx->space_diff += mail_ctx->mail.space; + if (sync_ctx->space_diff < 0) { + if (sync_ctx->expunged_space > 0) { + i_assert(sync_ctx->expunged_space == + mail_ctx->mail.space); + sync_ctx->expunged_space = 0; + } + return 0; + } + + /* we have enough space now */ + if (mail_ctx->mail.uid == 0) { + /* this message was expunged. fill more or less of the space. + space_diff now consists of a negative "bytes needed" sum, + plus the expunged space of this message. so it contains how + many bytes of _extra_ space we have. */ + i_assert(mail_ctx->mail.space >= sync_ctx->space_diff); + extra_space = MBOX_HEADER_PADDING * + (sync_ctx->seq - sync_ctx->need_space_seq + 1); + needed_space = mail_ctx->mail.space - sync_ctx->space_diff; + if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) { + /* don't waste too much on padding */ + move_diff = needed_space + extra_space; + sync_ctx->expunged_space = + mail_ctx->mail.space - move_diff; + } else { + move_diff = mail_ctx->mail.space; + extra_space = sync_ctx->space_diff; + sync_ctx->expunged_space = 0; + } + last_seq = sync_ctx->seq - 1; + array_pop_back(&sync_ctx->mails); + end_offset = mail_ctx->mail.from_offset; + } else { + /* this message gave enough space from headers. rewriting stops + at the end of this message's headers. */ + sync_ctx->expunged_space = 0; + last_seq = sync_ctx->seq; + end_offset = mail_ctx->body_offset; + + move_diff = 0; + extra_space = sync_ctx->space_diff; + } + + mbox_sync_file_update_ext_modified(sync_ctx); + if (mbox_sync_rewrite(sync_ctx, + last_seq == sync_ctx->seq ? mail_ctx : NULL, + end_offset, move_diff, extra_space, + sync_ctx->need_space_seq, last_seq) < 0) + return -1; + + update_from_offsets(sync_ctx); + + /* mail_ctx may contain wrong data after rewrite, so make sure we + don't try to access it */ + i_zero(mail_ctx); + + sync_ctx->need_space_seq = 0; + sync_ctx->space_diff = 0; + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + return 0; +} + +static int +mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq) +{ + struct mbox_mailbox *mbox = sync_ctx->mbox; + uoff_t old_offset, offset; + uint32_t uid; + int ret; + bool deleted; + + if (seq == 0) { + if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) { + mbox->invalid_mbox_file = TRUE; + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOTPOSSIBLE, + "Mailbox isn't a valid mbox file"); + return -1; + } + seq++; + } else { + old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + + ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted); + if (ret < 0) { + if (deleted) { + mbox_sync_set_critical(sync_ctx, + "Message was expunged unexpectedly"); + } + return -1; + } + if (ret == 0) { + if (istream_raw_mbox_seek(mbox->mbox_stream, + old_offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Error seeking back to original " + "offset %s", dec2str(old_offset)); + return -1; + } + return 0; + } + } + + if (seq <= 1) + uid = 0; + else + mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid); + + sync_ctx->prev_msg_uid = uid; + + /* set to -1, since it's always increased later */ + sync_ctx->seq = seq-1; + if (sync_ctx->seq == 0 && + istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) { + /* this mbox has pseudo mail which contains the X-IMAP header */ + sync_ctx->seq++; + } + + sync_ctx->idx_seq = seq; + sync_ctx->dest_first_mail = sync_ctx->seq == 0; + if (istream_raw_mbox_get_body_offset(sync_ctx->input, &offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Message body offset lookup failed"); + return -1; + } + return 1; +} + +static int +mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid) +{ + struct mail_index_view *sync_view = sync_ctx->sync_view; + uint32_t seq1, seq2; + uoff_t size; + int ret; + + i_assert(!sync_ctx->index_reset); + + if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1, + &seq1, &seq2)) { + /* doesn't exist anymore, seek to end of file */ + ret = i_stream_get_size(sync_ctx->file_input, TRUE, &size); + if (ret < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_get_size()"); + return -1; + } + i_assert(ret != 0); + + if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream, + size) < 0) { + mbox_sync_set_critical(sync_ctx, + "Error seeking to end of mbox"); + return -1; + } + sync_ctx->idx_seq = + mail_index_view_get_messages_count(sync_view) + 1; + return 1; + } + + return mbox_sync_seek_to_seq(sync_ctx, seq1); +} + +static int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx, + uint32_t next_uid, bool *partial, + bool *skipped_mails) +{ + uint32_t messages_count, uid; + int ret; + + i_assert(!sync_ctx->index_reset); + + /* delete sync records up to next message. so if there's still + something left in array, it means the next message needs modifying */ + index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid); + if (index_sync_changes_have(sync_ctx->sync_changes)) + return 1; + + if (sync_ctx->hdr->first_recent_uid <= next_uid && + !sync_ctx->keep_recent) { + /* we'll need to rewrite Status: O headers */ + return 1; + } + + uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes); + + if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid && + (uid > sync_ctx->hdr->first_recent_uid || uid == 0) && + !sync_ctx->keep_recent) { + /* we'll need to rewrite Status: O headers */ + uid = sync_ctx->hdr->first_recent_uid; + } + + if (uid != 0) { + /* we can skip forward to next record which needs updating. */ + if (uid != next_uid) { + *skipped_mails = TRUE; + next_uid = uid; + } + ret = mbox_sync_seek_to_uid(sync_ctx, next_uid); + } else { + /* if there's no sync records left, we can stop. except if + this is a dirty sync, check if there are new messages. */ + if (sync_ctx->mbox->mbox_hdr.dirty_flag == 0) + return 0; + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + if (sync_ctx->seq + 1 != messages_count) { + ret = mbox_sync_seek_to_seq(sync_ctx, messages_count); + *skipped_mails = TRUE; + } else { + ret = 1; + } + *partial = FALSE; + } + + if (ret == 0) { + /* seek failed because the offset is dirty. just ignore and + continue from where we are now. */ + *partial = FALSE; + ret = 1; + } + return ret; +} + +static void mbox_sync_hdr_update(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update; + + if (update->uid_validity != 0) { + sync_ctx->base_uid_validity = update->uid_validity; + mail_ctx->imapbase_rewrite = TRUE; + mail_ctx->need_rewrite = TRUE; + } + if (update->min_next_uid != 0 && + sync_ctx->base_uid_last+1 < update->min_next_uid) { + i_assert(sync_ctx->next_uid <= update->min_next_uid); + sync_ctx->base_uid_last = update->min_next_uid-1; + sync_ctx->next_uid = update->min_next_uid; + mail_ctx->imapbase_rewrite = TRUE; + mail_ctx->need_rewrite = TRUE; + } +} + +static bool mbox_sync_imapbase(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + if (sync_ctx->base_uid_validity != 0 && + sync_ctx->hdr->uid_validity != 0 && + sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) { + i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s", + sync_ctx->hdr->uid_validity, + sync_ctx->base_uid_validity, + mailbox_get_path(&sync_ctx->mbox->box)); + sync_ctx->index_reset = TRUE; + return TRUE; + } + if (sync_ctx->mbox->sync_hdr_update != NULL) + mbox_sync_hdr_update(sync_ctx, mail_ctx); + return FALSE; +} + +static int mbox_sync_loop(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + bool partial) +{ + const struct mail_index_record *rec; + uint32_t uid, messages_count; + uoff_t offset; + int ret; + bool expunged, skipped_mails, uids_broken; + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + + /* always start from first message so we can read X-IMAP or + X-IMAPbase header */ + ret = mbox_sync_seek_to_seq(sync_ctx, 0); + if (ret <= 0) + return ret; + + if (sync_ctx->renumber_uids) { + /* expunge everything */ + while (sync_ctx->idx_seq <= messages_count) { + mail_index_expunge(sync_ctx->t, + sync_ctx->idx_seq++); + } + } + + skipped_mails = uids_broken = FALSE; + while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) { + uid = mail_ctx->mail.uid; + + if (mail_ctx->seq == 1) { + if (mbox_sync_imapbase(sync_ctx, mail_ctx)) { + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + return 0; + } + } + + if (mail_ctx->mail.uid_broken && partial) { + /* UID ordering problems, resync everything to make + sure we get everything right */ + if (sync_ctx->mbox->mbox_hdr.dirty_flag != 0) + return 0; + + mbox_sync_set_critical(sync_ctx, + "UIDs broken with partial sync"); + + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + return 0; + } + if (mail_ctx->mail.uid_broken) + uids_broken = TRUE; + + if (mail_ctx->mail.pseudo) + uid = 0; + + rec = NULL; ret = 1; + if (uid != 0) { + if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec)) + ret = 0; + } + + if (ret == 0) { + /* UID found but it's broken */ + uid = 0; + } else if (uid == 0 && + !mail_ctx->mail.pseudo && + (sync_ctx->delay_writes || + sync_ctx->idx_seq <= messages_count)) { + /* If we can't use/store X-UID header, use MD5 sum. + Also check for existing MD5 sums when we're actually + able to write X-UIDs. */ + sync_ctx->mbox->mbox_save_md5 = TRUE; + + mbox_sync_find_index_md5(sync_ctx, + mail_ctx->hdr_md5_sum, &rec); + if (rec != NULL) + uid = mail_ctx->mail.uid = rec->uid; + } + + /* get all sync records related to this message. with pseudo + message just get the first sync record so we can jump to + it with partial seeking. */ + mbox_sync_read_index_syncs(sync_ctx, + mail_ctx->mail.pseudo ? 1 : uid, + &expunged); + + if (mail_ctx->mail.pseudo) { + /* if it was set, it was for the next message */ + expunged = FALSE; + } else { + if (rec == NULL) { + /* message wasn't found from index. we have to + read everything from now on, no skipping */ + partial = FALSE; + } + } + + if (uid == 0 && !mail_ctx->mail.pseudo) { + /* missing/broken X-UID. all the rest of the mails + need new UIDs. */ + while (sync_ctx->idx_seq <= messages_count) { + mail_index_expunge(sync_ctx->t, + sync_ctx->idx_seq++); + } + + if (sync_ctx->next_uid == (uint32_t)-1) { + /* oh no, we're out of UIDs. this shouldn't + happen normally, so just try to get it fixed + without crashing. */ + mailbox_set_critical(&sync_ctx->mbox->box, + "Out of UIDs, renumbering them in mbox"); + sync_ctx->renumber_uids = TRUE; + return 0; + } + + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.uid = sync_ctx->next_uid++; + } + sync_ctx->prev_msg_uid = mail_ctx->mail.uid; + + if (!mail_ctx->mail.pseudo) + mail_ctx->mail.idx_seq = sync_ctx->idx_seq; + + if (!expunged) { + if (!mail_ctx->mail.pseudo) T_BEGIN { + mbox_sync_update_flags(mail_ctx, rec); + } T_END; + if (mbox_sync_handle_header(mail_ctx) < 0) + return -1; + sync_ctx->dest_first_mail = FALSE; + } else { + mbox_sync_handle_expunge(mail_ctx); + } + + if (!mail_ctx->mail.pseudo) { + if (!expunged) T_BEGIN { + mbox_sync_update_index(mail_ctx, rec); + } T_END; + sync_ctx->idx_seq++; + } + + if (istream_raw_mbox_next(sync_ctx->input, + mail_ctx->mail.body_size) < 0) + return -1; + offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + + if (sync_ctx->need_space_seq != 0) { + if (mbox_sync_handle_missing_space(mail_ctx) < 0) + return -1; + if (mbox_sync_seek(sync_ctx, offset) < 0) + return -1; + } else if (sync_ctx->expunged_space > 0) { + if (!expunged) { + /* move the body */ + mbox_sync_file_update_ext_modified(sync_ctx); + if (mbox_move(sync_ctx, + mail_ctx->body_offset - + sync_ctx->expunged_space, + mail_ctx->body_offset, + mail_ctx->mail.body_size) < 0) + return -1; + if (mbox_sync_seek(sync_ctx, offset) < 0) + return -1; + } + } else if (partial) { + ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1, + &partial, + &skipped_mails); + if (ret <= 0) + break; + } + } + if (ret < 0) + return -1; + + if (istream_raw_mbox_is_eof(sync_ctx->input)) { + /* rest of the messages in index don't exist -> expunge them */ + while (sync_ctx->idx_seq <= messages_count) + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++); + } + + if (!skipped_mails) + sync_ctx->mbox->mbox_hdr.dirty_flag = 0; + sync_ctx->mbox->mbox_broken_offsets = FALSE; + + if (uids_broken && sync_ctx->delay_writes) { + /* once we get around to writing the changes, we'll need to do + a full sync to avoid the "UIDs broken in partial sync" + error */ + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } + return 1; +} + +static int mbox_write_pseudo(struct mbox_sync_context *sync_ctx, bool force) +{ + string_t *str; + unsigned int uid_validity; + + i_assert(sync_ctx->write_fd != -1); + + if (sync_ctx->mbox->sync_hdr_update != NULL) { + const struct mailbox_update *update = + sync_ctx->mbox->sync_hdr_update; + bool change = FALSE; + + if (update->uid_validity != 0) { + sync_ctx->base_uid_validity = update->uid_validity; + change = TRUE; + } + if (update->min_next_uid != 0) { + sync_ctx->base_uid_last = update->min_next_uid-1; + change = TRUE; + } + if (!change && !force) + return 0; + } + + uid_validity = sync_ctx->base_uid_validity != 0 ? + sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity; + i_assert(uid_validity != 0); + + str = t_str_new(1024); + str_printfa(str, "%sDate: %s\n" + "From: Mail System Internal Data <MAILER-DAEMON@%s>\n" + "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA" + "\nMessage-ID: <%s@%s>\n" + "X-IMAP: %u %010u\n" + "Status: RO\n" + "\n" + PSEUDO_MESSAGE_BODY + "\n", + mbox_from_create("MAILER_DAEMON", ioloop_time), + message_date_create(ioloop_time), + my_hostname, dec2str(ioloop_time), my_hostname, + uid_validity, sync_ctx->next_uid-1); + + if (pwrite_full(sync_ctx->write_fd, + str_data(str), str_len(str), 0) < 0) { + if (!ENOSPACE(errno)) { + mbox_set_syscall_error(sync_ctx->mbox, + "pwrite_full()"); + return -1; + } + + /* out of disk space, truncate to empty */ + if (ftruncate(sync_ctx->write_fd, 0) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + } + + sync_ctx->base_uid_validity = uid_validity; + sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */ + sync_ctx->base_uid_last = sync_ctx->next_uid-1; + return 0; +} + +static int mbox_append_zero(struct mbox_sync_context *sync_ctx, + uoff_t orig_file_size, uoff_t count) +{ + char block[IO_BLOCK_SIZE]; + uoff_t offset = orig_file_size; + ssize_t ret = 0; + + memset(block, 0, I_MIN(sizeof(block), count)); + while (count > 0) { + ret = pwrite(sync_ctx->write_fd, block, + I_MIN(sizeof(block), count), offset); + if (ret < 0) + break; + offset += ret; + count -= ret; + } + + if (ret < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite()"); + if (ftruncate(sync_ctx->write_fd, orig_file_size) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + return -1; + } + return 0; +} + +static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + uoff_t file_size, offset, padding, trailer_size; + int ret; + + if (!istream_raw_mbox_is_eof(sync_ctx->input)) { + i_assert(sync_ctx->need_space_seq == 0); + i_assert(sync_ctx->expunged_space == 0); + return 0; + } + + ret = i_stream_get_size(sync_ctx->file_input, TRUE, &file_size); + if (ret < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_get_size()"); + return -1; + } + if (ret == 0) { + /* Not a file - allow anyway */ + return 0; + } + + if (file_size < sync_ctx->file_input->v_offset) { + mbox_sync_set_critical(sync_ctx, + "file size unexpectedly shrank " + "(%"PRIuUOFF_T" vs %"PRIuUOFF_T")", file_size, + sync_ctx->file_input->v_offset); + return -1; + } + trailer_size = file_size - sync_ctx->file_input->v_offset; + i_assert(trailer_size <= 2); + + if (sync_ctx->need_space_seq != 0) { + i_assert(sync_ctx->write_fd != -1); + + i_assert(sync_ctx->space_diff < 0); + padding = MBOX_HEADER_PADDING * + (sync_ctx->seq - sync_ctx->need_space_seq + 1); + sync_ctx->space_diff -= padding; + + i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff); + sync_ctx->space_diff += sync_ctx->expunged_space; + sync_ctx->expunged_space = 0; + + if (mail_ctx->have_eoh && !mail_ctx->updated) + str_append_c(mail_ctx->header, '\n'); + + i_assert(sync_ctx->space_diff < 0); + + if (mbox_append_zero(sync_ctx, file_size, + -sync_ctx->space_diff) < 0) + return -1; + mbox_sync_file_updated(sync_ctx, FALSE); + + if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size, + -sync_ctx->space_diff, padding, + sync_ctx->need_space_seq, + sync_ctx->seq) < 0) + return -1; + + update_from_offsets(sync_ctx); + + sync_ctx->need_space_seq = 0; + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + } + + if (sync_ctx->expunged_space > 0) { + i_assert(sync_ctx->write_fd != -1); + + mbox_sync_file_update_ext_modified(sync_ctx); + + /* copy trailer, then truncate the file */ + file_size = sync_ctx->last_stat.st_size; + if (file_size == (uoff_t)sync_ctx->expunged_space) { + /* everything deleted, the trailer_size still contains + the \n trailer though */ + trailer_size = 0; + } else if (sync_ctx->expunged_space == (off_t)file_size + 1 || + sync_ctx->expunged_space == (off_t)file_size + 2) { + /* everything deleted and we didn't have a proper + trailer. */ + trailer_size = 0; + sync_ctx->expunged_space = file_size; + } + + i_assert(file_size >= sync_ctx->expunged_space + trailer_size); + offset = file_size - sync_ctx->expunged_space - trailer_size; + i_assert(offset == 0 || offset > 31); + + if (mbox_move(sync_ctx, offset, + offset + sync_ctx->expunged_space, + trailer_size) < 0) + return -1; + if (ftruncate(sync_ctx->write_fd, + offset + trailer_size) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + return -1; + } + + if (offset == 0) { + if (mbox_write_pseudo(sync_ctx, TRUE) < 0) + return -1; + } + + sync_ctx->expunged_space = 0; + mbox_sync_file_updated(sync_ctx, FALSE); + } else { + if (file_size == 0 && sync_ctx->mbox->sync_hdr_update != NULL) { + if (mbox_write_pseudo(sync_ctx, FALSE) < 0) + return -1; + } + } + return 0; +} + +static void +mbox_sync_index_update_ext_header(struct mbox_mailbox *mbox, + struct mail_index_transaction *trans) +{ + const struct mailbox_update *update = mbox->sync_hdr_update; + const void *data; + size_t data_size; + + if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) { + memcpy(mbox->mbox_hdr.mailbox_guid, update->mailbox_guid, + sizeof(mbox->mbox_hdr.mailbox_guid)); + } else if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + guid_128_generate(mbox->mbox_hdr.mailbox_guid); + } + + mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx, + &data, &data_size); + if (data_size != sizeof(mbox->mbox_hdr) || + memcmp(data, &mbox->mbox_hdr, data_size) != 0) { + if (data_size != sizeof(mbox->mbox_hdr)) { + /* upgrading from v1.x */ + mail_index_ext_resize(trans, mbox->mbox_ext_idx, + sizeof(mbox->mbox_hdr), + sizeof(uint64_t), + sizeof(uint64_t)); + } + mail_index_update_header_ext(trans, mbox->mbox_ext_idx, + 0, &mbox->mbox_hdr, + sizeof(mbox->mbox_hdr)); + } +} + +static uint32_t mbox_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"MBOX_UIDVALIDITY_FNAME, NULL); + return mailbox_uidvalidity_next(list, path); +} + +static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx) +{ + struct mail_index_view *view; + const struct stat *st; + uint32_t first_recent_uid, seq, seq2; + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + + if (sync_ctx->moved_offsets && + ((uint64_t)st->st_size == sync_ctx->mbox->mbox_hdr.sync_size || + (uint64_t)st->st_size == sync_ctx->orig_size)) { + /* We moved messages inside the mbox file without changing + the file's size. If mtime doesn't change, another process + not using the same index file as us can't know that the file + was changed. So make sure the mtime changes. This should + happen rarely enough that the sleeping doesn't become a + performance problem. + + Note that to do this perfectly safe we should do this wait + whenever mails are moved or expunged, regardless of whether + the file's size changed. That however could become a + performance problem and the consequences of being wrong are + quite minimal (an extra logged error message). */ + while (sync_ctx->orig_mtime == st->st_mtime) { + i_sleep_msecs(500); + if (utime(mailbox_get_path(&sync_ctx->mbox->box), NULL) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, + "utime()"); + return -1; + } + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + } + } + + sync_ctx->mbox->mbox_hdr.sync_mtime = st->st_mtime; + sync_ctx->mbox->mbox_hdr.sync_size = st->st_size; + mbox_sync_index_update_ext_header(sync_ctx->mbox, sync_ctx->t); + + /* only reason not to have UID validity at this point is if the file + is entirely empty. In that case just make up a new one if needed. */ + i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0); + + if (sync_ctx->base_uid_validity == 0) { + sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ? + sync_ctx->hdr->uid_validity : + mbox_get_uidvalidity_next(sync_ctx->mbox->box.list); + } + if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) { + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, uid_validity), + &sync_ctx->base_uid_validity, + sizeof(sync_ctx->base_uid_validity), TRUE); + } + + if (istream_raw_mbox_is_eof(sync_ctx->input) && + sync_ctx->next_uid != sync_ctx->hdr->next_uid) { + i_assert(sync_ctx->next_uid != 0); + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, next_uid), + &sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE); + } + + if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) { + /* other sessions have already marked more messages as + recent. */ + sync_ctx->last_nonrecent_uid = + sync_ctx->hdr->first_recent_uid - 1; + } + + /* mark recent messages */ + view = mail_index_transaction_open_updated_view(sync_ctx->t); + if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1, + (uint32_t)-1, &seq, &seq2)) { + mailbox_recent_flags_set_seqs(&sync_ctx->mbox->box, + view, seq, seq2); + } + mail_index_view_close(&view); + + first_recent_uid = !sync_ctx->keep_recent ? + sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1; + if (sync_ctx->hdr->first_recent_uid < first_recent_uid) { + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + return 0; +} + +static void mbox_sync_restart(struct mbox_sync_context *sync_ctx) +{ + sync_ctx->base_uid_validity = 0; + sync_ctx->base_uid_last = 0; + sync_ctx->base_uid_last_offset = 0; + + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + + index_sync_changes_reset(sync_ctx->sync_changes); + mail_index_sync_reset(sync_ctx->index_sync_ctx); + mail_index_transaction_reset(sync_ctx->t); + + if (sync_ctx->index_reset) { + mail_index_reset(sync_ctx->t); + sync_ctx->reset_hdr.next_uid = 1; + sync_ctx->hdr = &sync_ctx->reset_hdr; + mailbox_recent_flags_reset(&sync_ctx->mbox->box); + } + + sync_ctx->prev_msg_uid = 0; + sync_ctx->next_uid = sync_ctx->hdr->next_uid; + sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid; + sync_ctx->seq = 0; + sync_ctx->idx_seq = 1; + sync_ctx->need_space_seq = 0; + sync_ctx->expunged_space = 0; + sync_ctx->space_diff = 0; + + sync_ctx->dest_first_mail = TRUE; + sync_ctx->ext_modified = FALSE; + sync_ctx->errors = FALSE; +} + +static int mbox_sync_do(struct mbox_sync_context *sync_ctx, + enum mbox_sync_flags flags) +{ + struct mbox_index_header *mbox_hdr = &sync_ctx->mbox->mbox_hdr; + struct mbox_sync_mail_context mail_ctx; + const struct stat *st; + unsigned int i; + bool partial; + int ret; + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + sync_ctx->last_stat = *st; + sync_ctx->orig_size = st->st_size; + sync_ctx->orig_atime = st->st_atime; + sync_ctx->orig_mtime = st->st_mtime; + + if ((flags & MBOX_SYNC_FORCE_SYNC) != 0) { + /* forcing a full sync. assume file has changed. */ + partial = FALSE; + mbox_hdr->dirty_flag = 1; + } else if ((uint32_t)st->st_mtime == mbox_hdr->sync_mtime && + (uint64_t)st->st_size == mbox_hdr->sync_size) { + /* file is fully synced */ + if (mbox_hdr->dirty_flag != 0 && (flags & MBOX_SYNC_UNDIRTY) != 0) + partial = FALSE; + else + partial = TRUE; + } else if ((flags & MBOX_SYNC_UNDIRTY) != 0 || + (uint64_t)st->st_size == mbox_hdr->sync_size) { + /* we want to do full syncing. always do this if + file size hasn't changed but timestamp has. it most + likely means that someone had modified some header + and we probably want to know about it */ + partial = FALSE; + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } else { + /* see if we can delay syncing the whole file. + normally we only notice expunges and appends + in partial syncing. */ + partial = TRUE; + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } + + mbox_sync_restart(sync_ctx); + for (i = 0;;) { + ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial); + if (ret > 0 && !sync_ctx->errors) + break; + if (ret < 0) + return -1; + + /* a) partial sync didn't work + b) we ran out of UIDs + c) syncing had errors */ + if (sync_ctx->delay_writes && + (sync_ctx->errors || sync_ctx->renumber_uids)) { + /* fixing a broken mbox state, be sure to write + the changes (except if we're readonly). */ + if (!sync_ctx->readonly) + sync_ctx->delay_writes = FALSE; + } + if (++i == 3) + break; + + mbox_sync_restart(sync_ctx); + partial = FALSE; + } + + if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0) + return -1; + + /* only syncs left should be just appends (and their updates) + which weren't synced yet for some reason (crash). we'll just + ignore them, as we've overwritten them above. */ + index_sync_changes_reset(sync_ctx->sync_changes); + + if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 && + ret > 0 && !sync_ctx->delay_writes && + sync_ctx->base_uid_last_offset != 0) { + /* Rewrite uid_last in X-IMAPbase header if we've seen it + (ie. the file isn't empty) */ + ret = mbox_rewrite_base_uid_last(sync_ctx); + } else { + ret = 0; + } + + if (mbox_sync_update_index_header(sync_ctx) < 0) + return -1; + return ret; +} + +int mbox_sync_header_refresh(struct mbox_mailbox *mbox) +{ + const void *data; + size_t data_size; + + if (mail_index_refresh(mbox->box.index) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx, + &data, &data_size); + if (data_size == 0) { + /* doesn't exist yet. */ + i_zero(&mbox->mbox_hdr); + return 0; + } + + memcpy(&mbox->mbox_hdr, data, I_MIN(sizeof(mbox->mbox_hdr), data_size)); + if (mbox->mbox_broken_offsets) + mbox->mbox_hdr.dirty_flag = 1; + return 0; +} + +int mbox_sync_get_guid(struct mbox_mailbox *mbox) +{ + struct mail_index_transaction *trans; + unsigned int lock_id; + int ret; + + if (mbox_lock(mbox, F_WRLCK, &lock_id) <= 0) + return -1; + + ret = mbox_sync_header_refresh(mbox); + if (ret == 0) { + trans = mail_index_transaction_begin(mbox->box.view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mbox_sync_index_update_ext_header(mbox, trans); + ret = mail_index_transaction_commit(&trans); + } + mbox_unlock(mbox, lock_id); + return ret; +} + +int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty) +{ + const struct stat *st; + struct stat statbuf; + + if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) { + /* read-only stream */ + if (i_stream_stat(mbox->mbox_file_stream, FALSE, &st) < 0) { + if (errno == ENOENT) { + mailbox_set_deleted(&mbox->box); + return 0; + } + mbox_istream_set_syscall_error(mbox, + mbox->mbox_file_stream, "i_stream_stat()"); + return -1; + } + } else { + if (stat(mailbox_get_path(&mbox->box), &statbuf) < 0) { + if (errno == ENOENT) { + mailbox_set_deleted(&mbox->box); + return 0; + } + mbox_set_syscall_error(mbox, "stat()"); + return -1; + } + st = &statbuf; + } + + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + + if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + /* need to assign mailbox GUID */ + return 1; + } + + if ((uint32_t)st->st_mtime == mbox->mbox_hdr.sync_mtime && + (uint64_t)st->st_size == mbox->mbox_hdr.sync_size) { + /* fully synced */ + if (mbox->mbox_hdr.dirty_flag != 0 || leave_dirty) + return 0; + /* flushing dirtiness */ + } + + /* file changed */ + return 1; +} + +static void mbox_sync_context_free(struct mbox_sync_context *sync_ctx) +{ + index_sync_changes_deinit(&sync_ctx->sync_changes); + index_storage_expunging_deinit(&sync_ctx->mbox->box); + if (sync_ctx->index_sync_ctx != NULL) + mail_index_sync_rollback(&sync_ctx->index_sync_ctx); + pool_unref(&sync_ctx->mail_keyword_pool); + pool_unref(&sync_ctx->saved_keywords_pool); + str_free(&sync_ctx->header); + str_free(&sync_ctx->from_line); + array_free(&sync_ctx->mails); +} + +static int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags, + unsigned int *lock_id) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mbox_sync_context sync_ctx; + enum mail_index_sync_flags sync_flags; + int ret; + bool changed, delay_writes, readonly; + + readonly = mbox_is_backend_readonly(mbox) || + (flags & MBOX_SYNC_READONLY) != 0; + delay_writes = readonly || + ((flags & MBOX_SYNC_REWRITE) == 0 && + mbox->storage->set->mbox_lazy_writes); + + if (!mbox->storage->set->mbox_dirty_syncs && + !mbox->storage->set->mbox_very_dirty_syncs) + flags |= MBOX_SYNC_UNDIRTY; + + if ((flags & MBOX_SYNC_LOCK_READING) != 0) { + if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0) + return -1; + } + + if ((flags & MBOX_SYNC_HEADER) != 0 || + (flags & MBOX_SYNC_FORCE_SYNC) != 0) { + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + changed = TRUE; + } else { + bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0; + if ((ret = mbox_sync_has_changed(mbox, leave_dirty)) < 0) + return -1; + changed = ret > 0; + } + + if ((flags & MBOX_SYNC_LOCK_READING) != 0) { + /* we just want to lock it for reading. if mbox hasn't been + modified don't do any syncing. */ + if (!changed) + return 0; + + /* have to sync to make sure offsets have stayed the same */ + mbox_unlock(mbox, *lock_id); + *lock_id = 0; + } + + /* flush input streams' buffers */ + if (mbox->mbox_stream != NULL) + i_stream_sync(mbox->mbox_stream); + if (mbox->mbox_file_stream != NULL) + i_stream_sync(mbox->mbox_file_stream); + +again: + if (changed) { + /* we're most likely modifying the mbox while syncing, just + lock it for writing immediately. the mbox must be locked + before index syncing is started to avoid deadlocks, so we + don't have much choice either (well, easy ones anyway). */ + int lock_type = readonly ? F_RDLCK : F_WRLCK; + + if ((ret = mbox_lock(mbox, lock_type, lock_id)) <= 0) { + if (ret == 0 || lock_type == F_RDLCK) + return -1; + + /* try as read-only */ + if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0) + return -1; + mbox->backend_readonly = readonly = TRUE; + mbox->backend_readonly_set = TRUE; + delay_writes = TRUE; + } + } + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if ((flags & MBOX_SYNC_REWRITE) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY; + + ret = index_storage_expunged_sync_begin(&mbox->box, &index_sync_ctx, + &sync_view, &trans, sync_flags); + if (ret <= 0) + return ret; + + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) { + /* see if we need to drop recent flags */ + sync_ctx.hdr = mail_index_get_header(sync_view); + if (sync_ctx.hdr->first_recent_uid < sync_ctx.hdr->next_uid) + changed = TRUE; + } + + if (!changed && !mail_index_sync_have_more(index_sync_ctx)) { + /* nothing to do */ + nothing_to_do: + /* index may need to do internal syncing though, so commit + instead of rolling back. */ + index_storage_expunging_deinit(&mbox->box); + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + return 0; + } + + i_zero(&sync_ctx); + sync_ctx.mbox = mbox; + sync_ctx.keep_recent = + (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0; + + sync_ctx.hdr = mail_index_get_header(sync_view); + sync_ctx.from_line = str_new(default_pool, 256); + sync_ctx.header = str_new(default_pool, 4096); + + sync_ctx.index_sync_ctx = index_sync_ctx; + sync_ctx.sync_view = sync_view; + sync_ctx.t = trans; + sync_ctx.mail_keyword_pool = + pool_alloconly_create("mbox keywords", 512); + sync_ctx.saved_keywords_pool = + pool_alloconly_create("mbox saved keywords", 4096); + + /* make sure we've read the latest keywords in index */ + (void)mail_index_get_keywords(mbox->box.index); + + i_array_init(&sync_ctx.mails, 64); + + sync_ctx.flags = flags; + sync_ctx.readonly = readonly; + sync_ctx.delay_writes = delay_writes; + + sync_ctx.sync_changes = + index_sync_changes_init(index_sync_ctx, sync_view, trans, + sync_ctx.delay_writes); + + if (!changed && delay_writes) { + /* if we have only flag changes, we don't need to open the + mbox file */ + bool expunged; + uint32_t uid; + + mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged); + uid = expunged ? 1 : + index_sync_changes_get_next_uid(sync_ctx.sync_changes); + if (uid == 0) { + sync_ctx.index_sync_ctx = NULL; + mbox_sync_context_free(&sync_ctx); + goto nothing_to_do; + } + } + + if (*lock_id == 0) { + /* ok, we have something to do but no locks. we'll have to + restart syncing to avoid deadlocking. */ + mbox_sync_context_free(&sync_ctx); + changed = TRUE; + goto again; + } + + if (mbox_file_open_stream(mbox) < 0) { + mbox_sync_context_free(&sync_ctx); + return -1; + } + + sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream; + sync_ctx.input = sync_ctx.mbox->mbox_stream; + sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 : + sync_ctx.mbox->mbox_fd; + + ret = mbox_sync_do(&sync_ctx, flags); + + if (ret < 0) + mail_index_sync_rollback(&index_sync_ctx); + else if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } + sync_ctx.t = NULL; + sync_ctx.index_sync_ctx = NULL; + + if (ret == 0 && mbox->mbox_fd != -1 && sync_ctx.keep_recent && + !readonly) { + /* try to set atime back to its original value. + (it'll fail with EPERM for shared mailboxes where we aren't + the file's owner) */ + struct utimbuf buf; + struct stat st; + + if (fstat(mbox->mbox_fd, &st) < 0) + mbox_set_syscall_error(mbox, "fstat()"); + else { + buf.modtime = st.st_mtime; + buf.actime = sync_ctx.orig_atime; + if (utime(mailbox_get_path(&mbox->box), &buf) < 0 && + errno != EPERM) + mbox_set_syscall_error(mbox, "utime()"); + } + } + + i_assert(*lock_id != 0); + + if (mbox->storage->storage.set->mail_nfs_storage && + mbox->mbox_fd != -1) { + if (fdatasync(mbox->mbox_fd) < 0) { + mbox_set_syscall_error(mbox, "fdatasync()"); + ret = -1; + } + } + + mbox_sync_context_free(&sync_ctx); + return ret; +} + +int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags) +{ + unsigned int lock_id = 0; + int ret; + + i_assert(mbox->mbox_lock_type != F_RDLCK || + (flags & MBOX_SYNC_READONLY) != 0); + + mbox->syncing = TRUE; + ret = mbox_sync_int(mbox, flags, &lock_id); + mbox->syncing = FALSE; + + if (lock_id != 0) { + if (ret < 0) { + /* syncing failed, don't leave it locked */ + mbox_unlock(mbox, lock_id); + } else if ((flags & MBOX_SYNC_LOCK_READING) == 0) { + if (mbox_unlock(mbox, lock_id) < 0) + ret = -1; + } else if (mbox->mbox_lock_type != F_RDLCK) { + /* drop to read lock */ + unsigned int read_lock_id = 0; + + if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0) + ret = -1; + if (mbox_unlock(mbox, lock_id) < 0) + ret = -1; + } + } + + mailbox_sync_notify(&mbox->box, 0, 0); + return ret; +} + +struct mailbox_sync_context * +mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + enum mbox_sync_flags mbox_sync_flags = 0; + int ret = 0; + + if (index_mailbox_want_full_sync(&mbox->box, flags)) { + if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 && + !mbox->storage->set->mbox_very_dirty_syncs) + mbox_sync_flags |= MBOX_SYNC_UNDIRTY; + if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0) + mbox_sync_flags |= MBOX_SYNC_REWRITE; + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) { + mbox_sync_flags |= MBOX_SYNC_UNDIRTY | + MBOX_SYNC_REWRITE | MBOX_SYNC_FORCE_SYNC; + } + + ret = mbox_sync(mbox, mbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +} diff --git a/src/lib-storage/index/pop3c/Makefile.am b/src/lib-storage/index/pop3c/Makefile.am new file mode 100644 index 0000000..868dc1e --- /dev/null +++ b/src/lib-storage/index/pop3c/Makefile.am @@ -0,0 +1,28 @@ +noinst_LTLIBRARIES = libstorage_pop3c.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-dns \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_pop3c_la_SOURCES = \ + pop3c-client.c \ + pop3c-mail.c \ + pop3c-settings.c \ + pop3c-storage.c \ + pop3c-sync.c + +headers = \ + pop3c-client.h \ + pop3c-settings.h \ + pop3c-storage.h \ + pop3c-sync.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/pop3c/Makefile.in b/src/lib-storage/index/pop3c/Makefile.in new file mode 100644 index 0000000..81f62db --- /dev/null +++ b/src/lib-storage/index/pop3c/Makefile.in @@ -0,0 +1,837 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/pop3c +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_pop3c_la_LIBADD = +am_libstorage_pop3c_la_OBJECTS = pop3c-client.lo pop3c-mail.lo \ + pop3c-settings.lo pop3c-storage.lo pop3c-sync.lo +libstorage_pop3c_la_OBJECTS = $(am_libstorage_pop3c_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/pop3c-client.Plo \ + ./$(DEPDIR)/pop3c-mail.Plo ./$(DEPDIR)/pop3c-settings.Plo \ + ./$(DEPDIR)/pop3c-storage.Plo ./$(DEPDIR)/pop3c-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_pop3c_la_SOURCES) +DIST_SOURCES = $(libstorage_pop3c_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_pop3c.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-dns \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_pop3c_la_SOURCES = \ + pop3c-client.c \ + pop3c-mail.c \ + pop3c-settings.c \ + pop3c-storage.c \ + pop3c-sync.c + +headers = \ + pop3c-client.h \ + pop3c-settings.h \ + pop3c-storage.h \ + pop3c-sync.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/pop3c/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/pop3c/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_pop3c.la: $(libstorage_pop3c_la_OBJECTS) $(libstorage_pop3c_la_DEPENDENCIES) $(EXTRA_libstorage_pop3c_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_pop3c_la_OBJECTS) $(libstorage_pop3c_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-client.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/pop3c-client.Plo + -rm -f ./$(DEPDIR)/pop3c-mail.Plo + -rm -f ./$(DEPDIR)/pop3c-settings.Plo + -rm -f ./$(DEPDIR)/pop3c-storage.Plo + -rm -f ./$(DEPDIR)/pop3c-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/pop3c-client.Plo + -rm -f ./$(DEPDIR)/pop3c-mail.Plo + -rm -f ./$(DEPDIR)/pop3c-settings.Plo + -rm -f ./$(DEPDIR)/pop3c-storage.Plo + -rm -f ./$(DEPDIR)/pop3c-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/pop3c/pop3c-client.c b/src/lib-storage/index/pop3c/pop3c-client.c new file mode 100644 index 0000000..44544a3 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-client.c @@ -0,0 +1,902 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "net.h" +#include "istream.h" +#include "istream-chain.h" +#include "istream-dot.h" +#include "istream-seekable.h" +#include "ostream.h" +#include "iostream-rawlog.h" +#include "iostream-ssl.h" +#include "safe-mkstemp.h" +#include "base64.h" +#include "str.h" +#include "dns-lookup.h" +#include "pop3c-client.h" + +#include <unistd.h> + +#define POP3C_MAX_INBUF_SIZE (1024*32) +#define POP3C_DNS_LOOKUP_TIMEOUT_MSECS (1000*30) +#define POP3C_CONNECT_TIMEOUT_MSECS (1000*30) +#define POP3C_COMMAND_TIMEOUT_MSECS (1000*60*5) + +enum pop3c_client_state { + /* No connection */ + POP3C_CLIENT_STATE_DISCONNECTED = 0, + /* Trying to connect */ + POP3C_CLIENT_STATE_CONNECTING, + POP3C_CLIENT_STATE_STARTTLS, + /* Connected, trying to authenticate */ + POP3C_CLIENT_STATE_USER, + POP3C_CLIENT_STATE_AUTH, + POP3C_CLIENT_STATE_PASS, + /* Post-authentication, asking for capabilities */ + POP3C_CLIENT_STATE_CAPA, + /* Authenticated, ready to accept commands */ + POP3C_CLIENT_STATE_DONE +}; + +struct pop3c_client_sync_cmd_ctx { + enum pop3c_command_state state; + char *reply; +}; + +struct pop3c_client_cmd { + struct istream *input; + struct istream_chain *chain; + bool reading_dot; + + pop3c_cmd_callback_t *callback; + void *context; +}; + +struct pop3c_client { + pool_t pool; + struct event *event; + struct pop3c_client_settings set; + struct ssl_iostream_context *ssl_ctx; + struct ip_addr ip; + + int fd; + struct io *io; + struct istream *input, *raw_input; + struct ostream *output, *raw_output; + struct ssl_iostream *ssl_iostream; + struct timeout *to; + struct dns_lookup *dns_lookup; + + enum pop3c_client_state state; + enum pop3c_capability capabilities; + const char *auth_mech; + + pop3c_login_callback_t *login_callback; + void *login_context; + + ARRAY(struct pop3c_client_cmd) commands; + const char *input_line; + struct istream *dot_input; + + bool running:1; +}; + +static void +pop3c_dns_callback(const struct dns_lookup_result *result, + struct pop3c_client *client); +static void pop3c_client_connect_ip(struct pop3c_client *client); +static int pop3c_client_ssl_init(struct pop3c_client *client); +static void pop3c_client_input(struct pop3c_client *client); + +struct pop3c_client * +pop3c_client_init(const struct pop3c_client_settings *set, + struct event *event_parent) +{ + struct pop3c_client *client; + const char *error; + pool_t pool; + + pool = pool_alloconly_create("pop3c client", 1024); + client = p_new(pool, struct pop3c_client, 1); + client->pool = pool; + client->event = event_create(event_parent); + client->fd = -1; + p_array_init(&client->commands, pool, 16); + + client->set.debug = set->debug; + client->set.host = p_strdup(pool, set->host); + client->set.port = set->port; + client->set.master_user = p_strdup_empty(pool, set->master_user); + client->set.username = p_strdup(pool, set->username); + client->set.password = p_strdup(pool, set->password); + client->set.dns_client_socket_path = + p_strdup(pool, set->dns_client_socket_path); + client->set.temp_path_prefix = p_strdup(pool, set->temp_path_prefix); + client->set.rawlog_dir = p_strdup(pool, set->rawlog_dir); + client->set.ssl_mode = set->ssl_mode; + + if (set->ssl_mode != POP3C_CLIENT_SSL_MODE_NONE) { + ssl_iostream_settings_init_from(client->pool, &client->set.ssl_set, &set->ssl_set); + client->set.ssl_set.verbose_invalid_cert = !client->set.ssl_set.allow_invalid_cert; + if (ssl_iostream_client_context_cache_get(&set->ssl_set, + &client->ssl_ctx, + &error) < 0) { + i_error("pop3c(%s:%u): Couldn't initialize SSL context: %s", + set->host, set->port, error); + } + } + return client; +} + +static void +client_login_callback(struct pop3c_client *client, + enum pop3c_command_state state, const char *reason) +{ + pop3c_login_callback_t *callback = client->login_callback; + void *context = client->login_context; + + if (client->login_callback != NULL) { + client->login_callback = NULL; + client->login_context = NULL; + callback(state, reason, context); + } +} + +static void +pop3c_client_async_callback(struct pop3c_client *client, + enum pop3c_command_state state, const char *reply) +{ + struct pop3c_client_cmd *cmd, cmd_copy; + bool running = client->running; + + i_assert(reply != NULL); + i_assert(array_count(&client->commands) > 0); + + cmd = array_front_modifiable(&client->commands); + if (cmd->input != NULL && state == POP3C_COMMAND_STATE_OK && + !cmd->reading_dot) { + /* read the full input into seekable-istream before calling + the callback */ + i_assert(client->dot_input == NULL); + i_stream_chain_append(cmd->chain, client->input); + client->dot_input = cmd->input; + cmd->reading_dot = TRUE; + return; + } + cmd_copy = *cmd; + array_pop_front(&client->commands); + + if (cmd_copy.input != NULL) { + i_stream_seek(cmd_copy.input, 0); + i_stream_unref(&cmd_copy.input); + } + if (cmd_copy.callback != NULL) + cmd_copy.callback(state, reply, cmd_copy.context); + if (running) + io_loop_stop(current_ioloop); +} + +static void +pop3c_client_async_callback_disconnected(struct pop3c_client *client) +{ + pop3c_client_async_callback(client, POP3C_COMMAND_STATE_DISCONNECTED, + "Disconnected"); +} + +static void pop3c_client_disconnect(struct pop3c_client *client) +{ + client->state = POP3C_CLIENT_STATE_DISCONNECTED; + + if (client->running) + io_loop_stop(current_ioloop); + + if (client->dns_lookup != NULL) + dns_lookup_abort(&client->dns_lookup); + timeout_remove(&client->to); + io_remove(&client->io); + i_stream_destroy(&client->input); + o_stream_destroy(&client->output); + ssl_iostream_destroy(&client->ssl_iostream); + i_close_fd(&client->fd); + while (array_count(&client->commands) > 0) + pop3c_client_async_callback_disconnected(client); + client_login_callback(client, POP3C_COMMAND_STATE_DISCONNECTED, + "Disconnected"); +} + +void pop3c_client_deinit(struct pop3c_client **_client) +{ + struct pop3c_client *client = *_client; + + pop3c_client_disconnect(client); + if (client->ssl_ctx != NULL) + ssl_iostream_context_unref(&client->ssl_ctx); + event_unref(&client->event); + pool_unref(&client->pool); +} + +static void pop3c_client_ioloop_changed(struct pop3c_client *client) +{ + if (client->to != NULL) + client->to = io_loop_move_timeout(&client->to); + if (client->io != NULL) + client->io = io_loop_move_io(&client->io); + if (client->output != NULL) + o_stream_switch_ioloop(client->output); +} + +static void pop3c_client_timeout(struct pop3c_client *client) +{ + switch (client->state) { + case POP3C_CLIENT_STATE_CONNECTING: + i_error("pop3c(%s): connect(%s, %u) timed out after %u seconds", + client->set.host, net_ip2addr(&client->ip), + client->set.port, POP3C_CONNECT_TIMEOUT_MSECS/1000); + break; + case POP3C_CLIENT_STATE_DONE: + i_error("pop3c(%s): Command timed out after %u seconds", + client->set.host, POP3C_COMMAND_TIMEOUT_MSECS/1000); + break; + default: + i_error("pop3c(%s): Authentication timed out after %u seconds", + client->set.host, POP3C_CONNECT_TIMEOUT_MSECS/1000); + break; + } + pop3c_client_disconnect(client); +} + +static int pop3c_client_dns_lookup(struct pop3c_client *client) +{ + struct dns_lookup_settings dns_set; + + i_assert(client->state == POP3C_CLIENT_STATE_CONNECTING); + + if (client->set.dns_client_socket_path[0] == '\0') { + struct ip_addr *ips; + unsigned int ips_count; + int ret; + + ret = net_gethostbyname(client->set.host, &ips, &ips_count); + if (ret != 0) { + i_error("pop3c(%s): net_gethostbyname() failed: %s", + client->set.host, net_gethosterror(ret)); + return -1; + } + i_assert(ips_count > 0); + client->ip = ips[0]; + pop3c_client_connect_ip(client); + } else { + i_zero(&dns_set); + dns_set.dns_client_socket_path = + client->set.dns_client_socket_path; + dns_set.timeout_msecs = POP3C_DNS_LOOKUP_TIMEOUT_MSECS; + dns_set.event_parent = client->event; + if (dns_lookup(client->set.host, &dns_set, + pop3c_dns_callback, client, + &client->dns_lookup) < 0) + return -1; + } + return 0; +} + +void pop3c_client_wait_one(struct pop3c_client *client) +{ + struct ioloop *ioloop, *prev_ioloop = current_ioloop; + bool timeout_added = FALSE, failed = FALSE; + + if (client->state == POP3C_CLIENT_STATE_DISCONNECTED && + array_count(&client->commands) > 0) { + while (array_count(&client->commands) > 0) + pop3c_client_async_callback_disconnected(client); + return; + } + + i_assert(client->fd != -1 || + client->state == POP3C_CLIENT_STATE_CONNECTING); + i_assert(array_count(&client->commands) > 0 || + client->state == POP3C_CLIENT_STATE_CONNECTING); + + ioloop = io_loop_create(); + pop3c_client_ioloop_changed(client); + + if (client->ip.family == 0) { + /* we're connecting, start DNS lookup after our ioloop + is created */ + if (pop3c_client_dns_lookup(client) < 0) + failed = TRUE; + } else if (client->to == NULL) { + client->to = timeout_add(POP3C_COMMAND_TIMEOUT_MSECS, + pop3c_client_timeout, client); + timeout_added = TRUE; + } + + if (!failed) { + client->running = TRUE; + io_loop_run(ioloop); + client->running = FALSE; + } + + if (timeout_added && client->to != NULL) + timeout_remove(&client->to); + + io_loop_set_current(prev_ioloop); + pop3c_client_ioloop_changed(client); + io_loop_set_current(ioloop); + io_loop_destroy(&ioloop); +} + +static void pop3c_client_starttls(struct pop3c_client *client) +{ + o_stream_nsend_str(client->output, "STLS\r\n"); + client->state = POP3C_CLIENT_STATE_STARTTLS; +} + +static void pop3c_client_authenticate1(struct pop3c_client *client) +{ + const struct pop3c_client_settings *set = &client->set; + + if (client->set.debug) { + if (set->master_user == NULL) { + i_debug("pop3c(%s): Authenticating as '%s' (with USER+PASS)", + client->set.host, set->username); + } else { + i_debug("pop3c(%s): Authenticating as master user '%s' for user '%s' (with SASL PLAIN)", + client->set.host, set->master_user, + set->username); + } + } + + if (set->master_user == NULL) { + o_stream_nsend_str(client->output, + t_strdup_printf("USER %s\r\n", set->username)); + client->state = POP3C_CLIENT_STATE_USER; + } else { + client->state = POP3C_CLIENT_STATE_AUTH; + o_stream_nsend_str(client->output, "AUTH PLAIN\r\n"); + } +} + +static const char * +pop3c_client_get_sasl_plain_request(struct pop3c_client *client) +{ + const struct pop3c_client_settings *set = &client->set; + string_t *in, *out; + + in = t_str_new(128); + if (set->master_user != NULL) { + str_append(in, set->username); + str_append_c(in, '\0'); + str_append(in, set->master_user); + } else { + str_append_c(in, '\0'); + str_append(in, set->username); + } + str_append_c(in, '\0'); + str_append(in, set->password); + + out = t_str_new(128); + base64_encode(str_data(in), str_len(in), out); + str_append(out, "\r\n"); + return str_c(out); +} + +static void pop3c_client_login_finished(struct pop3c_client *client) +{ + io_remove(&client->io); + client->io = io_add(client->fd, IO_READ, pop3c_client_input, client); + + timeout_remove(&client->to); + client->state = POP3C_CLIENT_STATE_DONE; + + if (client->running) + io_loop_stop(current_ioloop); +} + +static int +pop3c_client_prelogin_input_line(struct pop3c_client *client, const char *line) +{ + bool success = line[0] == '+'; + const char *reply; + + switch (client->state) { + case POP3C_CLIENT_STATE_CONNECTING: + if (!success) { + i_error("pop3c(%s): Server sent invalid banner: %s", + client->set.host, line); + return -1; + } + if (client->set.ssl_mode == POP3C_CLIENT_SSL_MODE_STARTTLS) + pop3c_client_starttls(client); + else + pop3c_client_authenticate1(client); + break; + case POP3C_CLIENT_STATE_STARTTLS: + if (!success) { + i_error("pop3c(%s): STLS failed: %s", + client->set.host, line); + return -1; + } + if (pop3c_client_ssl_init(client) < 0) + pop3c_client_disconnect(client); + break; + case POP3C_CLIENT_STATE_USER: + if (!success) { + i_error("pop3c(%s): USER failed: %s", + client->set.host, line); + return -1; + } + + /* the PASS reply can take a long time. + switch to command timeout. */ + timeout_remove(&client->to); + client->to = timeout_add(POP3C_COMMAND_TIMEOUT_MSECS, + pop3c_client_timeout, client); + + o_stream_nsend_str(client->output, + t_strdup_printf("PASS %s\r\n", client->set.password)); + client->state = POP3C_CLIENT_STATE_PASS; + client->auth_mech = "USER+PASS"; + break; + case POP3C_CLIENT_STATE_AUTH: + if (line[0] != '+') { + i_error("pop3c(%s): AUTH PLAIN failed: %s", + client->set.host, line); + return -1; + } + o_stream_nsend_str(client->output, + pop3c_client_get_sasl_plain_request(client)); + client->state = POP3C_CLIENT_STATE_PASS; + client->auth_mech = "AUTH PLAIN"; + break; + case POP3C_CLIENT_STATE_PASS: + if (client->login_callback != NULL) { + reply = strncasecmp(line, "+OK ", 4) == 0 ? line + 4 : + strncasecmp(line, "-ERR ", 5) == 0 ? line + 5 : + line; + client_login_callback(client, success ? + POP3C_COMMAND_STATE_OK : + POP3C_COMMAND_STATE_ERR, reply); + } else if (!success) { + i_error("pop3c(%s): Authentication via %s failed: %s", + client->set.host, client->auth_mech, line); + } + if (!success) + return -1; + + o_stream_nsend_str(client->output, "CAPA\r\n"); + client->state = POP3C_CLIENT_STATE_CAPA; + break; + case POP3C_CLIENT_STATE_CAPA: + if (strncasecmp(line, "-ERR", 4) == 0) { + /* CAPA command not supported. some commands still + support UIDL though. */ + client->capabilities |= POP3C_CAPABILITY_UIDL; + pop3c_client_login_finished(client); + break; + } else if (strcmp(line, ".") == 0) { + pop3c_client_login_finished(client); + break; + } + if ((client->set.parsed_features & POP3C_FEATURE_NO_PIPELINING) == 0 && + strcasecmp(line, "PIPELINING") == 0) + client->capabilities |= POP3C_CAPABILITY_PIPELINING; + else if (strcasecmp(line, "TOP") == 0) + client->capabilities |= POP3C_CAPABILITY_TOP; + else if (strcasecmp(line, "UIDL") == 0) + client->capabilities |= POP3C_CAPABILITY_UIDL; + break; + case POP3C_CLIENT_STATE_DISCONNECTED: + case POP3C_CLIENT_STATE_DONE: + i_unreached(); + } + return 0; +} + +static void pop3c_client_prelogin_input(struct pop3c_client *client) +{ + const char *line, *errstr; + + i_assert(client->state != POP3C_CLIENT_STATE_DONE); + + /* we need to read as much as we can with SSL streams to avoid + hanging */ + while ((line = i_stream_read_next_line(client->input)) != NULL) { + if (pop3c_client_prelogin_input_line(client, line) < 0) { + pop3c_client_disconnect(client); + return; + } + } + + if (client->input->closed || client->input->eof || + client->input->stream_errno != 0) { + /* disconnected */ + if (client->ssl_iostream == NULL) { + i_error("pop3c(%s): Server disconnected unexpectedly", + client->set.host); + } else { + errstr = ssl_iostream_get_last_error(client->ssl_iostream); + if (errstr == NULL) { + errstr = client->input->stream_errno == 0 ? "EOF" : + strerror(client->input->stream_errno); + } + i_error("pop3c(%s): Server disconnected: %s", + client->set.host, errstr); + } + pop3c_client_disconnect(client); + } +} + +static int pop3c_client_ssl_handshaked(const char **error_r, void *context) +{ + struct pop3c_client *client = context; + const char *error; + + if (ssl_iostream_check_cert_validity(client->ssl_iostream, + client->set.host, &error) == 0) { + if (client->set.debug) { + i_debug("pop3c(%s): SSL handshake successful", + client->set.host); + } + return 0; + } else if (client->set.ssl_set.allow_invalid_cert) { + if (client->set.debug) { + i_debug("pop3c(%s): SSL handshake successful, " + "ignoring invalid certificate: %s", + client->set.host, error); + } + return 0; + } else { + *error_r = error; + return -1; + } +} + +static int pop3c_client_ssl_init(struct pop3c_client *client) +{ + const char *error; + + if (client->ssl_ctx == NULL) { + i_error("pop3c(%s): No SSL context", client->set.host); + return -1; + } + + if (client->set.debug) + i_debug("pop3c(%s): Starting SSL handshake", client->set.host); + + if (client->raw_input != client->input) { + /* recreate rawlog after STARTTLS */ + i_stream_ref(client->raw_input); + o_stream_ref(client->raw_output); + i_stream_destroy(&client->input); + o_stream_destroy(&client->output); + client->input = client->raw_input; + client->output = client->raw_output; + } + + if (io_stream_create_ssl_client(client->ssl_ctx, client->set.host, + &client->set.ssl_set, &client->input, + &client->output, &client->ssl_iostream, &error) < 0) { + i_error("pop3c(%s): Couldn't initialize SSL client: %s", + client->set.host, error); + return -1; + } + ssl_iostream_set_handshake_callback(client->ssl_iostream, + pop3c_client_ssl_handshaked, + client); + if (ssl_iostream_handshake(client->ssl_iostream) < 0) { + i_error("pop3c(%s): SSL handshake failed: %s", client->set.host, + ssl_iostream_get_last_error(client->ssl_iostream)); + return -1; + } + + if (*client->set.rawlog_dir != '\0') { + iostream_rawlog_create(client->set.rawlog_dir, + &client->input, &client->output); + } + return 0; +} + +static void pop3c_client_connected(struct pop3c_client *client) +{ + int err; + + err = net_geterror(client->fd); + if (err != 0) { + i_error("pop3c(%s): connect(%s, %u) failed: %s", + client->set.host, net_ip2addr(&client->ip), + client->set.port, strerror(err)); + pop3c_client_disconnect(client); + return; + } + io_remove(&client->io); + client->io = io_add(client->fd, IO_READ, + pop3c_client_prelogin_input, client); + + if (client->set.ssl_mode == POP3C_CLIENT_SSL_MODE_IMMEDIATE) { + if (pop3c_client_ssl_init(client) < 0) + pop3c_client_disconnect(client); + } +} + +static void pop3c_client_connect_ip(struct pop3c_client *client) +{ + client->fd = net_connect_ip(&client->ip, client->set.port, NULL); + if (client->fd == -1) { + pop3c_client_disconnect(client); + return; + } + + client->input = client->raw_input = + i_stream_create_fd(client->fd, POP3C_MAX_INBUF_SIZE); + client->output = client->raw_output = + o_stream_create_fd(client->fd, SIZE_MAX); + o_stream_set_no_error_handling(client->output, TRUE); + + if (*client->set.rawlog_dir != '\0' && + client->set.ssl_mode != POP3C_CLIENT_SSL_MODE_IMMEDIATE) { + iostream_rawlog_create(client->set.rawlog_dir, + &client->input, &client->output); + } + client->io = io_add(client->fd, IO_WRITE, + pop3c_client_connected, client); + client->to = timeout_add(POP3C_CONNECT_TIMEOUT_MSECS, + pop3c_client_timeout, client); + if (client->set.debug) { + i_debug("pop3c(%s): Connecting to %s:%u", client->set.host, + net_ip2addr(&client->ip), client->set.port); + } +} + +static void +pop3c_dns_callback(const struct dns_lookup_result *result, + struct pop3c_client *client) +{ + client->dns_lookup = NULL; + + if (result->ret != 0) { + i_error("pop3c(%s): dns_lookup() failed: %s", + client->set.host, result->error); + pop3c_client_disconnect(client); + return; + } + + i_assert(result->ips_count > 0); + client->ip = result->ips[0]; + pop3c_client_connect_ip(client); +} + +void pop3c_client_login(struct pop3c_client *client, + pop3c_login_callback_t *callback, void *context) +{ + if (client->fd != -1) { + i_assert(callback == NULL); + return; + } + i_assert(client->login_callback == NULL); + client->login_callback = callback; + client->login_context = context; + client->state = POP3C_CLIENT_STATE_CONNECTING; + + if (client->set.debug) + i_debug("pop3c(%s): Looking up IP address", client->set.host); +} + +bool pop3c_client_is_connected(struct pop3c_client *client) +{ + return client->fd != -1; +} + +enum pop3c_capability +pop3c_client_get_capabilities(struct pop3c_client *client) +{ + return client->capabilities; +} + +static int pop3c_client_dot_input(struct pop3c_client *client) +{ + ssize_t ret; + + while ((ret = i_stream_read(client->dot_input)) > 0 || ret == -2) { + i_stream_skip(client->dot_input, + i_stream_get_data_size(client->dot_input)); + } + if (ret == 0) + return 0; + i_assert(ret == -1); + + if (client->dot_input->stream_errno == 0) + ret = 1; + client->dot_input = NULL; + + if (ret > 0) { + /* currently we don't actually care about preserving the + +OK reply line for multi-line replies, so just return + it as empty */ + pop3c_client_async_callback(client, POP3C_COMMAND_STATE_OK, ""); + return 1; + } else { + pop3c_client_async_callback_disconnected(client); + return -1; + } +} + +static int +pop3c_client_input_next_reply(struct pop3c_client *client) +{ + const char *line; + enum pop3c_command_state state; + + line = i_stream_read_next_line(client->input); + if (line == NULL) + return client->input->eof ? -1 : 0; + + if (strncasecmp(line, "+OK", 3) == 0) { + line += 3; + state = POP3C_COMMAND_STATE_OK; + } else if (strncasecmp(line, "-ERR", 4) == 0) { + line += 4; + state = POP3C_COMMAND_STATE_ERR; + } else { + i_error("pop3c(%s): Server sent unrecognized line: %s", + client->set.host, line); + state = POP3C_COMMAND_STATE_ERR; + } + if (line[0] == ' ') + line++; + if (array_count(&client->commands) == 0) { + i_error("pop3c(%s): Server sent line when no command was running: %s", + client->set.host, line); + } else { + pop3c_client_async_callback(client, state, line); + } + return 1; +} + +static void pop3c_client_input(struct pop3c_client *client) +{ + int ret; + + if (client->to != NULL) + timeout_reset(client->to); + do { + if (client->dot_input != NULL) { + /* continue reading the current multiline reply */ + if ((ret = pop3c_client_dot_input(client)) == 0) + return; + } else { + ret = pop3c_client_input_next_reply(client); + } + } while (ret > 0); + + if (ret < 0) { + i_error("pop3c(%s): Server disconnected unexpectedly", + client->set.host); + pop3c_client_disconnect(client); + } +} + +static void pop3c_client_cmd_reply(enum pop3c_command_state state, + const char *reply, void *context) +{ + struct pop3c_client_sync_cmd_ctx *ctx = context; + + i_assert(ctx->reply == NULL); + + ctx->state = state; + ctx->reply = i_strdup(reply); +} + +int pop3c_client_cmd_line(struct pop3c_client *client, const char *cmdline, + const char **reply_r) +{ + struct pop3c_client_sync_cmd_ctx ctx; + + i_zero(&ctx); + pop3c_client_cmd_line_async(client, cmdline, pop3c_client_cmd_reply, &ctx); + while (ctx.reply == NULL) + pop3c_client_wait_one(client); + *reply_r = t_strdup(ctx.reply); + i_free(ctx.reply); + return ctx.state == POP3C_COMMAND_STATE_OK ? 0 : -1; +} + +struct pop3c_client_cmd * +pop3c_client_cmd_line_async(struct pop3c_client *client, const char *cmdline, + pop3c_cmd_callback_t *callback, void *context) +{ + struct pop3c_client_cmd *cmd; + + if ((client->capabilities & POP3C_CAPABILITY_PIPELINING) == 0) { + while (array_count(&client->commands) > 0) + pop3c_client_wait_one(client); + } + i_assert(client->state == POP3C_CLIENT_STATE_DISCONNECTED || + client->state == POP3C_CLIENT_STATE_DONE); + if (client->state == POP3C_CLIENT_STATE_DONE) + o_stream_nsend_str(client->output, cmdline); + + cmd = array_append_space(&client->commands); + cmd->callback = callback; + cmd->context = context; + return cmd; +} + +void pop3c_client_cmd_line_async_nocb(struct pop3c_client *client, + const char *cmdline) +{ + pop3c_client_cmd_line_async(client, cmdline, NULL, NULL); +} + +static int seekable_fd_callback(const char **path_r, void *context) +{ + struct pop3c_client *client = context; + string_t *path; + int fd; + + path = t_str_new(128); + str_append(path, client->set.temp_path_prefix); + fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (i_unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_close_fd(&fd); + return -1; + } + + *path_r = str_c(path); + return fd; +} + +int pop3c_client_cmd_stream(struct pop3c_client *client, const char *cmdline, + struct istream **input_r, const char **error_r) +{ + struct pop3c_client_sync_cmd_ctx ctx; + const char *reply; + + if (client->state == POP3C_CLIENT_STATE_DISCONNECTED) { + *error_r = "Disconnected from server"; + return -1; + } + + i_zero(&ctx); + *input_r = pop3c_client_cmd_stream_async(client, cmdline, + pop3c_client_cmd_reply, &ctx); + while (ctx.reply == NULL) + pop3c_client_wait_one(client); + reply = t_strdup(ctx.reply); + i_free(ctx.reply); + + if (ctx.state == POP3C_COMMAND_STATE_OK) + return 0; + i_stream_unref(input_r); + *error_r = reply; + return -1; +} + +struct istream * +pop3c_client_cmd_stream_async(struct pop3c_client *client, const char *cmdline, + pop3c_cmd_callback_t *callback, void *context) +{ + struct istream *input, *inputs[2]; + struct pop3c_client_cmd *cmd; + + cmd = pop3c_client_cmd_line_async(client, cmdline, callback, context); + + input = i_stream_create_chain(&cmd->chain, POP3C_MAX_INBUF_SIZE); + inputs[0] = i_stream_create_dot(input, TRUE); + inputs[1] = NULL; + cmd->input = i_stream_create_seekable(inputs, POP3C_MAX_INBUF_SIZE, + seekable_fd_callback, client); + i_stream_unref(&input); + i_stream_unref(&inputs[0]); + + i_stream_ref(cmd->input); + return cmd->input; +} diff --git a/src/lib-storage/index/pop3c/pop3c-client.h b/src/lib-storage/index/pop3c/pop3c-client.h new file mode 100644 index 0000000..f0bbd64 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-client.h @@ -0,0 +1,86 @@ +#ifndef POP3C_CLIENT_H +#define POP3C_CLIENT_H + +#include "net.h" +#include "pop3c-settings.h" +#include "iostream-ssl.h" + +enum pop3c_capability { + POP3C_CAPABILITY_PIPELINING = 0x01, + POP3C_CAPABILITY_TOP = 0x02, + POP3C_CAPABILITY_UIDL = 0x04 +}; + +enum pop3c_command_state { + POP3C_COMMAND_STATE_OK, + POP3C_COMMAND_STATE_ERR, + POP3C_COMMAND_STATE_DISCONNECTED +}; + +enum pop3c_client_ssl_mode { + POP3C_CLIENT_SSL_MODE_NONE, + POP3C_CLIENT_SSL_MODE_IMMEDIATE, + POP3C_CLIENT_SSL_MODE_STARTTLS +}; + +struct pop3c_client_settings { + const char *host; + in_port_t port; + + const char *master_user; + const char *username; + const char *password; + + const char *dns_client_socket_path; + const char *temp_path_prefix; + + enum pop3c_client_ssl_mode ssl_mode; + enum pop3c_features parsed_features; + struct ssl_iostream_settings ssl_set; + + const char *rawlog_dir; + const char *ssl_crypto_device; + bool debug; +}; + +typedef void pop3c_login_callback_t(enum pop3c_command_state state, + const char *reply, void *context); +typedef void pop3c_cmd_callback_t(enum pop3c_command_state state, + const char *reply, void *context); + +struct pop3c_client * +pop3c_client_init(const struct pop3c_client_settings *set, + struct event *event_parent); +void pop3c_client_deinit(struct pop3c_client **client); + +void pop3c_client_login(struct pop3c_client *client, + pop3c_login_callback_t *callback, void *context); + +bool pop3c_client_is_connected(struct pop3c_client *client); +enum pop3c_capability +pop3c_client_get_capabilities(struct pop3c_client *client); + +/* Returns 0 if received +OK reply, reply contains the text without the +OK. + Returns -1 if received -ERR reply or disconnected. */ +int pop3c_client_cmd_line(struct pop3c_client *client, const char *cmdline, + const char **reply_r); +/* Start the command asynchronously. Call the callback when finished. */ +struct pop3c_client_cmd * +pop3c_client_cmd_line_async(struct pop3c_client *client, const char *cmdline, + pop3c_cmd_callback_t *callback, void *context); +/* Send a command, don't care if it succeeds or not. */ +void pop3c_client_cmd_line_async_nocb(struct pop3c_client *client, + const char *cmdline); +/* Returns 0 and stream if succeeded, -1 and error if received -ERR reply or + disconnected. */ +int pop3c_client_cmd_stream(struct pop3c_client *client, const char *cmdline, + struct istream **input_r, const char **error_r); +/* Start the command asynchronously. Call the callback when finished. */ +struct istream * +pop3c_client_cmd_stream_async(struct pop3c_client *client, const char *cmdline, + pop3c_cmd_callback_t *callback, void *context); +/* Wait for the next async command to finish. It's an error to call this when + there are no pending async commands. */ +void pop3c_client_wait_one(struct pop3c_client *client); + +#endif diff --git a/src/lib-storage/index/pop3c/pop3c-mail.c b/src/lib-storage/index/pop3c/pop3c-mail.c new file mode 100644 index 0000000..27a8427 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-mail.c @@ -0,0 +1,304 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "index-mail.h" +#include "pop3c-client.h" +#include "pop3c-sync.h" +#include "pop3c-storage.h" + +struct mail * +pop3c_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct pop3c_mail *mail; + pool_t pool; + + pool = pool_alloconly_create("mail", 2048); + mail = p_new(pool, struct pop3c_mail, 1); + + index_mail_init(&mail->imail, t, wanted_fields, wanted_headers, pool, NULL); + return &mail->imail.mail.mail; +} + +static void pop3c_mail_close(struct mail *_mail) +{ + struct pop3c_mail *pmail = POP3C_MAIL(_mail); + struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box); + + /* wait for any prefetch to finish before closing the mail */ + while (pmail->prefetching) + pop3c_client_wait_one(mbox->client); + i_stream_unref(&pmail->prefetch_stream); + index_mail_close(_mail); +} + +static int pop3c_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box); + int tz; + + if (mbox->storage->set->pop3c_quick_received_date) { + /* we don't care about the date, just return the current date */ + *date_r = ioloop_time; + return 0; + } + + /* FIXME: we could also parse the first Received: header and get + the date from there, but since this code is unlikely to be called + except during migration, I don't think it really matters. */ + return index_mail_get_date(_mail, date_r, &tz); +} + +static int pop3c_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + if (data->save_date == (time_t)-1) { + /* FIXME: we could use a value stored in cache */ + if (pop3c_mail_get_received_date(_mail, date_r) < 0) + return -1; + return 0; + } + *date_r = data->save_date; + return 0; +} + +static int pop3c_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box); + struct message_size hdr_size, body_size; + struct istream *input; + + if (mail->data.virtual_size != UOFF_T_MAX) { + /* virtual size is already known. it's the same as our + (correct) physical size */ + *size_r = mail->data.virtual_size; + return 0; + } + if (index_mail_get_physical_size(_mail, size_r) == 0) { + *size_r = mail->data.physical_size; + return 0; + } + + if (_mail->lookup_abort == MAIL_LOOKUP_ABORT_READ_MAIL && + (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) { + /* kludge: we want output for POP3 LIST with + pop3_fast_size_lookups=yes. use the remote's LIST values + regardless of their correctness */ + if (mbox->msg_sizes == NULL) { + if (pop3c_sync_get_sizes(mbox) < 0) + return -1; + } + i_assert(_mail->seq <= mbox->msg_count); + *size_r = mbox->msg_sizes[_mail->seq-1]; + return 0; + } + + /* slow way: get the whole message body */ + if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0) + return -1; + + i_assert(mail->data.physical_size != UOFF_T_MAX); + *size_r = mail->data.physical_size; + return 0; +} + +static void pop3c_mail_cache_size(struct index_mail *mail) +{ + uoff_t size; + + if (i_stream_get_size(mail->data.stream, TRUE, &size) <= 0) + return; + mail->data.virtual_size = size; + /* it'll be actually added to index when closing the mail in + index_mail_cache_sizes() */ +} + +static void +pop3c_mail_prefetch_done(enum pop3c_command_state state, + const char *reply ATTR_UNUSED, void *context) +{ + struct pop3c_mail *pmail = context; + + switch (state) { + case POP3C_COMMAND_STATE_OK: + break; + case POP3C_COMMAND_STATE_ERR: + case POP3C_COMMAND_STATE_DISCONNECTED: + i_stream_unref(&pmail->prefetch_stream); + /* let pop3c_mail_get_stream() figure out the error handling. + in case of a -ERR a retry might even work. */ + break; + } + pmail->prefetching = FALSE; +} + +static bool pop3c_mail_prefetch(struct mail *_mail) +{ + struct pop3c_mail *pmail = POP3C_MAIL(_mail); + struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box); + enum pop3c_capability capa; + const char *cmd; + + if (pmail->imail.data.access_part != 0 && + pmail->imail.data.stream == NULL && + mail_stream_access_start(_mail)) { + capa = pop3c_client_get_capabilities(mbox->client); + pmail->prefetching_body = (capa & POP3C_CAPABILITY_TOP) == 0 || + (pmail->imail.data.access_part & (READ_BODY | PARSE_BODY)) != 0; + if (pmail->prefetching_body) + cmd = t_strdup_printf("RETR %u\r\n", _mail->seq); + else + cmd = t_strdup_printf("TOP %u 0\r\n", _mail->seq); + + pmail->prefetching = TRUE; + pmail->prefetch_stream = + pop3c_client_cmd_stream_async(mbox->client, cmd, + pop3c_mail_prefetch_done, pmail); + i_stream_set_name(pmail->prefetch_stream, t_strcut(cmd, '\r')); + return !pmail->prefetching; + } + return index_mail_prefetch(_mail); +} + +static int +pop3c_mail_get_stream(struct mail *_mail, bool get_body, + struct message_size *hdr_size, + struct message_size *body_size, struct istream **stream_r) +{ + struct pop3c_mail *pmail = POP3C_MAIL(_mail); + struct index_mail *mail = &pmail->imail; + struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box); + enum pop3c_capability capa; + const char *name, *cmd, *error; + struct istream *input; + bool new_stream = FALSE; + + if ((mail->data.access_part & (READ_BODY | PARSE_BODY)) != 0) + get_body = TRUE; + + while (pmail->prefetching) { + /* wait for prefetch to finish */ + pop3c_client_wait_one(mbox->client); + } + + if (pmail->prefetch_stream != NULL && mail->data.stream == NULL) { + mail->data.stream = pmail->prefetch_stream; + pmail->prefetch_stream = NULL; + new_stream = TRUE; + } + + if (get_body && mail->data.stream != NULL) { + name = i_stream_get_name(mail->data.stream); + if (str_begins(name, "RETR")) { + /* we've fetched the body */ + } else if (str_begins(name, "TOP")) { + /* we've fetched the header, but we need the body + now too */ + index_mail_close_streams(mail); + } else { + i_panic("Unexpected POP3 stream name: %s", name); + } + } + + if (mail->data.stream == NULL) { + if (!mail_stream_access_start(_mail)) + return -1; + capa = pop3c_client_get_capabilities(mbox->client); + if (get_body || (capa & POP3C_CAPABILITY_TOP) == 0) { + cmd = t_strdup_printf("RETR %u\r\n", _mail->seq); + get_body = TRUE; + } else { + cmd = t_strdup_printf("TOP %u 0\r\n", _mail->seq); + } + if (pop3c_client_cmd_stream(mbox->client, cmd, + &input, &error) < 0) { + mail_storage_set_error(mbox->box.storage, + !pop3c_client_is_connected(mbox->client) ? + MAIL_ERROR_TEMP : MAIL_ERROR_EXPUNGED, error); + return -1; + } + mail->data.stream = input; + i_stream_set_name(mail->data.stream, t_strcut(cmd, '\r')); + new_stream = TRUE; + } + if (new_stream) { + if (mail->mail.v.istream_opened != NULL) { + if (mail->mail.v.istream_opened(_mail, + &mail->data.stream) < 0) { + index_mail_close_streams(mail); + return -1; + } + } + if (get_body) + pop3c_mail_cache_size(mail); + } + /* if this stream is used by some filter stream, make the + filter stream blocking */ + mail->data.stream->blocking = TRUE; + return index_mail_init_stream(mail, hdr_size, body_size, stream_r); +} + +static int +pop3c_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box); + + switch (field) { + case MAIL_FETCH_UIDL_BACKEND: + case MAIL_FETCH_GUID: + if (mbox->msg_uidls == NULL) { + if (pop3c_sync_get_uidls(mbox) < 0) + return -1; + } + i_assert(_mail->seq <= mbox->msg_count); + *value_r = mbox->msg_uidls[_mail->seq-1]; + return 0; + default: + return index_mail_get_special(_mail, field, value_r); + } +} + +struct mail_vfuncs pop3c_mail_vfuncs = { + pop3c_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + pop3c_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + pop3c_mail_get_received_date, + pop3c_mail_get_save_date, + index_mail_get_virtual_size, + pop3c_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + pop3c_mail_get_stream, + index_mail_get_binary_stream, + pop3c_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/pop3c/pop3c-settings.c b/src/lib-storage/index/pop3c/pop3c-settings.c new file mode 100644 index 0000000..db876e1 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-settings.c @@ -0,0 +1,116 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "pop3c-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct pop3c_settings) + +static const struct setting_define pop3c_setting_defines[] = { + DEF(STR, pop3c_host), + DEF(IN_PORT, pop3c_port), + + DEF(STR_VARS, pop3c_user), + DEF(STR_VARS, pop3c_master_user), + DEF(STR, pop3c_password), + + DEF(ENUM, pop3c_ssl), + DEF(BOOL, pop3c_ssl_verify), + + DEF(STR, pop3c_rawlog_dir), + DEF(BOOL, pop3c_quick_received_date), + + DEF(STR, pop3c_features), + + SETTING_DEFINE_LIST_END +}; + +static const struct pop3c_settings pop3c_default_settings = { + .pop3c_host = "", + .pop3c_port = 110, + + .pop3c_user = "%u", + .pop3c_master_user = "", + .pop3c_password = "", + + .pop3c_ssl = "no:pop3s:starttls", + .pop3c_ssl_verify = TRUE, + + .pop3c_rawlog_dir = "", + .pop3c_quick_received_date = FALSE, + + .pop3c_features = "" +}; + +/* <settings checks> */ +struct pop3c_feature_list { + const char *name; + enum pop3c_features num; +}; + +static const struct pop3c_feature_list pop3c_feature_list[] = { + { "no-pipelining", POP3C_FEATURE_NO_PIPELINING }, + { NULL, 0 } +}; + +static int +pop3c_settings_parse_features(struct pop3c_settings *set, + const char **error_r) +{ + enum pop3c_features features = 0; + const struct pop3c_feature_list *list; + const char *const *str; + + str = t_strsplit_spaces(set->pop3c_features, " ,"); + for (; *str != NULL; str++) { + list = pop3c_feature_list; + for (; list->name != NULL; list++) { + if (strcasecmp(*str, list->name) == 0) { + features |= list->num; + break; + } + } + if (list->name == NULL) { + *error_r = t_strdup_printf("pop3c_features: " + "Unknown feature: %s", *str); + return -1; + } + } + set->parsed_features = features; + return 0; +} + +static bool pop3c_settings_check(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) +{ + struct pop3c_settings *set = _set; + + if (pop3c_settings_parse_features(set, error_r) < 0) + return FALSE; + return TRUE; +} +/* </settings checks> */ + +static const struct setting_parser_info pop3c_setting_parser_info = { + .module_name = "pop3c", + .defines = pop3c_setting_defines, + .defaults = &pop3c_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct pop3c_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info, + + .check_func = pop3c_settings_check +}; + +const struct setting_parser_info *pop3c_get_setting_parser_info(void) +{ + return &pop3c_setting_parser_info; +} diff --git a/src/lib-storage/index/pop3c/pop3c-settings.h b/src/lib-storage/index/pop3c/pop3c-settings.h new file mode 100644 index 0000000..bf44e24 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-settings.h @@ -0,0 +1,33 @@ +#ifndef POP3C_SETTINGS_H +#define POP3C_SETTINGS_H + +#include "net.h" + +/* <settings checks> */ +enum pop3c_features { + POP3C_FEATURE_NO_PIPELINING = 0x1, +}; +/* </settings checks> */ + + +struct pop3c_settings { + const char *pop3c_host; + in_port_t pop3c_port; + + const char *pop3c_user; + const char *pop3c_master_user; + const char *pop3c_password; + + const char *pop3c_ssl; + bool pop3c_ssl_verify; + + const char *pop3c_rawlog_dir; + bool pop3c_quick_received_date; + + const char *pop3c_features; + enum pop3c_features parsed_features; +}; + +const struct setting_parser_info *pop3c_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/pop3c/pop3c-storage.c b/src/lib-storage/index/pop3c/pop3c-storage.c new file mode 100644 index 0000000..f784683 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-storage.c @@ -0,0 +1,368 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "mail-copy.h" +#include "mail-user.h" +#include "mailbox-list-private.h" +#include "index-mail.h" +#include "pop3c-client.h" +#include "pop3c-sync.h" +#include "pop3c-storage.h" + +#define DNS_CLIENT_SOCKET_NAME "dns-client" + +extern struct mail_storage pop3c_storage; +extern struct mailbox pop3c_mailbox; + +static struct event_category event_category_pop3c = { + .name = "pop3c", + .parent = &event_category_storage, +}; + +static struct mail_storage *pop3c_storage_alloc(void) +{ + struct pop3c_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("pop3c storage", 512+256); + storage = p_new(pool, struct pop3c_storage, 1); + storage->storage = pop3c_storage; + storage->storage.pool = pool; + return &storage->storage; +} + +static int +pop3c_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, + const char **error_r) +{ + struct pop3c_storage *storage = POP3C_STORAGE(_storage); + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + if (storage->set->pop3c_host[0] == '\0') { + *error_r = "missing pop3c_host"; + return -1; + } + if (storage->set->pop3c_password[0] == '\0') { + *error_r = "missing pop3c_password"; + return -1; + } + + return 0; +} + +static struct pop3c_client * +pop3c_client_create_from_set(struct mail_storage *storage, + const struct pop3c_settings *set) +{ + struct pop3c_client_settings client_set; + string_t *str; + + i_zero(&client_set); + client_set.host = set->pop3c_host; + client_set.port = set->pop3c_port; + client_set.username = set->pop3c_user; + client_set.master_user = set->pop3c_master_user; + client_set.password = set->pop3c_password; + client_set.dns_client_socket_path = + storage->user->set->base_dir[0] == '\0' ? "" : + t_strconcat(storage->user->set->base_dir, "/", + DNS_CLIENT_SOCKET_NAME, NULL); + str = t_str_new(128); + mail_user_set_get_temp_prefix(str, storage->user->set); + client_set.temp_path_prefix = str_c(str); + + client_set.debug = storage->user->mail_debug; + client_set.rawlog_dir = + mail_user_home_expand(storage->user, set->pop3c_rawlog_dir); + + mail_user_init_ssl_client_settings(storage->user, &client_set.ssl_set); + + if (!set->pop3c_ssl_verify) + client_set.ssl_set.allow_invalid_cert = TRUE; + + if (strcmp(set->pop3c_ssl, "pop3s") == 0) + client_set.ssl_mode = POP3C_CLIENT_SSL_MODE_IMMEDIATE; + else if (strcmp(set->pop3c_ssl, "starttls") == 0) + client_set.ssl_mode = POP3C_CLIENT_SSL_MODE_STARTTLS; + else + client_set.ssl_mode = POP3C_CLIENT_SSL_MODE_NONE; + return pop3c_client_init(&client_set, storage->event); +} + +static void +pop3c_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, + struct mailbox_list_settings *set) +{ + set->layout = MAILBOX_LIST_NAME_FS; + if (set->root_dir != NULL && *set->root_dir != '\0' && + set->index_dir == NULL) { + /* we don't really care about root_dir, but we + just need to get index_dir autocreated. */ + set->index_dir = set->root_dir; + } +} + +static struct mailbox * +pop3c_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct pop3c_mailbox *mbox; + pool_t pool; + + pool = pool_alloconly_create("pop3c mailbox", 1024*3); + mbox = p_new(pool, struct pop3c_mailbox, 1); + mbox->box = pop3c_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.list->props |= MAILBOX_LIST_PROP_AUTOCREATE_DIRS; + mbox->box.mail_vfuncs = &pop3c_mail_vfuncs; + mbox->storage = POP3C_STORAGE(storage); + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + return &mbox->box; +} + +static int +pop3c_mailbox_exists(struct mailbox *box, bool auto_boxes, + enum mailbox_existence *existence_r) +{ + if ((auto_boxes && mailbox_is_autocreated(box)) || box->inbox_any) + *existence_r = MAILBOX_EXISTENCE_SELECT; + else + *existence_r = MAILBOX_EXISTENCE_NONE; + return 0; +} + +static void pop3c_login_callback(enum pop3c_command_state state, + const char *reply, void *context) +{ + struct pop3c_mailbox *mbox = context; + + switch (state) { + case POP3C_COMMAND_STATE_OK: + mbox->logged_in = TRUE; + break; + case POP3C_COMMAND_STATE_ERR: + if (str_begins(reply, "[IN-USE] ")) { + mail_storage_set_error(mbox->box.storage, + MAIL_ERROR_INUSE, reply + 9); + } else { + /* authentication failure probably */ + mail_storage_set_error(mbox->box.storage, + MAIL_ERROR_PARAMS, reply); + } + break; + case POP3C_COMMAND_STATE_DISCONNECTED: + mailbox_set_critical(&mbox->box, + "pop3c: Disconnected from remote server"); + break; + } +} + +static int pop3c_mailbox_open(struct mailbox *box) +{ + struct pop3c_mailbox *mbox = POP3C_MAILBOX(box); + + if (strcmp(box->name, "INBOX") != 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } + + if (index_storage_mailbox_open(box, FALSE) < 0) + return -1; + + mbox->client = pop3c_client_create_from_set(box->storage, + mbox->storage->set); + pop3c_client_login(mbox->client, pop3c_login_callback, mbox); + pop3c_client_wait_one(mbox->client); + return mbox->logged_in ? 0 : -1; +} + +static void pop3c_mailbox_close(struct mailbox *box) +{ + struct pop3c_mailbox *mbox = POP3C_MAILBOX(box); + + pool_unref(&mbox->uidl_pool); + i_free_and_null(mbox->msg_uids); + i_free_and_null(mbox->msg_sizes); + pop3c_client_deinit(&mbox->client); + index_storage_mailbox_close(box); +} + +static int +pop3c_mailbox_create(struct mailbox *box, + const struct mailbox_update *update ATTR_UNUSED, + bool directory ATTR_UNUSED) +{ + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "POP3 mailbox creation isn't supported"); + return -1; +} + +static int +pop3c_mailbox_update(struct mailbox *box, + const struct mailbox_update *update ATTR_UNUSED) +{ + if (!guid_128_is_empty(update->mailbox_guid) || + update->uid_validity != 0 || update->min_next_uid != 0 || + update->min_first_recent_uid != 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "POP3 mailbox update isn't supported"); + } + return index_storage_mailbox_update(box, update); +} + +static int pop3c_mailbox_get_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + struct pop3c_mailbox *mbox = POP3C_MAILBOX(box); + + if (index_storage_get_status(box, items, status_r) < 0) + return -1; + + if ((pop3c_client_get_capabilities(mbox->client) & + POP3C_CAPABILITY_UIDL) == 0) + status_r->have_guids = FALSE; + return 0; +} + +static int pop3c_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + if ((items & MAILBOX_METADATA_GUID) != 0) { + /* a bit ugly way to do this, but better than nothing for now. + FIXME: if indexes are enabled, keep this there. */ + mail_generate_guid_128_hash(box->name, metadata_r->guid); + items &= ENUM_NEGATE(MAILBOX_METADATA_GUID); + } + if (items != 0) { + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + } + return 0; +} + +static void pop3c_notify_changes(struct mailbox *box ATTR_UNUSED) +{ +} + +static struct mail_save_context * +pop3c_save_alloc(struct mailbox_transaction_context *t) +{ + struct mail_save_context *ctx; + + ctx = i_new(struct mail_save_context, 1); + ctx->transaction = t; + return ctx; +} + +static int +pop3c_save_begin(struct mail_save_context *ctx, + struct istream *input ATTR_UNUSED) +{ + mail_storage_set_error(ctx->transaction->box->storage, + MAIL_ERROR_NOTPOSSIBLE, "POP3 doesn't support saving mails"); + return -1; +} + +static int pop3c_save_continue(struct mail_save_context *ctx ATTR_UNUSED) +{ + return -1; +} + +static int pop3c_save_finish(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); + return -1; +} + +static void +pop3c_save_cancel(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); +} + +static bool pop3c_storage_is_inconsistent(struct mailbox *box) +{ + struct pop3c_mailbox *mbox = POP3C_MAILBOX(box); + + return index_storage_is_inconsistent(box) || + !pop3c_client_is_connected(mbox->client); +} + +struct mail_storage pop3c_storage = { + .name = POP3C_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS, + .event_category = &event_category_pop3c, + + .v = { + pop3c_get_setting_parser_info, + pop3c_storage_alloc, + pop3c_storage_create, + index_storage_destroy, + NULL, + pop3c_storage_get_list_settings, + NULL, + pop3c_mailbox_alloc, + NULL, + NULL, + } +}; + +struct mailbox pop3c_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + pop3c_mailbox_exists, + pop3c_mailbox_open, + pop3c_mailbox_close, + index_storage_mailbox_free, + pop3c_mailbox_create, + pop3c_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + pop3c_mailbox_get_status, + pop3c_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + pop3c_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + pop3c_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + pop3c_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + pop3c_save_alloc, + pop3c_save_begin, + pop3c_save_continue, + pop3c_save_finish, + pop3c_save_cancel, + mail_storage_copy, + NULL, + NULL, + NULL, + pop3c_storage_is_inconsistent + } +}; diff --git a/src/lib-storage/index/pop3c/pop3c-storage.h b/src/lib-storage/index/pop3c/pop3c-storage.h new file mode 100644 index 0000000..f271a59 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-storage.h @@ -0,0 +1,51 @@ +#ifndef POP3C_STORAGE_H +#define POP3C_STORAGE_H + +#include "index-storage.h" + +#define POP3C_STORAGE_NAME "pop3c" + +struct pop3c_storage { + struct mail_storage storage; + const struct pop3c_settings *set; +}; + +struct pop3c_mailbox { + struct mailbox box; + struct pop3c_storage *storage; + + struct pop3c_client *client; + + pool_t uidl_pool; + unsigned int msg_count; + /* LIST sizes */ + uoff_t *msg_sizes; + /* UIDL strings */ + const char *const *msg_uidls; + /* index UIDs for each message in this session. + the UID may not exist for the entire session */ + uint32_t *msg_uids; + + bool logged_in:1; +}; + +struct pop3c_mail { + struct index_mail imail; + struct istream *prefetch_stream; + + bool prefetching:1; + bool prefetching_body:1; +}; + +#define POP3C_STORAGE(s) container_of(s, struct pop3c_storage, storage) +#define POP3C_MAILBOX(s) container_of(s, struct pop3c_mailbox, box) +#define POP3C_MAIL(s) container_of(s, struct pop3c_mail, imail.mail.mail) + +struct mail * +pop3c_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); + +extern struct mail_vfuncs pop3c_mail_vfuncs; + +#endif diff --git a/src/lib-storage/index/pop3c/pop3c-sync.c b/src/lib-storage/index/pop3c/pop3c-sync.c new file mode 100644 index 0000000..2d2dbe3 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-sync.c @@ -0,0 +1,361 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "bsearch-insert-pos.h" +#include "str.h" +#include "sort.h" +#include "strnum.h" +#include "index-mail.h" +#include "pop3c-client.h" +#include "pop3c-storage.h" +#include "pop3c-sync.h" +#include "mailbox-recent-flags.h" + +struct pop3c_sync_msg { + uint32_t seq; + const char *uidl; +}; +ARRAY_DEFINE_TYPE(pop3c_sync_msg, struct pop3c_sync_msg); + +int pop3c_sync_get_uidls(struct pop3c_mailbox *mbox) +{ + ARRAY_TYPE(const_string) uidls; + struct istream *input; + const char *error, *cline; + char *line, *p; + unsigned int seq, line_seq; + + if (mbox->msg_uidls != NULL) + return 0; + if ((pop3c_client_get_capabilities(mbox->client) & + POP3C_CAPABILITY_UIDL) == 0) { + mail_storage_set_error(mbox->box.storage, + MAIL_ERROR_NOTPOSSIBLE, + "UIDLs not supported by server"); + return -1; + } + + if (pop3c_client_cmd_stream(mbox->client, "UIDL\r\n", + &input, &error) < 0) { + mailbox_set_critical(&mbox->box, "UIDL failed: %s", error); + return -1; + } + + mbox->uidl_pool = pool_alloconly_create("POP3 UIDLs", 1024*32); + p_array_init(&uidls, mbox->uidl_pool, 64); seq = 0; + while ((line = i_stream_read_next_line(input)) != NULL) { + seq++; + p = strchr(line, ' '); + if (p == NULL) { + mailbox_set_critical(&mbox->box, + "Invalid UIDL line: %s", line); + break; + } + *p++ = '\0'; + if (str_to_uint(line, &line_seq) < 0 || line_seq != seq) { + mailbox_set_critical(&mbox->box, + "Unexpected UIDL seq: %s != %u", line, seq); + break; + } + + cline = p_strdup(mbox->uidl_pool, p); + array_push_back(&uidls, &cline); + } + i_stream_destroy(&input); + if (line != NULL) { + pool_unref(&mbox->uidl_pool); + return -1; + } + if (seq == 0) { + /* make msg_uidls non-NULL */ + array_append_zero(&uidls); + } + mbox->msg_uidls = array_front(&uidls); + mbox->msg_count = seq; + return 0; +} + +int pop3c_sync_get_sizes(struct pop3c_mailbox *mbox) +{ + struct istream *input; + const char *error; + char *line, *p; + unsigned int seq, line_seq; + + i_assert(mbox->msg_sizes == NULL); + + if (mbox->msg_uidls == NULL) { + if (pop3c_sync_get_uidls(mbox) < 0) + return -1; + } + if (mbox->msg_count == 0) { + mbox->msg_sizes = i_new(uoff_t, 1); + return 0; + } + + if (pop3c_client_cmd_stream(mbox->client, "LIST\r\n", + &input, &error) < 0) { + mailbox_set_critical(&mbox->box, "LIST failed: %s", error); + return -1; + } + + mbox->msg_sizes = i_new(uoff_t, mbox->msg_count); seq = 0; + while ((line = i_stream_read_next_line(input)) != NULL) { + if (++seq > mbox->msg_count) { + mailbox_set_critical(&mbox->box, + "Too much data in LIST: %s", line); + break; + } + p = strchr(line, ' '); + if (p == NULL) { + mailbox_set_critical(&mbox->box, + "Invalid LIST line: %s", line); + break; + } + *p++ = '\0'; + if (str_to_uint(line, &line_seq) < 0 || line_seq != seq) { + mailbox_set_critical(&mbox->box, + "Unexpected LIST seq: %s != %u", line, seq); + break; + } + if (str_to_uoff(p, &mbox->msg_sizes[seq-1]) < 0) { + mailbox_set_critical(&mbox->box, + "Invalid LIST size: %s", p); + break; + } + } + i_stream_destroy(&input); + if (line != NULL) { + i_free_and_null(mbox->msg_sizes); + return -1; + } + return 0; +} + +static void +pop3c_get_local_msgs(pool_t pool, ARRAY_TYPE(pop3c_sync_msg) *local_msgs, + uint32_t messages_count, + struct mail_cache_view *cache_view, + unsigned int cache_idx) +{ + string_t *str = t_str_new(128); + struct pop3c_sync_msg msg; + uint32_t seq; + + i_zero(&msg); + for (seq = 1; seq <= messages_count; seq++) { + str_truncate(str, 0); + if (mail_cache_lookup_field(cache_view, str, seq, + cache_idx) > 0) + msg.uidl = p_strdup(pool, str_c(str)); + msg.seq = seq; + array_idx_set(local_msgs, seq-1, &msg); + } +} + +static void +pop3c_get_remote_msgs(ARRAY_TYPE(pop3c_sync_msg) *remote_msgs, + struct pop3c_mailbox *mbox) +{ + struct pop3c_sync_msg *msg; + uint32_t seq; + + for (seq = 1; seq <= mbox->msg_count; seq++) { + msg = array_append_space(remote_msgs); + msg->seq = seq; + msg->uidl = mbox->msg_uidls[seq-1]; + } +} + +static int pop3c_sync_msg_uidl_cmp(const struct pop3c_sync_msg *msg1, + const struct pop3c_sync_msg *msg2) +{ + return null_strcmp(msg1->uidl, msg2->uidl); +} + +static void +pop3c_sync_messages(struct pop3c_mailbox *mbox, + struct mail_index_view *sync_view, + struct mail_index_transaction *sync_trans, + struct mail_cache_view *cache_view) +{ + struct index_mailbox_context *ibox = + INDEX_STORAGE_CONTEXT(&mbox->box); + const struct mail_index_header *hdr; + struct mail_cache_transaction_ctx *cache_trans; + ARRAY_TYPE(pop3c_sync_msg) local_msgs, remote_msgs; + const struct pop3c_sync_msg *lmsg, *rmsg; + uint32_t seq1, seq2, next_uid; + unsigned int lidx, ridx, lcount, rcount; + unsigned int cache_idx = ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx; + pool_t pool; + + i_assert(mbox->msg_uids == NULL); + + /* set our uidvalidity */ + hdr = mail_index_get_header(sync_view); + if (hdr->uid_validity == 0) { + uint32_t uid_validity = ioloop_time; + mail_index_update_header(sync_trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + pool = pool_alloconly_create(MEMPOOL_GROWING"pop3c sync", 10240); + p_array_init(&local_msgs, pool, hdr->messages_count); + pop3c_get_local_msgs(pool, &local_msgs, hdr->messages_count, + cache_view, cache_idx); + p_array_init(&remote_msgs, pool, mbox->msg_count); + pop3c_get_remote_msgs(&remote_msgs, mbox); + + /* sort the messages by UIDLs, because some servers reorder messages */ + array_sort(&local_msgs, pop3c_sync_msg_uidl_cmp); + array_sort(&remote_msgs, pop3c_sync_msg_uidl_cmp); + + /* skip over existing messages with matching UIDLs and expunge the ones + that no longer exist in remote. */ + mbox->msg_uids = mbox->msg_count == 0 ? + i_new(uint32_t, 1) : /* avoid malloc(0) assert */ + i_new(uint32_t, mbox->msg_count); + cache_trans = mail_cache_get_transaction(cache_view, sync_trans); + + lmsg = array_get(&local_msgs, &lcount); + rmsg = array_get(&remote_msgs, &rcount); + next_uid = hdr->next_uid; + lidx = ridx = 0; + while (lidx < lcount || ridx < rcount) { + uint32_t lseq = lidx < lcount ? lmsg[lidx].seq : 0; + uint32_t rseq = ridx < rcount ? rmsg[ridx].seq : 0; + int ret; + + if (lidx >= lcount) + ret = 1; + else if (ridx >= rcount || lmsg[lidx].uidl == NULL) + ret = -1; + else + ret = strcmp(lmsg[lidx].uidl, rmsg[ridx].uidl); + if (ret < 0) { + /* message expunged in remote, or we didn't have a + local message's UIDL in cache. */ + mail_index_expunge(sync_trans, lseq); + lidx++; + } else if (ret > 0) { + /* new message in remote */ + i_assert(mbox->msg_uids[rseq-1] == 0); + mbox->msg_uids[rseq-1] = next_uid; + mail_index_append(sync_trans, next_uid++, &lseq); + mail_cache_add(cache_trans, lseq, cache_idx, + rmsg[ridx].uidl, + strlen(rmsg[ridx].uidl)); + ridx++; + } else { + /* UIDL matched */ + i_assert(mbox->msg_uids[rseq-1] == 0); + mail_index_lookup_uid(sync_view, lseq, + &mbox->msg_uids[rseq-1]); + lidx++; + ridx++; + } + } + + /* mark the newly seen messages as recent */ + if (mail_index_lookup_seq_range(sync_view, hdr->first_recent_uid, + hdr->next_uid, &seq1, &seq2)) + mailbox_recent_flags_set_seqs(&mbox->box, sync_view, seq1, seq2); + pool_unref(&pool); +} + +int pop3c_sync(struct pop3c_mailbox *mbox) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view, *trans_view; + struct mail_index_transaction *sync_trans; + struct mail_index_sync_rec sync_rec; + struct mail_cache_view *cache_view = NULL; + enum mail_index_sync_flags sync_flags; + unsigned int idx; + string_t *str; + const char *reply; + int ret; + bool deletions = FALSE; + + if (pop3c_sync_get_uidls(mbox) < 0) + return -1; + + sync_flags = index_storage_get_sync_flags(&mbox->box) | + MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY; + + ret = mail_index_sync_begin(mbox->box.index, &index_sync_ctx, + &sync_view, &sync_trans, sync_flags); + if (ret <= 0) { + if (ret < 0) + mailbox_set_index_error(&mbox->box); + return ret; + } + + if (mbox->msg_uids == NULL) { + trans_view = mail_index_transaction_open_updated_view(sync_trans); + cache_view = mail_cache_view_open(mbox->box.cache, trans_view); + pop3c_sync_messages(mbox, sync_view, sync_trans, cache_view); + } + + /* mark expunges messages as deleted in this pop3 session, + if those exist */ + str = t_str_new(32); + while (mail_index_sync_next(index_sync_ctx, &sync_rec)) { + if (sync_rec.type != MAIL_INDEX_SYNC_TYPE_EXPUNGE) + continue; + + if (!bsearch_insert_pos(&sync_rec.uid1, mbox->msg_uids, + mbox->msg_count, sizeof(uint32_t), + uint32_cmp, &idx)) { + /* no such messages in this session */ + continue; + } + for (; idx < mbox->msg_count; idx++) { + i_assert(mbox->msg_uids[idx] >= sync_rec.uid1); + if (mbox->msg_uids[idx] > sync_rec.uid2) + break; + + str_truncate(str, 0); + str_printfa(str, "DELE %u\r\n", idx+1); + pop3c_client_cmd_line_async_nocb(mbox->client, str_c(str)); + deletions = TRUE; + } + } + + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + if (cache_view != NULL) { + mail_cache_view_close(&cache_view); + mail_index_view_close(&trans_view); + } + if (deletions) { + if (pop3c_client_cmd_line(mbox->client, "QUIT\r\n", + &reply) < 0) { + mail_storage_set_error(mbox->box.storage, + MAIL_ERROR_TEMP, reply); + return -1; + } + } + return 0; +} + +struct mailbox_sync_context * +pop3c_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct pop3c_mailbox *mbox = POP3C_MAILBOX(box); + int ret = 0; + + if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 && + mbox->msg_uidls == NULL) { + /* FIXME: reconnect */ + } + + ret = pop3c_sync(mbox); + return index_mailbox_sync_init(box, flags, ret < 0); +} diff --git a/src/lib-storage/index/pop3c/pop3c-sync.h b/src/lib-storage/index/pop3c/pop3c-sync.h new file mode 100644 index 0000000..bf3c802 --- /dev/null +++ b/src/lib-storage/index/pop3c/pop3c-sync.h @@ -0,0 +1,14 @@ +#ifndef POP3C_SYNC_H +#define POP3C_SYNC_H + +struct mailbox; +struct pop3c_mailbox; + +struct mailbox_sync_context * +pop3c_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); +int pop3c_sync(struct pop3c_mailbox *mbox); + +int pop3c_sync_get_sizes(struct pop3c_mailbox *mbox); +int pop3c_sync_get_uidls(struct pop3c_mailbox *mbox); + +#endif diff --git a/src/lib-storage/index/raw/Makefile.am b/src/lib-storage/index/raw/Makefile.am new file mode 100644 index 0000000..58a9df3 --- /dev/null +++ b/src/lib-storage/index/raw/Makefile.am @@ -0,0 +1,21 @@ +noinst_LTLIBRARIES = libstorage_raw.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_raw_la_SOURCES = \ + raw-mail.c \ + raw-sync.c \ + raw-storage.c + +headers = \ + raw-storage.h \ + raw-sync.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/raw/Makefile.in b/src/lib-storage/index/raw/Makefile.in new file mode 100644 index 0000000..a425fce --- /dev/null +++ b/src/lib-storage/index/raw/Makefile.in @@ -0,0 +1,822 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/raw +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_raw_la_LIBADD = +am_libstorage_raw_la_OBJECTS = raw-mail.lo raw-sync.lo raw-storage.lo +libstorage_raw_la_OBJECTS = $(am_libstorage_raw_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/raw-mail.Plo \ + ./$(DEPDIR)/raw-storage.Plo ./$(DEPDIR)/raw-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_raw_la_SOURCES) +DIST_SOURCES = $(libstorage_raw_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_raw.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_raw_la_SOURCES = \ + raw-mail.c \ + raw-sync.c \ + raw-storage.c + +headers = \ + raw-storage.h \ + raw-sync.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/raw/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/raw/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_raw.la: $(libstorage_raw_la_OBJECTS) $(libstorage_raw_la_DEPENDENCIES) $(EXTRA_libstorage_raw_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_raw_la_OBJECTS) $(libstorage_raw_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/raw-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/raw-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/raw-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/raw-mail.Plo + -rm -f ./$(DEPDIR)/raw-storage.Plo + -rm -f ./$(DEPDIR)/raw-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/raw-mail.Plo + -rm -f ./$(DEPDIR)/raw-storage.Plo + -rm -f ./$(DEPDIR)/raw-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/raw/raw-mail.c b/src/lib-storage/index/raw/raw-mail.c new file mode 100644 index 0000000..72d3688 --- /dev/null +++ b/src/lib-storage/index/raw/raw-mail.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "index-mail.h" +#include "raw-storage.h" + +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> + +static int raw_mail_stat(struct mail *mail) +{ + struct raw_mailbox *mbox = RAW_MAILBOX(mail->box); + const struct stat *st; + + if (!mail_metadata_access_start(mail)) + return -1; + + mail->transaction->stats.fstat_lookup_count++; + if (i_stream_stat(mail->box->input, TRUE, &st) < 0) { + mail_set_critical(mail, "stat(%s) failed: %m", + i_stream_get_name(mail->box->input)); + return -1; + } + + if (mbox->mtime != (time_t)-1) + mbox->mtime = st->st_mtime; + if (mbox->ctime != (time_t)-1) + mbox->ctime = st->st_ctime; + mbox->size = (size_t)st->st_size; + return 0; +} + +static int raw_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box); + + if (mbox->mtime == (time_t)-1) { + if (raw_mail_stat(_mail) < 0) + return -1; + } + + *date_r = mail->data.received_date = mbox->mtime; + return 0; +} + +static int raw_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box); + + if (mbox->ctime == (time_t)-1) { + if (raw_mail_stat(_mail) < 0) + return -1; + } + + *date_r = mail->data.save_date = mbox->ctime; + return 1; +} + +static int raw_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box); + + if (mbox->size == UOFF_T_MAX) { + if (raw_mail_stat(_mail) < 0) + return -1; + } + + *size_r = mail->data.physical_size = mbox->size; + return 0; +} + +static int +raw_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, + struct message_size *hdr_size, + struct message_size *body_size, struct istream **stream_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (mail->data.stream == NULL) { + if (!mail_stream_access_start(_mail)) + return -1; + /* we can't just reference mbox->input, because + index_mail_close() expects to be able to free the stream */ + mail->data.stream = + i_stream_create_limit(_mail->box->input, UOFF_T_MAX); + } + + return index_mail_init_stream(mail, hdr_size, body_size, stream_r); +} + +static int +raw_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box); + + switch (field) { + case MAIL_FETCH_FROM_ENVELOPE: + *value_r = mbox->envelope_sender != NULL ? + mbox->envelope_sender : ""; + return 0; + case MAIL_FETCH_STORAGE_ID: + *value_r = mbox->have_filename ? + mailbox_get_path(_mail->box) : ""; + return 0; + default: + return index_mail_get_special(_mail, field, value_r); + } +} + +struct mail_vfuncs raw_mail_vfuncs = { + index_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + raw_mail_get_received_date, + raw_mail_get_save_date, + index_mail_get_virtual_size, + raw_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + raw_mail_get_stream, + index_mail_get_binary_stream, + raw_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/raw/raw-storage.c b/src/lib-storage/index/raw/raw-storage.c new file mode 100644 index 0000000..e94ae10 --- /dev/null +++ b/src/lib-storage/index/raw/raw-storage.c @@ -0,0 +1,269 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "index-mail.h" +#include "mail-copy.h" +#include "mailbox-list-private.h" +#include "raw-sync.h" +#include "raw-storage.h" + +extern struct mail_storage raw_storage; +extern struct mailbox raw_mailbox; + +struct mail_user * +raw_storage_create_from_set(const struct setting_parser_info *set_info, + const struct mail_user_settings *set) +{ + struct mail_user *user; + struct mail_namespace *ns; + struct mail_namespace_settings *ns_set; + struct mail_storage_settings *mail_set; + const char *error; + + user = mail_user_alloc(NULL, "raw mail user", set_info, set); + user->autocreated = TRUE; + mail_user_set_home(user, "/"); + if (mail_user_init(user, &error) < 0) + i_fatal("Raw user initialization failed: %s", error); + + ns_set = p_new(user->pool, struct mail_namespace_settings, 1); + ns_set->name = "raw-storage"; + ns_set->location = ":LAYOUT=none"; + ns_set->separator = "/"; + + ns = mail_namespaces_init_empty(user); + /* raw storage doesn't have INBOX. We especially don't want LIST to + return INBOX. */ + ns->flags &= ENUM_NEGATE(NAMESPACE_FLAG_INBOX_USER); + ns->flags |= NAMESPACE_FLAG_NOQUOTA | NAMESPACE_FLAG_NOACL; + ns->set = ns_set; + /* absolute paths are ok with raw storage */ + mail_set = p_new(user->pool, struct mail_storage_settings, 1); + *mail_set = *ns->mail_set; + mail_set->mail_full_filesystem_access = TRUE; + ns->mail_set = mail_set; + + if (mail_storage_create(ns, "raw", 0, &error) < 0) + i_fatal("Couldn't create internal raw storage: %s", error); + if (mail_namespaces_init_finish(ns, &error) < 0) + i_fatal("Couldn't create internal raw namespace: %s", error); + return user; +} + +static int ATTR_NULL(2, 3) +raw_mailbox_alloc_common(struct mail_user *user, struct istream *input, + const char *path, time_t received_time, + const char *envelope_sender, struct mailbox **box_r) +{ + struct mail_namespace *ns = user->namespaces; + struct mailbox *box; + struct raw_mailbox *raw_box; + const char *name; + + name = path != NULL ? path : i_stream_get_name(input); + box = *box_r = mailbox_alloc(ns->list, name, + MAILBOX_FLAG_NO_INDEX_FILES); + if (input != NULL) { + if (mailbox_open_stream(box, input) < 0) + return -1; + } else { + if (mailbox_open(box) < 0) + return -1; + } + if (mailbox_sync(box, 0) < 0) + return -1; + + i_assert(strcmp(box->storage->name, RAW_STORAGE_NAME) == 0); + raw_box = RAW_MAILBOX(box); + raw_box->envelope_sender = p_strdup(box->pool, envelope_sender); + raw_box->mtime = received_time; + return 0; +} + +int raw_mailbox_alloc_stream(struct mail_user *user, struct istream *input, + time_t received_time, const char *envelope_sender, + struct mailbox **box_r) +{ + return raw_mailbox_alloc_common(user, input, NULL, received_time, + envelope_sender, box_r); +} + +int raw_mailbox_alloc_path(struct mail_user *user, const char *path, + time_t received_time, const char *envelope_sender, + struct mailbox **box_r) +{ + return raw_mailbox_alloc_common(user, NULL, path, received_time, + envelope_sender, box_r); +} + +static struct mail_storage *raw_storage_alloc(void) +{ + struct raw_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("raw storage", 512+256); + storage = p_new(pool, struct raw_storage, 1); + storage->storage = raw_storage; + storage->storage.pool = pool; + return &storage->storage; +} + +static void +raw_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_FS; + if (set->subscription_fname == NULL) + set->subscription_fname = RAW_SUBSCRIPTION_FILE_NAME; +} + +static struct mailbox * +raw_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct raw_mailbox *mbox; + pool_t pool; + + flags |= MAILBOX_FLAG_READONLY | MAILBOX_FLAG_NO_INDEX_FILES; + + pool = pool_alloconly_create("raw mailbox", 1024*3); + mbox = p_new(pool, struct raw_mailbox, 1); + mbox->box = raw_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &raw_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, "dovecot.index"); + + mbox->mtime = mbox->ctime = (time_t)-1; + mbox->storage = RAW_STORAGE(storage); + mbox->size = UOFF_T_MAX; + return &mbox->box; +} + +static int raw_mailbox_open(struct mailbox *box) +{ + struct raw_mailbox *mbox = RAW_MAILBOX(box); + const char *path; + int fd; + + if (box->input != NULL) { + mbox->mtime = mbox->ctime = ioloop_time; + return index_storage_mailbox_open(box, FALSE); + } + + path = box->_path = box->name; + mbox->have_filename = TRUE; + fd = open(path, O_RDONLY); + if (fd == -1) { + if (ENOTFOUND(errno)) { + mail_storage_set_error(box->storage, + MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + } else if (!mail_storage_set_error_from_errno(box->storage)) { + mailbox_set_critical(box, "open(%s) failed: %m", path); + } + return -1; + } + box->input = i_stream_create_fd_autoclose(&fd, MAIL_READ_FULL_BLOCK_SIZE); + i_stream_set_name(box->input, path); + i_stream_set_init_buffer_size(box->input, MAIL_READ_FULL_BLOCK_SIZE); + return index_storage_mailbox_open(box, FALSE); +} + +static int +raw_mailbox_create(struct mailbox *box, + const struct mailbox_update *update ATTR_UNUSED, + bool directory ATTR_UNUSED) +{ + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "Raw mailbox creation isn't supported"); + return -1; +} + +static int +raw_mailbox_update(struct mailbox *box, + const struct mailbox_update *update ATTR_UNUSED) +{ + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "Raw mailbox update isn't supported"); + return -1; +} + +static void raw_notify_changes(struct mailbox *box ATTR_UNUSED) +{ +} + +struct mail_storage raw_storage = { + .name = RAW_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_MAILBOX_IS_FILE | + MAIL_STORAGE_CLASS_FLAG_OPEN_STREAMS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + + .v = { + NULL, + raw_storage_alloc, + NULL, + index_storage_destroy, + NULL, + raw_storage_get_list_settings, + NULL, + raw_mailbox_alloc, + NULL, + NULL, + } +}; + +struct mailbox raw_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + raw_mailbox_open, + index_storage_mailbox_close, + index_storage_mailbox_free, + raw_mailbox_create, + raw_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + index_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + raw_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + raw_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + index_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + NULL, + NULL, + NULL, + NULL, + NULL, + mail_storage_copy, + NULL, + NULL, + NULL, + index_storage_is_inconsistent + } +}; diff --git a/src/lib-storage/index/raw/raw-storage.h b/src/lib-storage/index/raw/raw-storage.h new file mode 100644 index 0000000..124ca7e --- /dev/null +++ b/src/lib-storage/index/raw/raw-storage.h @@ -0,0 +1,41 @@ +#ifndef RAW_STORAGE_H +#define RAW_STORAGE_H + +#include "index-storage.h" + +#define RAW_STORAGE_NAME "raw" +#define RAW_SUBSCRIPTION_FILE_NAME "subscriptions" + +struct raw_storage { + struct mail_storage storage; +}; + +struct raw_mailbox { + struct mailbox box; + struct raw_storage *storage; + + time_t mtime, ctime; + uoff_t size; + const char *envelope_sender; + + bool synced:1; + bool have_filename:1; +}; + +#define RAW_STORAGE(s) container_of(s, struct raw_storage, storage) +#define RAW_MAILBOX(s) container_of(s, struct raw_mailbox, box) + +extern struct mail_vfuncs raw_mail_vfuncs; + +struct mail_user * +raw_storage_create_from_set(const struct setting_parser_info *set_info, + const struct mail_user_settings *set); + +int raw_mailbox_alloc_stream(struct mail_user *user, struct istream *input, + time_t received_time, const char *envelope_sender, + struct mailbox **box_r); +int raw_mailbox_alloc_path(struct mail_user *user, const char *path, + time_t received_time, const char *envelope_sender, + struct mailbox **box_r); + +#endif diff --git a/src/lib-storage/index/raw/raw-sync.c b/src/lib-storage/index/raw/raw-sync.c new file mode 100644 index 0000000..6511f72 --- /dev/null +++ b/src/lib-storage/index/raw/raw-sync.c @@ -0,0 +1,67 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "raw-storage.h" +#include "raw-sync.h" +#include "mailbox-recent-flags.h" + +static int raw_sync(struct raw_mailbox *mbox) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_sync_rec sync_rec; + struct mail_index_transaction *trans; + uint32_t seq, uid_validity = ioloop_time; + enum mail_index_sync_flags sync_flags; + int ret; + + i_assert(!mbox->synced); + + sync_flags = index_storage_get_sync_flags(&mbox->box) | + MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY | + MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + + if (mail_index_view_get_messages_count(mbox->box.view) > 0) { + /* already-synced index was opened via + mail-index-alloc-cache. */ + return 0; + } + + ret = mail_index_sync_begin(mbox->box.index, &index_sync_ctx, + &sync_view, &trans, sync_flags); + if (ret <= 0) { + if (ret < 0) + mailbox_set_index_error(&mbox->box); + return ret; + } + + /* set our uidvalidity */ + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + + /* add our one and only message */ + mail_index_append(trans, 1, &seq); + mailbox_recent_flags_set_uid(&mbox->box, 1); + + while (mail_index_sync_next(index_sync_ctx, &sync_rec)) ; + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + mbox->synced = TRUE; + return 0; +} + +struct mailbox_sync_context * +raw_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct raw_mailbox *mbox = RAW_MAILBOX(box); + int ret = 0; + + if (!mbox->synced) + ret = raw_sync(mbox); + + return index_mailbox_sync_init(box, flags, ret < 0); +} diff --git a/src/lib-storage/index/raw/raw-sync.h b/src/lib-storage/index/raw/raw-sync.h new file mode 100644 index 0000000..2c29c1f --- /dev/null +++ b/src/lib-storage/index/raw/raw-sync.h @@ -0,0 +1,9 @@ +#ifndef RAW_SYNC_H +#define RAW_SYNC_H + +struct mailbox; + +struct mailbox_sync_context * +raw_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +#endif diff --git a/src/lib-storage/index/shared/Makefile.am b/src/lib-storage/index/shared/Makefile.am new file mode 100644 index 0000000..07980d6 --- /dev/null +++ b/src/lib-storage/index/shared/Makefile.am @@ -0,0 +1,19 @@ +noinst_LTLIBRARIES = libstorage_shared.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_shared_la_SOURCES = \ + shared-list.c \ + shared-storage.c + +headers = \ + shared-storage.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/shared/Makefile.in b/src/lib-storage/index/shared/Makefile.in new file mode 100644 index 0000000..b1aeaa5 --- /dev/null +++ b/src/lib-storage/index/shared/Makefile.in @@ -0,0 +1,817 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/shared +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_shared_la_LIBADD = +am_libstorage_shared_la_OBJECTS = shared-list.lo shared-storage.lo +libstorage_shared_la_OBJECTS = $(am_libstorage_shared_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/shared-list.Plo \ + ./$(DEPDIR)/shared-storage.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_shared_la_SOURCES) +DIST_SOURCES = $(libstorage_shared_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_shared.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_shared_la_SOURCES = \ + shared-list.c \ + shared-storage.c + +headers = \ + shared-storage.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/shared/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/shared/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_shared.la: $(libstorage_shared_la_OBJECTS) $(libstorage_shared_la_DEPENDENCIES) $(EXTRA_libstorage_shared_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_shared_la_OBJECTS) $(libstorage_shared_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/shared-list.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/shared-storage.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/shared-list.Plo + -rm -f ./$(DEPDIR)/shared-storage.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/shared-list.Plo + -rm -f ./$(DEPDIR)/shared-storage.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/shared/shared-list.c b/src/lib-storage/index/shared/shared-list.c new file mode 100644 index 0000000..c69c4db --- /dev/null +++ b/src/lib-storage/index/shared/shared-list.c @@ -0,0 +1,310 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "imap-match.h" +#include "mailbox-tree.h" +#include "mailbox-list-private.h" +#include "index-storage.h" +#include "shared-storage.h" + +extern struct mailbox_list shared_mailbox_list; + +static struct mailbox_list *shared_list_alloc(void) +{ + struct mailbox_list *list; + pool_t pool; + + pool = pool_alloconly_create("shared list", 2048); + list = p_new(pool, struct mailbox_list, 1); + *list = shared_mailbox_list; + list->pool = pool; + return list; +} + +static void shared_list_deinit(struct mailbox_list *list) +{ + pool_unref(&list->pool); +} + +static void shared_list_copy_error(struct mailbox_list *shared_list, + struct mail_namespace *backend_ns) +{ + const char *str; + enum mail_error error; + + str = mailbox_list_get_last_error(backend_ns->list, &error); + mailbox_list_set_error(shared_list, error, str); +} + +static int +shared_get_storage(struct mailbox_list **list, const char *vname, + struct mail_storage **storage_r) +{ + struct mail_namespace *ns = (*list)->ns; + const char *name; + + name = mailbox_list_get_storage_name(*list, vname); + if (*name == '\0' && (ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) { + /* trying to access the shared/ prefix itself */ + *storage_r = ns->storage; + return 0; + } + + if (shared_storage_get_namespace(&ns, &name) < 0) + return -1; + *list = ns->list; + return mailbox_list_get_storage(list, vname, storage_r); +} + +static char shared_list_get_hierarchy_sep(struct mailbox_list *list ATTR_UNUSED) +{ + return '/'; +} + +static int +shared_list_get_path(struct mailbox_list *list, const char *name, + enum mailbox_list_path_type type, const char **path_r) +{ + struct mail_namespace *ns = list->ns; + + if (mail_namespace_get_default_storage(list->ns) == NULL || + name == NULL || + shared_storage_get_namespace(&ns, &name) < 0) { + /* we don't have a directory we can use. */ + *path_r = NULL; + return 0; + } + return mailbox_list_get_path(ns->list, name, type, path_r); +} + +static const char * +shared_list_get_temp_prefix(struct mailbox_list *list, bool global ATTR_UNUSED) +{ + i_panic("shared mailbox list: Can't return a temp prefix for '%s'", + list->ns->prefix); +} + +static const char * +shared_list_join_refpattern(struct mailbox_list *list, + const char *ref, const char *pattern) +{ + struct mail_namespace *ns = list->ns; + const char *ns_ref, *prefix = list->ns->prefix; + size_t prefix_len = strlen(prefix); + + if (*ref != '\0' && str_begins(ref, prefix)) + ns_ref = ref + prefix_len; + else + ns_ref = NULL; + + if (ns_ref != NULL && *ns_ref != '\0' && + shared_storage_get_namespace(&ns, &ns_ref) == 0) + return mailbox_list_join_refpattern(ns->list, ref, pattern); + + /* fallback to default behavior */ + if (*ref != '\0') + pattern = t_strconcat(ref, pattern, NULL); + return pattern; +} + +static void +shared_list_create_missing_namespaces(struct mailbox_list *list, + const char *const *patterns) +{ + struct mail_namespace *ns; + char sep = mail_namespace_get_sep(list->ns); + const char *list_pat, *name; + unsigned int i; + + for (i = 0; patterns[i] != NULL; i++) { + const char *last = NULL, *p; + + /* we'll require that the pattern begins with the list's + namespace prefix. we could also handle other patterns + (e.g. %/user/%), but it's more of a theoretical problem. */ + if (strncmp(list->ns->prefix, patterns[i], + list->ns->prefix_len) != 0) + continue; + list_pat = patterns[i] + list->ns->prefix_len; + + for (p = list_pat; *p != '\0'; p++) { + if (*p == '%' || *p == '*') + break; + if (*p == sep) + last = p; + } + if (last != NULL) { + ns = list->ns; + name = t_strdup_until(list_pat, last); + (void)shared_storage_get_namespace(&ns, &name); + } + } +} + +static struct mailbox_list_iterate_context * +shared_list_iter_init(struct mailbox_list *list, const char *const *patterns, + enum mailbox_list_iter_flags flags) +{ + struct mailbox_list_iterate_context *ctx; + pool_t pool; + char sep = mail_namespace_get_sep(list->ns); + + pool = pool_alloconly_create("mailbox list shared iter", 1024); + ctx = p_new(pool, struct mailbox_list_iterate_context, 1); + ctx->pool = pool; + ctx->list = list; + ctx->flags = flags; + ctx->glob = imap_match_init_multiple(pool, patterns, FALSE, sep); + array_create(&ctx->module_contexts, pool, sizeof(void *), 5); + + if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0 && + (list->ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) T_BEGIN { + shared_list_create_missing_namespaces(list, patterns); + } T_END; + return ctx; +} + +static const struct mailbox_info * +shared_list_iter_next(struct mailbox_list_iterate_context *ctx ATTR_UNUSED) +{ + return NULL; +} + +static int shared_list_iter_deinit(struct mailbox_list_iterate_context *ctx) +{ + pool_unref(&ctx->pool); + return 0; +} + +static int +shared_list_subscriptions_refresh(struct mailbox_list *src_list, + struct mailbox_list *dest_list) +{ + char sep; + + if (dest_list->subscriptions == NULL) { + sep = mail_namespace_get_sep(src_list->ns); + dest_list->subscriptions = mailbox_tree_init(sep); + } + return 0; +} + +static int shared_list_set_subscribed(struct mailbox_list *list, + const char *name, bool set) +{ + struct mail_namespace *ns = list->ns; + int ret; + + if (shared_storage_get_namespace(&ns, &name) < 0) + return -1; + ret = mailbox_list_set_subscribed(ns->list, name, set); + if (ret < 0) + shared_list_copy_error(list, ns); + return ret; +} + +static int +shared_list_delete_mailbox(struct mailbox_list *list, const char *name) +{ + struct mail_namespace *ns = list->ns; + int ret; + + if (shared_storage_get_namespace(&ns, &name) < 0) + return -1; + ret = ns->list->v.delete_mailbox(ns->list, name); + if (ret < 0) + shared_list_copy_error(list, ns); + return ret; +} + +static int +shared_list_delete_dir(struct mailbox_list *list, const char *name) +{ + struct mail_namespace *ns = list->ns; + int ret; + + if (shared_storage_get_namespace(&ns, &name) < 0) + return -1; + ret = mailbox_list_delete_dir(ns->list, name); + if (ret < 0) + shared_list_copy_error(list, ns); + return ret; +} + +static int +shared_list_delete_symlink(struct mailbox_list *list, const char *name) +{ + struct mail_namespace *ns = list->ns; + int ret; + + if (shared_storage_get_namespace(&ns, &name) < 0) + return -1; + ret = mailbox_list_delete_symlink(ns->list, name); + if (ret < 0) + shared_list_copy_error(list, ns); + return ret; +} + +static int shared_list_rename_get_ns(struct mailbox_list *oldlist, + const char **oldname, + struct mailbox_list *newlist, + const char **newname, + struct mail_namespace **ns_r) +{ + struct mail_namespace *old_ns = oldlist->ns, *new_ns = newlist->ns; + + if (shared_storage_get_namespace(&old_ns, oldname) < 0 || + shared_storage_get_namespace(&new_ns, newname) < 0) + return -1; + if (old_ns != new_ns) { + mailbox_list_set_error(oldlist, MAIL_ERROR_NOTPOSSIBLE, + "Can't rename shared mailboxes across storages."); + return -1; + } + *ns_r = old_ns; + return 0; +} + +static int +shared_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname, + struct mailbox_list *newlist, const char *newname) +{ + struct mail_namespace *ns; + int ret; + + if (shared_list_rename_get_ns(oldlist, &oldname, + newlist, &newname, &ns) < 0) + return -1; + + ret = ns->list->v.rename_mailbox(ns->list, oldname, ns->list, newname); + if (ret < 0) + shared_list_copy_error(oldlist, ns); + return ret; +} + +struct mailbox_list shared_mailbox_list = { + .name = "shared", + .props = 0, + .mailbox_name_max_length = MAILBOX_LIST_NAME_MAX_LENGTH, + + .v = { + .alloc = shared_list_alloc, + .deinit = shared_list_deinit, + .get_storage = shared_get_storage, + .get_hierarchy_sep = shared_list_get_hierarchy_sep, + .get_vname = mailbox_list_default_get_vname, + .get_storage_name = mailbox_list_default_get_storage_name, + .get_path = shared_list_get_path, + .get_temp_prefix = shared_list_get_temp_prefix, + .join_refpattern = shared_list_join_refpattern, + .iter_init = shared_list_iter_init, + .iter_next = shared_list_iter_next, + .iter_deinit = shared_list_iter_deinit, + .subscriptions_refresh = shared_list_subscriptions_refresh, + .set_subscribed = shared_list_set_subscribed, + .delete_mailbox = shared_list_delete_mailbox, + .delete_dir = shared_list_delete_dir, + .delete_symlink = shared_list_delete_symlink, + .rename_mailbox = shared_list_rename_mailbox, + } +}; diff --git a/src/lib-storage/index/shared/shared-storage.c b/src/lib-storage/index/shared/shared-storage.c new file mode 100644 index 0000000..978949e --- /dev/null +++ b/src/lib-storage/index/shared/shared-storage.c @@ -0,0 +1,379 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "ioloop.h" +#include "var-expand.h" +#include "index-storage.h" +#include "mail-storage-service.h" +#include "mailbox-list-private.h" +#include "fail-mail-storage.h" +#include "shared-storage.h" + +#include <ctype.h> + +extern struct mail_storage shared_storage; + +static struct mail_storage *shared_storage_alloc(void) +{ + struct shared_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("shared storage", 1024); + storage = p_new(pool, struct shared_storage, 1); + storage->storage = shared_storage; + storage->storage.pool = pool; + return &storage->storage; +} + +static int +shared_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, + const char **error_r) +{ + struct shared_storage *storage = SHARED_STORAGE(_storage); + const char *driver, *p; + char *wildcardp, key; + bool have_username; + + /* location must begin with the actual mailbox driver */ + p = strchr(ns->set->location, ':'); + if (p == NULL) { + *error_r = "Shared mailbox location not prefixed with driver"; + return -1; + } + driver = t_strdup_until(ns->set->location, p); + storage->location = p_strdup(_storage->pool, ns->set->location); + storage->unexpanded_location = + p_strdup(_storage->pool, ns->unexpanded_set->location); + storage->storage_class_name = p_strdup(_storage->pool, driver); + + if (mail_user_get_storage_class(_storage->user, driver) == NULL && + strcmp(driver, "auto") != 0) { + *error_r = t_strconcat("Unknown shared storage driver: ", + driver, NULL); + return -1; + } + + wildcardp = strchr(ns->prefix, '%'); + if (wildcardp == NULL) { + *error_r = "Shared namespace prefix doesn't contain %"; + return -1; + } + storage->ns_prefix_pattern = p_strdup(_storage->pool, wildcardp); + + have_username = FALSE; + for (p = storage->ns_prefix_pattern; *p != '\0'; p++) { + if (*p != '%') + continue; + + key = p[1]; + if (key == 'u' || key == 'n') + have_username = TRUE; + else if (key != '%' && key != 'd') + break; + } + if (*p != '\0') { + *error_r = "Shared namespace prefix contains unknown variables"; + return -1; + } + if (!have_username) { + *error_r = "Shared namespace prefix doesn't contain %u or %n"; + return -1; + } + if (p[-1] != mail_namespace_get_sep(ns) && + (ns->flags & (NAMESPACE_FLAG_LIST_PREFIX | + NAMESPACE_FLAG_LIST_CHILDREN)) != 0) { + *error_r = "Shared namespace prefix doesn't end with hierarchy separator"; + return -1; + } + + /* truncate prefix after the above checks are done, so they can log + the full prefix in error conditions */ + *wildcardp = '\0'; + ns->prefix_len = strlen(ns->prefix); + return 0; +} + +static void +shared_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, + struct mailbox_list_settings *set) +{ + set->layout = "shared"; +} + +static void +get_nonexistent_user_location(struct shared_storage *storage, + const char *username, string_t *location) +{ + /* user wasn't found. we'll still need to create the storage + to avoid exposing which users exist and which don't. */ + str_append(location, storage->storage_class_name); + str_append_c(location, ':'); + + /* use a reachable but nonexistent path as the mail root directory */ + str_append(location, storage->storage.user->set->base_dir); + str_append(location, "/user-not-found/"); + str_append(location, username); +} + +static bool shared_namespace_exists(struct mail_namespace *ns) +{ + const char *path; + struct stat st; + + if (!mailbox_list_get_root_path(ns->list, MAILBOX_LIST_PATH_TYPE_DIR, + &path)) { + /* we can't know if this exists */ + return TRUE; + } + return stat(path, &st) == 0; +} + +int shared_storage_get_namespace(struct mail_namespace **_ns, + const char **_name) +{ + struct mail_storage *_storage = (*_ns)->storage; + struct mailbox_list *list = (*_ns)->list; + struct shared_storage *storage = SHARED_STORAGE(_storage); + struct mail_user *user = _storage->user; + struct mail_namespace *new_ns, *ns = *_ns; + struct mail_namespace_settings *ns_set, *unexpanded_ns_set; + struct mail_user *owner; + const char *domain = NULL, *username = NULL, *userdomain = NULL; + const char *name, *p, *next, **dest, *error; + string_t *prefix, *location; + char ns_sep = mail_namespace_get_sep(ns); + int ret; + + p = storage->ns_prefix_pattern; + for (name = *_name; *p != '\0';) { + if (*p != '%') { + if (*p != *name) + break; + p++; name++; + continue; + } + switch (*++p) { + case 'd': + dest = &domain; + break; + case 'n': + dest = &username; + break; + case 'u': + dest = &userdomain; + break; + default: + /* we checked this already above */ + i_unreached(); + } + p++; + + next = strchr(name, *p != '\0' ? *p : ns_sep); + if (next == NULL) { + *dest = name; + name = ""; + break; + } + *dest = t_strdup_until(name, next); + name = next; + } + if (*p != '\0') { + if (*name == '\0' || + (name[1] == '\0' && *name == ns_sep)) { + /* trying to open <prefix>/<user> mailbox */ + name = "INBOX"; + } else { + mailbox_list_set_critical(list, + "Invalid namespace prefix %s vs %s", + storage->ns_prefix_pattern, *_name); + return -1; + } + } + + /* successfully matched the name. */ + if (userdomain != NULL) { + /* user@domain given */ + domain = strchr(userdomain, '@'); + if (domain == NULL) + username = userdomain; + else { + username = t_strdup_until(userdomain, domain); + domain++; + } + } else if (username == NULL) { + /* trying to open namespace "shared/domain" + namespace prefix. */ + mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(*_name)); + return -1; + } else { + if (domain == NULL) { + /* no domain given, use ours (if we have one) */ + domain = i_strchr_to_next(user->username, '@'); + } + userdomain = domain == NULL ? username : + t_strconcat(username, "@", domain, NULL); + } + if (*userdomain == '\0') { + mailbox_list_set_error(list, MAIL_ERROR_PARAMS, + "Empty username doesn't exist"); + return -1; + } + + /* expand the namespace prefix and see if it already exists. + this should normally happen only when the mailbox is being opened */ + struct var_expand_table tab[] = { + { 'u', userdomain, "user" }, + { 'n', username, "username" }, + { 'd', domain, "domain" }, + { 'h', NULL, "home" }, + { '\0', NULL, NULL } + }; + + prefix = t_str_new(128); + str_append(prefix, ns->prefix); + if (var_expand(prefix, storage->ns_prefix_pattern, tab, &error) <= 0) { + mailbox_list_set_critical(list, + "Failed to expand namespace prefix '%s': %s", + storage->ns_prefix_pattern, error); + return -1; + } + + *_ns = mail_namespace_find_prefix(user->namespaces, str_c(prefix)); + if (*_ns != NULL) { + *_name = mailbox_list_get_storage_name(ns->list, + t_strconcat(ns->prefix, name, NULL)); + return 0; + } + + owner = mail_user_alloc(event_get_parent(user->event), userdomain, + user->set_info, user->unexpanded_set); + owner->_service_user = user->_service_user; + mail_storage_service_user_ref(owner->_service_user); + owner->creator = user; + owner->autocreated = TRUE; + owner->session_id = p_strdup(owner->pool, user->session_id); + if (mail_user_init(owner, &error) < 0) { + if (!owner->nonexistent) { + mailbox_list_set_critical(list, + "Couldn't create namespace '%s' for user %s: %s", + ns->prefix, userdomain, error); + mail_user_deinit(&owner); + return -1; + } + ret = 0; + } else if (!var_has_key(storage->location, 'h', "home")) { + ret = 1; + } else { + /* we'll need to look up the user's home directory */ + if ((ret = mail_user_get_home(owner, &tab[3].value)) < 0) { + mailbox_list_set_critical(list, "Namespace '%s': " + "Could not lookup home for user %s", + ns->prefix, userdomain); + mail_user_deinit(&owner); + return -1; + } + } + + location = t_str_new(256); + if (ret > 0 && + var_expand(location, storage->location, tab, &error) <= 0) { + mailbox_list_set_critical(list, + "Failed to expand namespace location '%s': %s", + storage->location, error); + return -1; + } + + /* create the new namespace */ + new_ns = i_new(struct mail_namespace, 1); + new_ns->refcount = 1; + new_ns->type = MAIL_NAMESPACE_TYPE_SHARED; + new_ns->user = user; + new_ns->prefix = i_strdup(str_c(prefix)); + new_ns->owner = owner; + new_ns->flags = (NAMESPACE_FLAG_SUBSCRIPTIONS & ns->flags) | + NAMESPACE_FLAG_LIST_PREFIX | NAMESPACE_FLAG_HIDDEN | + NAMESPACE_FLAG_AUTOCREATED | NAMESPACE_FLAG_INBOX_ANY; + new_ns->user_set = user->set; + new_ns->mail_set = _storage->set; + i_array_init(&new_ns->all_storages, 2); + + if (ret <= 0) { + get_nonexistent_user_location(storage, userdomain, location); + new_ns->flags |= NAMESPACE_FLAG_UNUSABLE; + e_debug(ns->user->event, + "shared: Tried to access mails of " + "nonexistent user %s", userdomain); + } + + ns_set = p_new(user->pool, struct mail_namespace_settings, 1); + ns_set->type = "shared"; + ns_set->separator = p_strdup_printf(user->pool, "%c", ns_sep); + ns_set->prefix = new_ns->prefix; + ns_set->location = p_strdup(user->pool, str_c(location)); + ns_set->hidden = TRUE; + ns_set->list = "yes"; + new_ns->set = ns_set; + + unexpanded_ns_set = + p_new(user->pool, struct mail_namespace_settings, 1); + *unexpanded_ns_set = *ns_set; + unexpanded_ns_set->location = + p_strdup(user->pool, storage->unexpanded_location); + new_ns->unexpanded_set = unexpanded_ns_set; + + /* We need to create a prefix="" namespace for the owner */ + if (mail_namespaces_init_location(owner, str_c(location), &error) < 0) { + /* owner gets freed by namespace deinit */ + mail_namespace_destroy(new_ns); + return -1; + } + + if (mail_storage_create(new_ns, NULL, _storage->flags | + MAIL_STORAGE_FLAG_NO_AUTOVERIFY, &error) < 0) { + mailbox_list_set_critical(list, "Namespace '%s': %s", + new_ns->prefix, error); + /* owner gets freed by namespace deinit */ + mail_namespace_destroy(new_ns); + return -1; + } + if ((new_ns->flags & NAMESPACE_FLAG_UNUSABLE) == 0 && + !shared_namespace_exists(new_ns)) { + /* this user doesn't have a usable storage */ + new_ns->flags |= NAMESPACE_FLAG_UNUSABLE; + } + /* mark the shared namespace root as usable, since it now has + child namespaces */ + ns->flags |= NAMESPACE_FLAG_USABLE; + *_name = mailbox_list_get_storage_name(new_ns->list, + t_strconcat(new_ns->prefix, name, NULL)); + *_ns = new_ns; + if (_storage->class_flags == 0) { + /* flags are unset if we were using "auto" storage */ + _storage->class_flags = + mail_namespace_get_default_storage(new_ns)->class_flags; + } + + mail_user_add_namespace(user, &new_ns); + return 0; +} + +struct mail_storage shared_storage = { + .name = MAIL_SHARED_STORAGE_NAME, + .class_flags = 0, /* unknown at this point */ + + .v = { + NULL, + shared_storage_alloc, + shared_storage_create, + index_storage_destroy, + NULL, + shared_storage_get_list_settings, + NULL, + fail_mailbox_alloc, + NULL, + NULL, + } +}; diff --git a/src/lib-storage/index/shared/shared-storage.h b/src/lib-storage/index/shared/shared-storage.h new file mode 100644 index 0000000..5463b1c --- /dev/null +++ b/src/lib-storage/index/shared/shared-storage.h @@ -0,0 +1,22 @@ +#ifndef SHARED_STORAGE_H +#define SHARED_STORAGE_H + +struct shared_storage { + struct mail_storage storage; + union mailbox_list_module_context list_module_ctx; + + const char *ns_prefix_pattern; + const char *location, *unexpanded_location; + + const char *storage_class_name; +}; + +#define SHARED_STORAGE(s) container_of(s, struct shared_storage, storage) + +struct mailbox_list *shared_mailbox_list_alloc(void); + +/* Returns -1 = error, 0 = user doesn't exist, 1 = ok */ +int shared_storage_get_namespace(struct mail_namespace **_ns, + const char **_name); + +#endif |