diff options
Diffstat (limited to 'src/lib-storage/index/mbox')
23 files changed, 9548 insertions, 0 deletions
diff --git a/src/lib-storage/index/mbox/Makefile.am b/src/lib-storage/index/mbox/Makefile.am new file mode 100644 index 0000000..4165a20 --- /dev/null +++ b/src/lib-storage/index/mbox/Makefile.am @@ -0,0 +1,39 @@ +noinst_LTLIBRARIES = libstorage_mbox.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_mbox_la_SOURCES = \ + istream-raw-mbox.c \ + mbox-file.c \ + mbox-lock.c \ + mbox-mail.c \ + mbox-md5-apop3d.c \ + mbox-md5-all.c \ + mbox-save.c \ + mbox-settings.c \ + mbox-sync-list-index.c \ + mbox-sync-parse.c \ + mbox-sync-rewrite.c \ + mbox-sync-update.c \ + mbox-sync.c \ + mbox-storage.c + +headers = \ + istream-raw-mbox.h \ + mbox-file.h \ + mbox-lock.h \ + mbox-md5.h \ + mbox-settings.h \ + mbox-storage.h \ + mbox-sync-private.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/mbox/Makefile.in b/src/lib-storage/index/mbox/Makefile.in new file mode 100644 index 0000000..c0b98cb --- /dev/null +++ b/src/lib-storage/index/mbox/Makefile.in @@ -0,0 +1,884 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/mbox +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_mbox_la_LIBADD = +am_libstorage_mbox_la_OBJECTS = istream-raw-mbox.lo mbox-file.lo \ + mbox-lock.lo mbox-mail.lo mbox-md5-apop3d.lo mbox-md5-all.lo \ + mbox-save.lo mbox-settings.lo mbox-sync-list-index.lo \ + mbox-sync-parse.lo mbox-sync-rewrite.lo mbox-sync-update.lo \ + mbox-sync.lo mbox-storage.lo +libstorage_mbox_la_OBJECTS = $(am_libstorage_mbox_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/istream-raw-mbox.Plo \ + ./$(DEPDIR)/mbox-file.Plo ./$(DEPDIR)/mbox-lock.Plo \ + ./$(DEPDIR)/mbox-mail.Plo ./$(DEPDIR)/mbox-md5-all.Plo \ + ./$(DEPDIR)/mbox-md5-apop3d.Plo ./$(DEPDIR)/mbox-save.Plo \ + ./$(DEPDIR)/mbox-settings.Plo ./$(DEPDIR)/mbox-storage.Plo \ + ./$(DEPDIR)/mbox-sync-list-index.Plo \ + ./$(DEPDIR)/mbox-sync-parse.Plo \ + ./$(DEPDIR)/mbox-sync-rewrite.Plo \ + ./$(DEPDIR)/mbox-sync-update.Plo ./$(DEPDIR)/mbox-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_mbox_la_SOURCES) +DIST_SOURCES = $(libstorage_mbox_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_mbox.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_mbox_la_SOURCES = \ + istream-raw-mbox.c \ + mbox-file.c \ + mbox-lock.c \ + mbox-mail.c \ + mbox-md5-apop3d.c \ + mbox-md5-all.c \ + mbox-save.c \ + mbox-settings.c \ + mbox-sync-list-index.c \ + mbox-sync-parse.c \ + mbox-sync-rewrite.c \ + mbox-sync-update.c \ + mbox-sync.c \ + mbox-storage.c + +headers = \ + istream-raw-mbox.h \ + mbox-file.h \ + mbox-lock.h \ + mbox-md5.h \ + mbox-settings.h \ + mbox-storage.h \ + mbox-sync-private.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/mbox/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/mbox/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_mbox.la: $(libstorage_mbox_la_OBJECTS) $(libstorage_mbox_la_DEPENDENCIES) $(EXTRA_libstorage_mbox_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_mbox_la_OBJECTS) $(libstorage_mbox_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-raw-mbox.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-lock.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-md5-all.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-md5-apop3d.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-list-index.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-parse.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-rewrite.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-update.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/istream-raw-mbox.Plo + -rm -f ./$(DEPDIR)/mbox-file.Plo + -rm -f ./$(DEPDIR)/mbox-lock.Plo + -rm -f ./$(DEPDIR)/mbox-mail.Plo + -rm -f ./$(DEPDIR)/mbox-md5-all.Plo + -rm -f ./$(DEPDIR)/mbox-md5-apop3d.Plo + -rm -f ./$(DEPDIR)/mbox-save.Plo + -rm -f ./$(DEPDIR)/mbox-settings.Plo + -rm -f ./$(DEPDIR)/mbox-storage.Plo + -rm -f ./$(DEPDIR)/mbox-sync-list-index.Plo + -rm -f ./$(DEPDIR)/mbox-sync-parse.Plo + -rm -f ./$(DEPDIR)/mbox-sync-rewrite.Plo + -rm -f ./$(DEPDIR)/mbox-sync-update.Plo + -rm -f ./$(DEPDIR)/mbox-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/istream-raw-mbox.Plo + -rm -f ./$(DEPDIR)/mbox-file.Plo + -rm -f ./$(DEPDIR)/mbox-lock.Plo + -rm -f ./$(DEPDIR)/mbox-mail.Plo + -rm -f ./$(DEPDIR)/mbox-md5-all.Plo + -rm -f ./$(DEPDIR)/mbox-md5-apop3d.Plo + -rm -f ./$(DEPDIR)/mbox-save.Plo + -rm -f ./$(DEPDIR)/mbox-settings.Plo + -rm -f ./$(DEPDIR)/mbox-storage.Plo + -rm -f ./$(DEPDIR)/mbox-sync-list-index.Plo + -rm -f ./$(DEPDIR)/mbox-sync-parse.Plo + -rm -f ./$(DEPDIR)/mbox-sync-rewrite.Plo + -rm -f ./$(DEPDIR)/mbox-sync-update.Plo + -rm -f ./$(DEPDIR)/mbox-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.c b/src/lib-storage/index/mbox/istream-raw-mbox.c new file mode 100644 index 0000000..e91e581 --- /dev/null +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c @@ -0,0 +1,821 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "istream-private.h" +#include "istream-raw-mbox.h" +#include "mbox-from.h" + +struct raw_mbox_istream { + struct istream_private istream; + + time_t received_time, next_received_time; + char *sender, *next_sender; + + uoff_t from_offset, hdr_offset, body_offset, mail_size; + uoff_t input_peak_offset; + + bool locked:1; + bool seeked:1; + bool crlf_ending:1; + bool corrupted:1; + bool mail_size_forced:1; + bool eof:1; + bool header_missing_eoh:1; +}; + +static void mbox_istream_log_read_error(struct raw_mbox_istream *rstream) +{ + if (rstream->istream.parent->stream_errno != 0) { + /* Log e.g. compression istream error */ + i_error("Failed to read mbox file %s: %s", + i_stream_get_name(&rstream->istream.istream), + i_stream_get_error(rstream->istream.parent)); + } +} + +static void i_stream_raw_mbox_destroy(struct iostream_private *stream) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + i_free(rstream->sender); + i_free(rstream->next_sender); + + i_stream_seek(rstream->istream.parent, + rstream->istream.istream.v_offset); +} + +static int mbox_read_from_line(struct raw_mbox_istream *rstream) +{ + const unsigned char *buf, *p; + char *sender; + time_t received_time; + size_t pos, line_pos; + ssize_t ret; + unsigned int skip; + int tz; + + buf = i_stream_get_data(rstream->istream.parent, &pos); + i_assert(pos > 0); + + /* from_offset points to "\nFrom ", so unless we're at the beginning + of the file, skip the initial \n */ + if (rstream->from_offset == 0) + skip = 0; + else { + skip = 1; + if (*buf == '\r') + skip++; + } + + while ((p = memchr(buf+skip, '\n', pos-skip)) == NULL) { + ret = i_stream_read_memarea(rstream->istream.parent); + buf = i_stream_get_data(rstream->istream.parent, &pos); + if (ret < 0) { + if (ret == -2) { + /* From_-line is too long, but we should be + able to parse what we have so far. */ + break; + } + /* EOF shouldn't happen */ + rstream->istream.istream.eof = + rstream->istream.parent->eof; + rstream->istream.istream.stream_errno = + rstream->istream.parent->stream_errno; + return -1; + } + i_assert(pos > 0); + } + line_pos = p == NULL ? 0 : (size_t)(p - buf); + + /* beginning of mbox */ + if (memcmp(buf+skip, "From ", 5) != 0 || + mbox_from_parse((buf+skip)+5, (pos-skip)-5, + &received_time, &tz, &sender) < 0) { + /* broken From - should happen only at beginning of + file if this isn't a mbox.. */ + io_stream_set_error(&rstream->istream.iostream, + "mbox file doesn't begin with 'From ' line"); + rstream->istream.istream.stream_errno = EINVAL; + return -1; + } + + if (rstream->istream.istream.v_offset == rstream->from_offset) { + rstream->received_time = received_time; + i_free(rstream->sender); + rstream->sender = sender; + } else { + rstream->next_received_time = received_time; + i_free(rstream->next_sender); + rstream->next_sender = sender; + } + + /* skip over From-line */ + if (line_pos == 0) { + /* line was too long. skip the input until we find LF. */ + rstream->istream.istream.v_offset += pos; + i_stream_skip(rstream->istream.parent, pos); + + while ((ret = i_stream_read_memarea(rstream->istream.parent)) > 0) { + p = memchr(buf, '\n', pos); + if (p != NULL) + break; + rstream->istream.istream.v_offset += pos; + i_stream_skip(rstream->istream.parent, pos); + } + if (ret <= 0) { + i_assert(ret == -1); + /* EOF shouldn't happen */ + rstream->istream.istream.eof = + rstream->istream.parent->eof; + rstream->istream.istream.stream_errno = + rstream->istream.parent->stream_errno; + return -1; + } + line_pos = (size_t)(p - buf); + } + rstream->istream.istream.v_offset += line_pos+1; + i_stream_skip(rstream->istream.parent, line_pos+1); + + rstream->hdr_offset = rstream->istream.istream.v_offset; + return 0; +} + +static void handle_end_of_mail(struct raw_mbox_istream *rstream, size_t pos) +{ + rstream->mail_size = rstream->istream.istream.v_offset + pos - + rstream->hdr_offset; + + if (rstream->hdr_offset + rstream->mail_size < rstream->body_offset) { + uoff_t new_body_offset = + rstream->hdr_offset + rstream->mail_size; + + if (rstream->body_offset != UOFF_T_MAX) { + /* Header didn't have ending \n */ + rstream->header_missing_eoh = TRUE; + } else { + /* "headers\n\nFrom ..", the second \n belongs to next + message which we didn't know at the time yet. */ + } + + /* The +2 check is for CR+LF linefeeds */ + i_assert(rstream->body_offset == UOFF_T_MAX || + rstream->body_offset == new_body_offset + 1 || + rstream->body_offset == new_body_offset + 2); + rstream->body_offset = new_body_offset; + } +} + +static ssize_t i_stream_raw_mbox_read(struct istream_private *stream) +{ + static const char *mbox_from = "\nFrom "; + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + const unsigned char *buf; + const char *fromp; + char *sender; + time_t received_time; + size_t i, pos, new_pos, from_start_pos, from_after_pos; + ssize_t ret = 0; + int eoh_char, tz; + bool crlf_ending = FALSE; + + i_assert(rstream->seeked); + i_assert(stream->istream.v_offset >= rstream->from_offset); + + if (stream->istream.eof) + return -1; + if (rstream->corrupted) { + rstream->istream.istream.stream_errno = EINVAL; + return -1; + } + + i_stream_seek(stream->parent, stream->istream.v_offset); + + stream->pos -= stream->skip; + stream->skip = 0; + stream->buffer = NULL; + + do { + buf = i_stream_get_data(stream->parent, &pos); + if (pos > 1 && stream->istream.v_offset + pos > + rstream->input_peak_offset) { + /* fake our read count. needed because if in the end + we have only one character in buffer and we skip it + (as potential CR), we want to get back to this + i_stream_raw_mbox_read() to read more data. */ + ret = pos; + break; + } + ret = i_stream_read_memarea(stream->parent); + } while (ret > 0); + stream->istream.stream_errno = stream->parent->stream_errno; + + if (ret < 0) { + if (ret == -1) + mbox_istream_log_read_error(rstream); + if (ret == -2) { + if (stream->skip == stream->pos) { + /* From_-line is longer than our input buffer. + finish the check without seeing the LF. */ + } else if (stream->istream.v_offset + pos == + rstream->input_peak_offset) { + /* we've read everything our parent stream + has to offer. */ + stream->buffer = buf; + return -2; + } + /* parent stream is full, but we haven't returned + all its bytes to our caller yet. */ + } else if (stream->istream.v_offset != 0 || pos == 0) { + /* we've read the whole file, final byte should be + the \n trailer */ + if (pos > 0 && buf[pos-1] == '\n') { + pos--; + if (pos > 0 && buf[pos-1] == '\r') { + crlf_ending = TRUE; + pos--; + } + } + + i_assert(pos >= stream->pos); + ret = pos == stream->pos ? -1 : + (ssize_t)(pos - stream->pos); + + stream->buffer = buf; + stream->pos = pos; + + if (stream->istream.v_offset == rstream->from_offset) { + /* haven't seen From-line yet, so this mbox + stream is now at EOF */ + rstream->eof = TRUE; + } + stream->istream.eof = TRUE; + rstream->crlf_ending = crlf_ending; + handle_end_of_mail(rstream, pos); + return ret < 0 ? i_stream_raw_mbox_read(stream) : ret; + } + } + + if (stream->istream.v_offset == rstream->from_offset) { + /* beginning of message, we haven't yet read our From-line */ + if (pos == 2 && ret > 0) { + /* we're at the end of file with CR+LF linefeeds? + need more data to verify it. */ + rstream->input_peak_offset = + stream->istream.v_offset + pos; + return i_stream_raw_mbox_read(stream); + } + if (mbox_read_from_line(rstream) < 0) { + io_stream_set_error(&stream->iostream, + "Next message unexpectedly corrupted in mbox file " + "%s at %"PRIuUOFF_T, + i_stream_get_name(&stream->istream), + stream->istream.v_offset); + if (stream->istream.v_offset != 0) + i_error("%s", stream->iostream.error); + stream->pos = 0; + rstream->eof = TRUE; + rstream->corrupted = TRUE; + return -1; + } + + /* got it. we don't want to return it however, + so start again from headers */ + buf = i_stream_get_data(stream->parent, &pos); + if (pos == 0) + return i_stream_raw_mbox_read(stream); + } + + /* See if we have From-line here - note that it works right only + because all characters are different in mbox_from. */ + fromp = mbox_from; from_start_pos = from_after_pos = SIZE_MAX; + eoh_char = rstream->body_offset == UOFF_T_MAX ? '\n' : -1; + for (i = stream->pos; i < pos; i++) { + if (buf[i] == eoh_char && + ((i > 0 && buf[i-1] == '\n') || + (i > 1 && buf[i-1] == '\r' && buf[i-2] == '\n') || + stream->istream.v_offset + i == rstream->hdr_offset)) { + rstream->body_offset = stream->istream.v_offset + i + 1; + eoh_char = -1; + } + if ((char)buf[i] == *fromp) { + if (*++fromp == '\0') { + /* potential From-line, see if we have the + rest of the line buffered. */ + i++; + if (i >= 7 && buf[i-7] == '\r') { + /* CR also belongs to it. */ + crlf_ending = TRUE; + from_start_pos = i - 7; + } else { + crlf_ending = FALSE; + from_start_pos = i - 6; + } + + if (rstream->mail_size == UOFF_T_MAX || + rstream->hdr_offset + rstream->mail_size == + stream->istream.v_offset + from_start_pos) { + from_after_pos = i; + if (ret == -2) { + /* even if we don't have the + whole line, we need to + finish this check now. */ + goto mbox_verify; + } + } + fromp = mbox_from; + } else if (from_after_pos != SIZE_MAX) { + /* we have the whole From-line here now. + See if it's a valid one. */ + mbox_verify: + if (mbox_from_parse(buf + from_after_pos, + pos - from_after_pos, + &received_time, &tz, + &sender) == 0) { + /* yep, we stop here. */ + rstream->next_received_time = + received_time; + i_free(rstream->next_sender); + rstream->next_sender = sender; + stream->istream.eof = TRUE; + + rstream->crlf_ending = crlf_ending; + handle_end_of_mail(rstream, + from_start_pos); + break; + } + from_after_pos = SIZE_MAX; + } + } else { + fromp = mbox_from; + if ((char)buf[i] == *fromp) + fromp++; + } + } + + /* we want to go at least one byte further next time */ + rstream->input_peak_offset = stream->istream.v_offset + i; + + if (from_after_pos != SIZE_MAX) { + /* we're waiting for the \n at the end of From-line */ + new_pos = from_start_pos; + } else { + /* leave out the beginnings of potential From-line + CR */ + new_pos = i - (fromp - mbox_from); + if (new_pos > 0) + new_pos--; + } + + if (stream->istream.v_offset - + rstream->hdr_offset + new_pos > rstream->mail_size) { + /* istream_raw_mbox_set_next_offset() used invalid + cached next_offset? */ + io_stream_set_error(&stream->iostream, + "Next message unexpectedly lost from mbox file " + "%s at %"PRIuUOFF_T" (%s)", + i_stream_get_name(&stream->istream), + rstream->hdr_offset + rstream->mail_size, + rstream->mail_size_forced ? "cached" : "noncached"); + i_error("%s", stream->iostream.error); + rstream->eof = TRUE; + rstream->corrupted = TRUE; + rstream->istream.istream.stream_errno = EINVAL; + stream->pos = 0; + return -1; + } + + stream->buffer = buf; + if (new_pos == stream->pos) { + if (stream->istream.eof || ret > 0) + return i_stream_raw_mbox_read(stream); + i_assert(new_pos > 0); + ret = -2; + } else { + i_assert(new_pos > stream->pos); + ret = new_pos - stream->pos; + stream->pos = new_pos; + } + return ret; +} + +static void i_stream_raw_mbox_seek(struct istream_private *stream, + uoff_t v_offset, bool mark ATTR_UNUSED) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + stream->istream.v_offset = v_offset; + stream->skip = stream->pos = 0; + stream->buffer = NULL; + + rstream->input_peak_offset = 0; + rstream->eof = FALSE; +} + +static void i_stream_raw_mbox_sync(struct istream_private *stream) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + i_stream_sync(stream->parent); + + rstream->istream.skip = 0; + rstream->istream.pos = 0; + rstream->input_peak_offset = 0; +} + +static int +i_stream_raw_mbox_stat(struct istream_private *stream, bool exact) +{ + const struct stat *st; + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; + return -1; + } + + stream->statbuf = *st; + stream->statbuf.st_size = + !exact && rstream->seeked && rstream->mail_size != UOFF_T_MAX ? + (off_t)rstream->mail_size : -1; + return 0; +} + +struct istream *i_stream_create_raw_mbox(struct istream *input) +{ + struct raw_mbox_istream *rstream; + + i_assert(input->v_offset == 0); + + rstream = i_new(struct raw_mbox_istream, 1); + + rstream->body_offset = UOFF_T_MAX; + rstream->mail_size = UOFF_T_MAX; + rstream->received_time = (time_t)-1; + rstream->next_received_time = (time_t)-1; + + rstream->istream.iostream.destroy = i_stream_raw_mbox_destroy; + rstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + rstream->istream.read = i_stream_raw_mbox_read; + rstream->istream.seek = i_stream_raw_mbox_seek; + rstream->istream.sync = i_stream_raw_mbox_sync; + rstream->istream.stat = i_stream_raw_mbox_stat; + + rstream->istream.istream.readable_fd = input->readable_fd; + rstream->istream.istream.blocking = input->blocking; + rstream->istream.istream.seekable = input->seekable; + + return i_stream_create(&rstream->istream, input, -1, 0); +} + +static int istream_raw_mbox_is_valid_from(struct raw_mbox_istream *rstream) +{ + const unsigned char *data; + size_t size = 0; + time_t received_time; + char *sender; + int tz; + ssize_t ret = 0; + + /* minimal: "From x Thu Nov 29 22:33:52 2001" = 31 chars */ + do { + data = i_stream_get_data(rstream->istream.parent, &size); + if (size >= 31) + break; + } while ((ret = i_stream_read_memarea(rstream->istream.parent)) > 0); + if (ret == -1) + mbox_istream_log_read_error(rstream); + + if ((size == 1 && data[0] == '\n') || + (size == 2 && data[0] == '\r' && data[1] == '\n')) { + /* EOF */ + return 1; + } + + if (size > 31 && memcmp(data, "\nFrom ", 6) == 0) { + data += 6; + size -= 6; + } else if (size > 32 && memcmp(data, "\r\nFrom ", 7) == 0) { + data += 7; + size -= 7; + } else { + return 0; + } + + while (memchr(data, '\n', size) == NULL) { + ret = i_stream_read_bytes(rstream->istream.parent, + &data, &size, size+1); + if (ret < 0) { + if (ret == -1) + mbox_istream_log_read_error(rstream); + break; + } + } + + if (mbox_from_parse(data, size, &received_time, &tz, &sender) < 0) + return 0; + + rstream->next_received_time = received_time; + i_free(rstream->next_sender); + rstream->next_sender = sender; + return 1; +} + +uoff_t istream_raw_mbox_get_start_offset(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + return rstream->from_offset; +} + +int istream_raw_mbox_get_header_offset(struct istream *stream, + uoff_t *hdr_offset_r) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + if (rstream->hdr_offset == rstream->from_offset) + (void)i_stream_read(stream); + + if (rstream->corrupted) { + i_error("Unexpectedly lost From-line from mbox file %s at " + "%"PRIuUOFF_T, i_stream_get_name(stream), + rstream->from_offset); + return -1; + } + if (stream->stream_errno != 0) + return -1; + + *hdr_offset_r = rstream->hdr_offset; + return 0; +} + +int istream_raw_mbox_get_body_offset(struct istream *stream, + uoff_t *body_offset_r) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + uoff_t offset; + + i_assert(rstream->seeked); + + if (rstream->body_offset != UOFF_T_MAX) { + *body_offset_r = rstream->body_offset; + return 0; + } + + offset = stream->v_offset; + i_stream_seek(stream, rstream->hdr_offset); + while (rstream->body_offset == UOFF_T_MAX) { + i_stream_skip(stream, i_stream_get_data_size(stream)); + + if (i_stream_read(stream) < 0) { + if (rstream->corrupted) { + i_error("Unexpectedly lost From-line from mbox file " + "%s at %"PRIuUOFF_T, + i_stream_get_name(stream), + rstream->from_offset); + } else { + i_assert(rstream->body_offset != UOFF_T_MAX); + } + return -1; + } + } + + i_stream_seek(stream, offset); + *body_offset_r = rstream->body_offset; + return 0; +} + +int istream_raw_mbox_get_body_size(struct istream *stream, + uoff_t expected_body_size, + uoff_t *body_size_r) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + const unsigned char *data; + size_t size; + uoff_t old_offset, body_offset, body_size, next_body_offset; + + i_assert(rstream->seeked); + i_assert(rstream->hdr_offset != UOFF_T_MAX); + + if (istream_raw_mbox_get_body_offset(stream, &body_offset) < 0) + return -1; + body_size = rstream->mail_size == UOFF_T_MAX ? UOFF_T_MAX : + rstream->mail_size - (rstream->body_offset - + rstream->hdr_offset); + old_offset = stream->v_offset; + if (expected_body_size != UOFF_T_MAX) { + /* if we already have the existing body size, use it as long as + it's >= expected body_size. otherwise the previous parsing + may have stopped at a From_-line that belongs to the body. */ + if (body_size != UOFF_T_MAX && body_size >= expected_body_size) { + *body_size_r = body_size; + return 0; + } + + next_body_offset = rstream->body_offset + expected_body_size; + /* If header_missing_eoh is set, the message body begins with + a From_-line and the body_offset is pointing to the line + *before* the first line of the body, i.e. the empty line + separating the headers from the body. If that is the case, + we'll have to skip over the empty line to get the correct + next_body_offset. */ + if (rstream->header_missing_eoh) { + i_assert(body_size == 0); + next_body_offset += rstream->crlf_ending ? 2 : 1; + } + + i_stream_seek(rstream->istream.parent, next_body_offset); + if (istream_raw_mbox_is_valid_from(rstream) > 0) { + rstream->mail_size = + next_body_offset - rstream->hdr_offset; + i_stream_seek(stream, old_offset); + *body_size_r = expected_body_size; + return 0; + } + /* invalid expected_body_size */ + } + if (body_size != UOFF_T_MAX) { + *body_size_r = body_size; + return 0; + } + + /* have to read through the message body */ + while (i_stream_read_more(stream, &data, &size) > 0) + i_stream_skip(stream, size); + i_stream_seek(stream, old_offset); + if (stream->stream_errno != 0) + return -1; + + i_assert(rstream->mail_size != UOFF_T_MAX); + *body_size_r = rstream->mail_size - + (rstream->body_offset - rstream->hdr_offset); + return 0; +} + +time_t istream_raw_mbox_get_received_time(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + if (rstream->received_time == (time_t)-1) + (void)i_stream_read(stream); + return rstream->received_time; +} + +const char *istream_raw_mbox_get_sender(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + if (rstream->sender == NULL) + (void)i_stream_read(stream); + return rstream->sender == NULL ? "" : rstream->sender; +} + +bool istream_raw_mbox_has_crlf_ending(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->seeked); + + return rstream->crlf_ending; +} + +int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + uoff_t body_size; + + if (istream_raw_mbox_get_body_size(stream, expected_body_size, + &body_size) < 0) + return -1; + rstream->mail_size = UOFF_T_MAX; + + rstream->received_time = rstream->next_received_time; + rstream->next_received_time = (time_t)-1; + + i_free(rstream->sender); + rstream->sender = rstream->next_sender; + rstream->next_sender = NULL; + + rstream->from_offset = rstream->body_offset + body_size; + rstream->hdr_offset = rstream->from_offset; + rstream->body_offset = UOFF_T_MAX; + rstream->header_missing_eoh = FALSE; + + if (stream->v_offset != rstream->from_offset) + i_stream_seek_mark(stream, rstream->from_offset); + i_stream_seek_mark(rstream->istream.parent, rstream->from_offset); + + rstream->eof = FALSE; + rstream->istream.istream.eof = FALSE; + return 0; +} + +int istream_raw_mbox_seek(struct istream *stream, uoff_t offset) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + bool check; + + i_assert(rstream->locked); + + /* reset any (corruption) errors */ + stream->stream_errno = 0; + i_free_and_null(stream->real_stream->iostream.error); + rstream->corrupted = FALSE; + rstream->eof = FALSE; + rstream->istream.istream.eof = FALSE; + + /* if seeked is FALSE, we unlocked in the middle. don't try to use + any cached state then. */ + if (rstream->mail_size != UOFF_T_MAX && rstream->seeked && + rstream->hdr_offset + rstream->mail_size == offset) + return istream_raw_mbox_next(stream, UOFF_T_MAX); + + if (offset == rstream->from_offset && rstream->seeked) { + /* back to beginning of current message */ + offset = rstream->hdr_offset; + check = offset == 0; + } else { + rstream->body_offset = UOFF_T_MAX; + rstream->mail_size = UOFF_T_MAX; + rstream->received_time = (time_t)-1; + rstream->next_received_time = (time_t)-1; + rstream->header_missing_eoh = FALSE; + + i_free(rstream->sender); + rstream->sender = NULL; + i_free(rstream->next_sender); + rstream->next_sender = NULL; + + rstream->from_offset = offset; + rstream->hdr_offset = offset; + check = TRUE; + } + rstream->seeked = TRUE; + + i_stream_seek_mark(stream, offset); + i_stream_seek_mark(rstream->istream.parent, offset); + + if (check) + (void)i_stream_read(stream); + return rstream->corrupted ? -1 : 0; +} + +void istream_raw_mbox_set_next_offset(struct istream *stream, uoff_t offset) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + i_assert(rstream->hdr_offset != UOFF_T_MAX); + + rstream->mail_size_forced = TRUE; + rstream->mail_size = offset - rstream->hdr_offset; +} + +bool istream_raw_mbox_is_eof(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + return rstream->eof; +} + +bool istream_raw_mbox_is_corrupted(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + return rstream->corrupted; +} + +void istream_raw_mbox_set_locked(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + rstream->locked = TRUE; +} + +void istream_raw_mbox_set_unlocked(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + rstream->locked = FALSE; + rstream->seeked = FALSE; +} diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.h b/src/lib-storage/index/mbox/istream-raw-mbox.h new file mode 100644 index 0000000..4543841 --- /dev/null +++ b/src/lib-storage/index/mbox/istream-raw-mbox.h @@ -0,0 +1,56 @@ +#ifndef ISTREAM_RAW_MBOX_H +#define ISTREAM_RAW_MBOX_H + +/* Create a mbox stream for parsing mbox. Reading stops before From-line, + you'll have to call istream_raw_mbox_next() to get to next message. + path is used only for logging purposes. */ +struct istream *i_stream_create_raw_mbox(struct istream *input); + +/* Return offset to beginning of the "\nFrom"-line. */ +uoff_t istream_raw_mbox_get_start_offset(struct istream *stream); +/* Return offset to beginning of the headers. */ +int istream_raw_mbox_get_header_offset(struct istream *stream, + uoff_t *hdr_offset_r); +/* Return offset to beginning of the body. */ +int istream_raw_mbox_get_body_offset(struct istream *stream, + uoff_t *body_offset_r); + +/* Return the number of bytes in the body of this message. If + expected_body_size isn't UOFF_T_MAX, we'll use it as potentially valid body + size to avoid actually reading through the whole message. */ +int istream_raw_mbox_get_body_size(struct istream *stream, + uoff_t expected_body_size, + uoff_t *body_size_r); + +/* Return received time of current message, or (time_t)-1 if the timestamp is + broken. */ +time_t istream_raw_mbox_get_received_time(struct istream *stream); + +/* Return sender of current message. */ +const char *istream_raw_mbox_get_sender(struct istream *stream); +/* Return TRUE if the empty line between this and the next mail contains CR. */ +bool istream_raw_mbox_has_crlf_ending(struct istream *stream); + +/* Jump to next message. If expected_body_size isn't UOFF_T_MAX, we'll use it + as potentially valid body size. */ +int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size); + +/* Seek to message at given offset. offset must point to beginning of + "\nFrom ", or 0 for beginning of file. Returns -1 if it offset doesn't + contain a valid From-line. */ +int istream_raw_mbox_seek(struct istream *stream, uoff_t offset); +/* Set next message's start offset. If this isn't set, read stops at the next + valid From_-line, even if it belongs to the current message's body + (Content-Length: header can be used to determine that). */ +void istream_raw_mbox_set_next_offset(struct istream *stream, uoff_t offset); + +/* Returns TRUE if we've read the whole mbox. */ +bool istream_raw_mbox_is_eof(struct istream *stream); +/* Returns TRUE if we've noticed corruption in used offsets/sizes. */ +bool istream_raw_mbox_is_corrupted(struct istream *stream); +/* Change stream's locking state. We'll assert-crash if stream is tried to be + read while it's unlocked. */ +void istream_raw_mbox_set_locked(struct istream *stream); +void istream_raw_mbox_set_unlocked(struct istream *stream); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-file.c b/src/lib-storage/index/mbox/mbox-file.c new file mode 100644 index 0000000..b71abb8 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-file.c @@ -0,0 +1,207 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" +#include "mbox-file.h" +#include "istream-raw-mbox.h" + +#include <sys/stat.h> +#include <utime.h> + +#define MBOX_READ_BLOCK_SIZE IO_BLOCK_SIZE + +int mbox_file_open(struct mbox_mailbox *mbox) +{ + struct stat st; + int fd; + + i_assert(mbox->mbox_fd == -1); + + if (mbox->mbox_file_stream != NULL) { + /* read-only mbox stream */ + i_assert(mbox_is_backend_readonly(mbox)); + return 0; + } + + fd = open(mailbox_get_path(&mbox->box), + mbox_is_backend_readonly(mbox) ? O_RDONLY : O_RDWR); + if (fd == -1 && errno == EACCES && !mbox->backend_readonly) { + mbox->backend_readonly = TRUE; + fd = open(mailbox_get_path(&mbox->box), O_RDONLY); + } + + if (fd == -1) { + mbox_set_syscall_error(mbox, "open()"); + return -1; + } + + if (fstat(fd, &st) < 0) { + mbox_set_syscall_error(mbox, "fstat()"); + i_close_fd(&fd); + return -1; + } + + mbox->mbox_writeonly = S_ISFIFO(st.st_mode); + mbox->mbox_fd = fd; + mbox->mbox_dev = st.st_dev; + mbox->mbox_ino = st.st_ino; + return 0; +} + +void mbox_file_close(struct mbox_mailbox *mbox) +{ + mbox_file_close_stream(mbox); + + if (mbox->mbox_fd != -1) { + if (close(mbox->mbox_fd) < 0) + mbox_set_syscall_error(mbox, "close()"); + mbox->mbox_fd = -1; + } +} + +int mbox_file_open_stream(struct mbox_mailbox *mbox) +{ + if (mbox->mbox_stream != NULL) + return 0; + + if (mbox->mbox_file_stream != NULL) { + /* read-only mbox stream */ + i_assert(mbox->mbox_fd == -1 && mbox_is_backend_readonly(mbox)); + } else { + if (mbox->mbox_fd == -1) { + if (mbox_file_open(mbox) < 0) + return -1; + } + + if (mbox->mbox_writeonly) { + mbox->mbox_file_stream = + i_stream_create_from_data("", 0); + } else { + mbox->mbox_file_stream = + i_stream_create_fd(mbox->mbox_fd, + MBOX_READ_BLOCK_SIZE); + i_stream_set_init_buffer_size(mbox->mbox_file_stream, + MBOX_READ_BLOCK_SIZE); + } + i_stream_set_name(mbox->mbox_file_stream, + mailbox_get_path(&mbox->box)); + } + + mbox->mbox_stream = i_stream_create_raw_mbox(mbox->mbox_file_stream); + if (mbox->mbox_lock_type != F_UNLCK) + istream_raw_mbox_set_locked(mbox->mbox_stream); + return 0; +} + +static void mbox_file_fix_atime(struct mbox_mailbox *mbox) +{ + struct utimbuf buf; + struct stat st; + + if (mbox->box.recent_flags_count > 0 && + (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0 && + mbox->mbox_fd != -1 && !mbox_is_backend_readonly(mbox)) { + /* we've seen recent messages which we want to keep recent. + keep file's atime lower than mtime so \Marked status + gets shown while listing */ + if (fstat(mbox->mbox_fd, &st) < 0) { + mbox_set_syscall_error(mbox, "fstat()"); + return; + } + if (st.st_atime >= st.st_mtime) { + buf.modtime = st.st_mtime; + buf.actime = buf.modtime - 1; + /* EPERM can happen with shared mailboxes */ + if (utime(mailbox_get_path(&mbox->box), &buf) < 0 && + errno != EPERM) + mbox_set_syscall_error(mbox, "utime()"); + } + } +} +void mbox_file_close_stream(struct mbox_mailbox *mbox) +{ + /* if we read anything, fix the atime if needed */ + mbox_file_fix_atime(mbox); + + i_stream_destroy(&mbox->mbox_stream); + + if (mbox->mbox_file_stream != NULL) { + if (mbox->mbox_fd == -1) { + /* read-only mbox stream */ + i_assert(mbox_is_backend_readonly(mbox)); + i_stream_seek(mbox->mbox_file_stream, 0); + } else { + i_stream_destroy(&mbox->mbox_file_stream); + } + } +} + +int mbox_file_lookup_offset(struct mbox_mailbox *mbox, + struct mail_index_view *view, + uint32_t seq, uoff_t *offset_r) +{ + const void *data; + bool deleted; + + mail_index_lookup_ext(view, seq, mbox->mbox_ext_idx, &data, &deleted); + if (deleted) + return -1; + + if (data == NULL) { + mailbox_set_critical(&mbox->box, + "Cached message offset lost for seq %u in mbox", seq); + mbox->mbox_hdr.dirty_flag = 1; + mbox->mbox_broken_offsets = TRUE; + return 0; + } + + *offset_r = *((const uint64_t *)data); + return 1; +} + +int mbox_file_seek(struct mbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, bool *deleted_r) +{ + uoff_t offset; + int ret; + + ret = mbox_file_lookup_offset(mbox, view, seq, &offset); + if (ret <= 0) { + *deleted_r = ret < 0; + return ret; + } + *deleted_r = FALSE; + + if (istream_raw_mbox_seek(mbox->mbox_stream, offset) < 0) { + if (offset == 0) { + mbox->invalid_mbox_file = TRUE; + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOTPOSSIBLE, + "Mailbox isn't a valid mbox file"); + return -1; + } + + if (mbox->mbox_hdr.dirty_flag != 0) + return 0; + + mailbox_set_critical(&mbox->box, + "Cached message offset %s is invalid for mbox", + dec2str(offset)); + mbox->mbox_hdr.dirty_flag = 1; + mbox->mbox_broken_offsets = TRUE; + return 0; + } + + if (mbox->mbox_hdr.dirty_flag != 0) { + /* we're dirty - make sure this is the correct mail */ + if (!mbox_sync_parse_match_mail(mbox, view, seq)) + return 0; + + ret = istream_raw_mbox_seek(mbox->mbox_stream, offset); + i_assert(ret == 0); + } + + return 1; +} diff --git a/src/lib-storage/index/mbox/mbox-file.h b/src/lib-storage/index/mbox/mbox-file.h new file mode 100644 index 0000000..529a2e0 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-file.h @@ -0,0 +1,16 @@ +#ifndef MBOX_FILE_H +#define MBOX_FILE_H + +int mbox_file_open(struct mbox_mailbox *mbox); +void mbox_file_close(struct mbox_mailbox *mbox); + +int mbox_file_open_stream(struct mbox_mailbox *mbox); +void mbox_file_close_stream(struct mbox_mailbox *mbox); + +int mbox_file_lookup_offset(struct mbox_mailbox *mbox, + struct mail_index_view *view, + uint32_t seq, uoff_t *offset_r); +int mbox_file_seek(struct mbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, bool *deleted_r); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-lock.c b/src/lib-storage/index/mbox/mbox-lock.c new file mode 100644 index 0000000..cebff48 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-lock.c @@ -0,0 +1,900 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "eacces-error.h" +#include "restrict-access.h" +#include "nfs-workarounds.h" +#include "ipwd.h" +#include "mail-index-private.h" +#include "mbox-storage.h" +#include "istream-raw-mbox.h" +#include "mbox-file.h" +#include "mbox-lock.h" + +#include <time.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> + +#ifdef HAVE_FLOCK +# include <sys/file.h> +#endif + +/* 0.1 .. 0.2msec */ +#define LOCK_RANDOM_USLEEP_TIME (100000 + (unsigned int)i_rand() % 100000) + +enum mbox_lock_type { + MBOX_LOCK_DOTLOCK, + MBOX_LOCK_DOTLOCK_TRY, + MBOX_LOCK_FCNTL, + MBOX_LOCK_FLOCK, + MBOX_LOCK_LOCKF, + + MBOX_LOCK_COUNT +}; + +enum mbox_dotlock_op { + MBOX_DOTLOCK_OP_LOCK, + MBOX_DOTLOCK_OP_UNLOCK, + MBOX_DOTLOCK_OP_TOUCH +}; + +struct mbox_lock_context { + struct mbox_mailbox *mbox; + bool locked_status[MBOX_LOCK_COUNT]; + bool checked_file; + + int lock_type; + bool dotlock_last_stale; + bool fcntl_locked; + bool using_privileges; +}; + +struct mbox_lock_data { + enum mbox_lock_type type; + const char *name; + int (*func)(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +}; + +static int mbox_lock_dotlock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +static int mbox_lock_dotlock_try(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +static int mbox_lock_fcntl(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +#ifdef HAVE_FLOCK +static int mbox_lock_flock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +#else +# define mbox_lock_flock NULL +#endif +#ifdef HAVE_LOCKF +static int mbox_lock_lockf(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time); +#else +# define mbox_lock_lockf NULL +#endif + +static struct mbox_lock_data lock_data[] = { + { MBOX_LOCK_DOTLOCK, "dotlock", mbox_lock_dotlock }, + { MBOX_LOCK_DOTLOCK_TRY, "dotlock_try", mbox_lock_dotlock_try }, + { MBOX_LOCK_FCNTL, "fcntl", mbox_lock_fcntl }, + { MBOX_LOCK_FLOCK, "flock", mbox_lock_flock }, + { MBOX_LOCK_LOCKF, "lockf", mbox_lock_lockf }, + { 0, NULL, NULL } +}; + +static int ATTR_NOWARN_UNUSED_RESULT +mbox_lock_list(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time, int idx); +static int ATTR_NOWARN_UNUSED_RESULT +mbox_unlock_files(struct mbox_lock_context *ctx); + +static void mbox_read_lock_methods(const char *str, const char *env, + enum mbox_lock_type *locks) +{ + enum mbox_lock_type type; + const char *const *lock; + int i, dest; + + for (lock = t_strsplit(str, " "), dest = 0; *lock != NULL; lock++) { + for (type = 0; lock_data[type].name != NULL; type++) { + if (strcasecmp(*lock, lock_data[type].name) == 0) { + type = lock_data[type].type; + break; + } + } + if (lock_data[type].name == NULL) + i_fatal("%s: Invalid value %s", env, *lock); + if (lock_data[type].func == NULL) { + i_fatal("%s: Support for lock type %s " + "not compiled into binary", env, *lock); + } + + for (i = 0; i < dest; i++) { + if (locks[i] == type) + i_fatal("%s: Duplicated value %s", env, *lock); + } + + /* @UNSAFE */ + locks[dest++] = type; + } + locks[dest] = (enum mbox_lock_type)-1; +} + +static void mbox_init_lock_settings(struct mbox_storage *storage) +{ + enum mbox_lock_type read_locks[MBOX_LOCK_COUNT+1]; + enum mbox_lock_type write_locks[MBOX_LOCK_COUNT+1]; + int r, w; + + mbox_read_lock_methods(storage->set->mbox_read_locks, + "mbox_read_locks", read_locks); + mbox_read_lock_methods(storage->set->mbox_write_locks, + "mbox_write_locks", write_locks); + + /* check that read/write list orders match. write_locks must contain + at least read_locks and possibly more. */ + for (r = w = 0; write_locks[w] != (enum mbox_lock_type)-1; w++) { + if (read_locks[r] == (enum mbox_lock_type)-1) + break; + if (read_locks[r] == write_locks[w]) + r++; + } + if (read_locks[r] != (enum mbox_lock_type)-1) { + i_fatal("mbox read/write lock list settings are invalid. " + "Lock ordering must be the same with both, " + "and write locks must contain all read locks " + "(and possibly more)"); + } + + storage->read_locks = p_new(storage->storage.pool, + enum mbox_lock_type, MBOX_LOCK_COUNT+1); + memcpy(storage->read_locks, read_locks, + sizeof(*storage->read_locks) * (MBOX_LOCK_COUNT+1)); + + storage->write_locks = p_new(storage->storage.pool, + enum mbox_lock_type, MBOX_LOCK_COUNT+1); + memcpy(storage->write_locks, write_locks, + sizeof(*storage->write_locks) * (MBOX_LOCK_COUNT+1)); + + storage->lock_settings_initialized = TRUE; +} + +static int mbox_file_open_latest(struct mbox_lock_context *ctx, int lock_type) +{ + struct mbox_mailbox *mbox = ctx->mbox; + struct stat st; + + if (ctx->checked_file || lock_type == F_UNLCK) + return 0; + + if (mbox->mbox_fd != -1) { + /* we could flush NFS file handle cache here if we wanted to + be sure that the file is latest, but mbox files get rarely + deleted and the flushing might cause errors (e.g. EBUSY for + trying to flush a /var/mail mountpoint) */ + if (nfs_safe_stat(mailbox_get_path(&mbox->box), &st) < 0) { + if (errno == ENOENT) + mailbox_set_deleted(&mbox->box); + else + mbox_set_syscall_error(mbox, "stat()"); + return -1; + } + + if (st.st_ino != mbox->mbox_ino || + !CMP_DEV_T(st.st_dev, mbox->mbox_dev)) + mbox_file_close(mbox); + } + + if (mbox->mbox_fd == -1) { + if (mbox_file_open(mbox) < 0) + return -1; + } + + ctx->checked_file = TRUE; + return 0; +} + +static bool dotlock_callback(unsigned int secs_left, bool stale, void *context) +{ + struct mbox_lock_context *ctx = context; + enum mbox_lock_type *lock_types; + int i; + + if (ctx->using_privileges) + restrict_access_drop_priv_gid(); + + if (stale && !ctx->dotlock_last_stale) { + /* get next index we wish to try locking. it's the one after + dotlocking. */ + lock_types = ctx->lock_type == F_WRLCK || + (ctx->lock_type == F_UNLCK && + ctx->mbox->mbox_lock_type == F_WRLCK) ? + ctx->mbox->storage->write_locks : + ctx->mbox->storage->read_locks; + + for (i = 0; lock_types[i] != (enum mbox_lock_type)-1; i++) { + if (lock_types[i] == MBOX_LOCK_DOTLOCK) + break; + } + + if (lock_types[i] != (enum mbox_lock_type)-1 && + lock_types[i+1] != (enum mbox_lock_type)-1) { + i++; + if (mbox_lock_list(ctx, ctx->lock_type, 0, i) <= 0) { + /* we couldn't get fd lock - + it's really locked */ + ctx->dotlock_last_stale = TRUE; + return FALSE; + } + mbox_lock_list(ctx, F_UNLCK, 0, i); + } + } + ctx->dotlock_last_stale = stale; + + index_storage_lock_notify(&ctx->mbox->box, stale ? + MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE : + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + secs_left); + if (ctx->using_privileges) { + if (restrict_access_use_priv_gid() < 0) { + /* shouldn't get here */ + return FALSE; + } + } + return TRUE; +} + +static int ATTR_NULL(2) ATTR_NOWARN_UNUSED_RESULT +mbox_dotlock_privileged_op(struct mbox_mailbox *mbox, + struct dotlock_settings *set, + enum mbox_dotlock_op op) +{ + const char *box_path, *dir, *fname; + int ret = -1, orig_dir_fd, orig_errno; + + orig_dir_fd = open(".", O_RDONLY); + if (orig_dir_fd == -1) { + mailbox_set_critical(&mbox->box, "open(.) failed: %m"); + return -1; + } + + /* allow dotlocks to be created only for files we can read while we're + unprivileged. to make sure there are no race conditions we first + have to chdir to the mbox file's directory and then use relative + paths. unless this is done, users could: + - create *.lock files to any directory writable by the + privileged group + - DoS other users by dotlocking their mailboxes infinitely + */ + box_path = mailbox_get_path(&mbox->box); + fname = strrchr(box_path, '/'); + if (fname == NULL) { + /* already relative */ + fname = box_path; + } else { + dir = t_strdup_until(box_path, fname); + if (chdir(dir) < 0) { + mailbox_set_critical(&mbox->box, + "chdir(%s) failed: %m", dir); + i_close_fd(&orig_dir_fd); + return -1; + } + fname++; + } + if (op == MBOX_DOTLOCK_OP_LOCK) { + if (access(fname, R_OK) < 0) { + mailbox_set_critical(&mbox->box, + "access(%s) failed: %m", box_path); + i_close_fd(&orig_dir_fd); + return -1; + } + } + + if (restrict_access_use_priv_gid() < 0) { + i_close_fd(&orig_dir_fd); + return -1; + } + + switch (op) { + case MBOX_DOTLOCK_OP_LOCK: + /* we're now privileged - avoid doing as much as possible */ + ret = file_dotlock_create(set, fname, 0, &mbox->mbox_dotlock); + if (ret > 0) + mbox->mbox_used_privileges = TRUE; + else if (ret < 0 && errno == EACCES) { + const char *errmsg = + eacces_error_get_creating("file_dotlock_create", + fname); + mailbox_set_critical(&mbox->box, "%s", errmsg); + } else { + mbox_set_syscall_error(mbox, "file_dotlock_create()"); + } + break; + case MBOX_DOTLOCK_OP_UNLOCK: + /* we're now privileged - avoid doing as much as possible */ + ret = file_dotlock_delete(&mbox->mbox_dotlock); + if (ret < 0) + mbox_set_syscall_error(mbox, "file_dotlock_delete()"); + mbox->mbox_used_privileges = FALSE; + break; + case MBOX_DOTLOCK_OP_TOUCH: + ret = file_dotlock_touch(mbox->mbox_dotlock); + if (ret < 0) + mbox_set_syscall_error(mbox, "file_dotlock_touch()"); + break; + } + + orig_errno = errno; + restrict_access_drop_priv_gid(); + + if (fchdir(orig_dir_fd) < 0) { + mailbox_set_critical(&mbox->box, "fchdir() failed: %m"); + } + i_close_fd(&orig_dir_fd); + errno = orig_errno; + return ret; +} + +static void +mbox_dotlock_log_eacces_error(struct mbox_mailbox *mbox, const char *path) +{ + const char *dir, *errmsg, *name; + struct stat st; + struct group group; + int orig_errno = errno; + + errmsg = eacces_error_get_creating("file_dotlock_create", path); + dir = strrchr(path, '/'); + dir = dir == NULL ? "." : t_strdup_until(path, dir); + /* allow privileged locking for + a) user's own INBOX, + b) another user's shared INBOX, and + c) anything called INBOX (in inbox=no namespace) */ + if (!mbox->box.inbox_any && strcmp(mbox->box.name, "INBOX") != 0) { + mailbox_set_critical(&mbox->box, + "%s (not INBOX -> no privileged locking)", errmsg); + } else if (!mbox->mbox_privileged_locking) { + dir = mailbox_list_get_root_forced(mbox->box.list, + MAILBOX_LIST_PATH_TYPE_DIR); + mailbox_set_critical(&mbox->box, + "%s (under root dir %s -> no privileged locking)", + errmsg, dir); + } else if (stat(dir, &st) == 0 && + (st.st_mode & 02) == 0 && /* not world-writable */ + (st.st_mode & 020) != 0) { /* group-writable */ + if (i_getgrgid(st.st_gid, &group) <= 0) + name = dec2str(st.st_gid); + else + name = group.gr_name; + mailbox_set_critical(&mbox->box, + "%s (set mail_privileged_group=%s)", errmsg, name); + } else { + mailbox_set_critical(&mbox->box, + "%s (nonstandard permissions in %s)", errmsg, dir); + } + errno = orig_errno; +} + +static int +mbox_lock_dotlock_int(struct mbox_lock_context *ctx, int lock_type, bool try) +{ + struct mbox_mailbox *mbox = ctx->mbox; + struct dotlock_settings set; + int ret; + + if (lock_type == F_UNLCK) { + if (!mbox->mbox_dotlocked) + return 1; + + if (!mbox->mbox_used_privileges) { + if (file_dotlock_delete(&mbox->mbox_dotlock) <= 0) { + mbox_set_syscall_error(mbox, + "file_dotlock_delete()"); + } + } else { + ctx->using_privileges = TRUE; + mbox_dotlock_privileged_op(mbox, NULL, + MBOX_DOTLOCK_OP_UNLOCK); + ctx->using_privileges = FALSE; + } + mbox->mbox_dotlocked = FALSE; + return 1; + } + + if (mbox->mbox_dotlocked) + return 1; + + ctx->dotlock_last_stale = TRUE; + + i_zero(&set); + set.use_excl_lock = mbox->storage->storage.set->dotlock_use_excl; + set.nfs_flush = mbox->storage->storage.set->mail_nfs_storage; + set.timeout = mail_storage_get_lock_timeout(&mbox->storage->storage, + mbox->storage->set->mbox_lock_timeout); + set.stale_timeout = mbox->storage->set->mbox_dotlock_change_timeout; + set.callback = dotlock_callback; + set.context = ctx; + + ret = file_dotlock_create(&set, mailbox_get_path(&mbox->box), 0, + &mbox->mbox_dotlock); + if (ret >= 0) { + /* success / timeout */ + } else if (errno == EACCES && restrict_access_have_priv_gid() && + mbox->mbox_privileged_locking) { + /* try again, this time with extra privileges */ + ret = mbox_dotlock_privileged_op(mbox, &set, + MBOX_DOTLOCK_OP_LOCK); + } else if (errno == EACCES) + mbox_dotlock_log_eacces_error(mbox, mailbox_get_path(&mbox->box)); + else + mbox_set_syscall_error(mbox, "file_dotlock_create()"); + + if (ret < 0) { + if ((ENOSPACE(errno) || errno == EACCES) && try) + return 1; + return -1; + } + if (ret == 0) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT); + return 0; + } + mbox->mbox_dotlocked = TRUE; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + return 1; +} + +static int mbox_lock_dotlock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time ATTR_UNUSED) +{ + return mbox_lock_dotlock_int(ctx, lock_type, FALSE); +} + +static int mbox_lock_dotlock_try(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time ATTR_UNUSED) +{ + return mbox_lock_dotlock_int(ctx, lock_type, TRUE); +} + +#ifdef HAVE_FLOCK +static int mbox_lock_flock(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time) +{ + time_t now; + unsigned int next_alarm; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + + if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1) + return 1; + + if (lock_type == F_WRLCK) + lock_type = LOCK_EX; + else if (lock_type == F_RDLCK) + lock_type = LOCK_SH; + else + lock_type = LOCK_UN; + + if (max_wait_time == 0) { + /* usually we're waiting here, but if we came from + mbox_lock_dotlock(), we just want to try locking */ + lock_type |= LOCK_NB; + } else { + now = time(NULL); + if (now >= max_wait_time) + alarm(1); + else + alarm(I_MIN(max_wait_time - now, 5)); + } + + while (flock(ctx->mbox->mbox_fd, lock_type) < 0) { + if (errno != EINTR) { + if (errno == EWOULDBLOCK && max_wait_time == 0) { + /* non-blocking lock trying failed */ + return 0; + } + alarm(0); + mbox_set_syscall_error(ctx->mbox, "flock()"); + return -1; + } + + now = time(NULL); + if (now >= max_wait_time) { + alarm(0); + return 0; + } + + /* notify locks once every 5 seconds. + try to use rounded values. */ + next_alarm = (max_wait_time - now) % 5; + if (next_alarm == 0) + next_alarm = 5; + alarm(next_alarm); + + index_storage_lock_notify(&ctx->mbox->box, + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + max_wait_time - now); + } + + alarm(0); + return 1; +} +#endif + +#ifdef HAVE_LOCKF +static int mbox_lock_lockf(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time) +{ + time_t now; + unsigned int next_alarm; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + + if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1) + return 1; + + if (lock_type == F_UNLCK) + lock_type = F_ULOCK; + else if (max_wait_time == 0) { + /* usually we're waiting here, but if we came from + mbox_lock_dotlock(), we just want to try locking */ + lock_type = F_TLOCK; + } else { + now = time(NULL); + if (now >= max_wait_time) + alarm(1); + else + alarm(I_MIN(max_wait_time - now, 5)); + lock_type = F_LOCK; + } + + while (lockf(ctx->mbox->mbox_fd, lock_type, 0) < 0) { + if (errno != EINTR) { + if ((errno == EACCES || errno == EAGAIN) && + max_wait_time == 0) { + /* non-blocking lock trying failed */ + return 0; + } + alarm(0); + mbox_set_syscall_error(ctx->mbox, "lockf()"); + return -1; + } + + now = time(NULL); + if (now >= max_wait_time) { + alarm(0); + return 0; + } + + /* notify locks once every 5 seconds. + try to use rounded values. */ + next_alarm = (max_wait_time - now) % 5; + if (next_alarm == 0) + next_alarm = 5; + alarm(next_alarm); + + index_storage_lock_notify(&ctx->mbox->box, + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + max_wait_time - now); + } + + alarm(0); + return 1; +} +#endif + +static int mbox_lock_fcntl(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time) +{ + struct flock fl; + time_t now; + unsigned int next_alarm; + int wait_type; + + if (mbox_file_open_latest(ctx, lock_type) < 0) + return -1; + + if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1) + return 1; + + i_zero(&fl); + fl.l_type = lock_type; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + + if (max_wait_time == 0) { + /* usually we're waiting here, but if we came from + mbox_lock_dotlock(), we just want to try locking */ + wait_type = F_SETLK; + } else { + wait_type = F_SETLKW; + now = time(NULL); + if (now >= max_wait_time) + alarm(1); + else + alarm(I_MIN(max_wait_time - now, 5)); + } + + while (fcntl(ctx->mbox->mbox_fd, wait_type, &fl) < 0) { + if (errno != EINTR) { + if ((errno == EACCES || errno == EAGAIN) && + wait_type == F_SETLK) { + /* non-blocking lock trying failed */ + return 0; + } + alarm(0); + if (errno != EACCES) { + mbox_set_syscall_error(ctx->mbox, "fcntl()"); + return -1; + } + mailbox_set_critical(&ctx->mbox->box, + "fcntl() failed with mbox file %s: " + "File is locked by another process (EACCES)", + mailbox_get_path(&ctx->mbox->box)); + return -1; + } + + now = time(NULL); + if (now >= max_wait_time) { + alarm(0); + return 0; + } + + /* notify locks once every 5 seconds. + try to use rounded values. */ + next_alarm = (max_wait_time - now) % 5; + if (next_alarm == 0) + next_alarm = 5; + alarm(next_alarm); + + index_storage_lock_notify(&ctx->mbox->box, + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + max_wait_time - now); + } + + alarm(0); + ctx->fcntl_locked = TRUE; + return 1; +} + +static int ATTR_NOWARN_UNUSED_RESULT +mbox_lock_list(struct mbox_lock_context *ctx, int lock_type, + time_t max_wait_time, int idx) +{ + enum mbox_lock_type *lock_types; + enum mbox_lock_type type; + int i, ret = 0; + bool locked_status; + + ctx->lock_type = lock_type; + + lock_types = lock_type == F_WRLCK || + (lock_type == F_UNLCK && ctx->mbox->mbox_lock_type == F_WRLCK) ? + ctx->mbox->storage->write_locks : + ctx->mbox->storage->read_locks; + for (i = idx; lock_types[i] != (enum mbox_lock_type)-1; i++) { + type = lock_types[i]; + locked_status = lock_type != F_UNLCK; + + if (ctx->locked_status[type] == locked_status) + continue; + ctx->locked_status[type] = locked_status; + + ret = lock_data[type].func(ctx, lock_type, max_wait_time); + if (ret <= 0) + break; + } + return ret; +} + +static int mbox_update_locking(struct mbox_mailbox *mbox, int lock_type, + bool *fcntl_locked_r) +{ + struct mbox_lock_context ctx; + time_t max_wait_time; + int ret, i; + bool drop_locks; + + *fcntl_locked_r = FALSE; + + index_storage_lock_notify_reset(&mbox->box); + + if (!mbox->storage->lock_settings_initialized) + mbox_init_lock_settings(mbox->storage); + + if (mbox->mbox_fd == -1 && mbox->mbox_file_stream != NULL) { + /* read-only mbox stream. no need to lock. */ + i_assert(mbox_is_backend_readonly(mbox)); + mbox->mbox_lock_type = lock_type; + return 1; + } + + max_wait_time = time(NULL) + + mail_storage_get_lock_timeout(&mbox->storage->storage, + mbox->storage->set->mbox_lock_timeout); + + i_zero(&ctx); + ctx.mbox = mbox; + + if (mbox->mbox_lock_type == F_WRLCK) { + /* dropping to shared lock. first drop those that we + don't remove completely. */ + const enum mbox_lock_type *read_locks = + mbox->storage->read_locks; + + for (i = 0; i < MBOX_LOCK_COUNT; i++) + ctx.locked_status[i] = TRUE; + for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++) + ctx.locked_status[read_locks[i]] = FALSE; + drop_locks = TRUE; + } else { + drop_locks = FALSE; + } + + mbox->mbox_lock_type = lock_type; + ret = mbox_lock_list(&ctx, lock_type, max_wait_time, 0); + if (ret <= 0) { + if (!drop_locks) + mbox_unlock_files(&ctx); + if (ret == 0) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT); + } + return ret; + } + + if (drop_locks) { + /* dropping to shared lock: drop the locks that are only + in write list */ + const enum mbox_lock_type *read_locks = + mbox->storage->read_locks; + const enum mbox_lock_type *write_locks = + mbox->storage->write_locks; + + memset(ctx.locked_status, 0, sizeof(ctx.locked_status)); + for (i = 0; write_locks[i] != (enum mbox_lock_type)-1; i++) + ctx.locked_status[write_locks[i]] = TRUE; + for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++) + ctx.locked_status[read_locks[i]] = FALSE; + + mbox->mbox_lock_type = F_WRLCK; + mbox_lock_list(&ctx, F_UNLCK, 0, 0); + mbox->mbox_lock_type = F_RDLCK; + } + + *fcntl_locked_r = ctx.fcntl_locked; + return 1; +} + +int mbox_lock(struct mbox_mailbox *mbox, int lock_type, + unsigned int *lock_id_r) +{ + const char *path = mailbox_get_path(&mbox->box); + int mbox_fd = mbox->mbox_fd; + bool fcntl_locked; + int ret; + + if (lock_type == F_RDLCK && mbox->external_transactions > 0 && + mbox->mbox_lock_type != F_RDLCK) { + /* we have a transaction open that is going to save mails + and apparently also wants to read from the same mailbox + (copy, move, catenate). we need to write lock the mailbox, + since we can't later upgrade a read lock to write lock. */ + lock_type = F_WRLCK; + } + + /* allow only unlock -> shared/exclusive or exclusive -> shared */ + i_assert(lock_type == F_RDLCK || lock_type == F_WRLCK); + i_assert(lock_type == F_RDLCK || mbox->mbox_lock_type != F_RDLCK); + + if (mbox->mbox_lock_type == F_UNLCK) { + ret = mbox_update_locking(mbox, lock_type, &fcntl_locked); + if (ret <= 0) + return ret; + + if (mbox->storage->storage.set->mail_nfs_storage) { + if (fcntl_locked) { + nfs_flush_attr_cache_fd_locked(path, mbox_fd); + nfs_flush_read_cache_locked(path, mbox_fd); + } else { + nfs_flush_attr_cache_unlocked(path); + nfs_flush_read_cache_unlocked(path, mbox_fd); + } + } + + mbox->mbox_lock_id += 2; + } + + if (lock_type == F_RDLCK) { + mbox->mbox_shared_locks++; + *lock_id_r = mbox->mbox_lock_id; + } else { + mbox->mbox_excl_locks++; + *lock_id_r = mbox->mbox_lock_id + 1; + } + if (mbox->mbox_stream != NULL) + istream_raw_mbox_set_locked(mbox->mbox_stream); + return 1; +} + +static int mbox_unlock_files(struct mbox_lock_context *ctx) +{ + int ret = 0; + + if (mbox_lock_list(ctx, F_UNLCK, 0, 0) < 0) + ret = -1; + + ctx->mbox->mbox_lock_id += 2; + ctx->mbox->mbox_lock_type = F_UNLCK; + return ret; +} + +int mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id) +{ + struct mbox_lock_context ctx; + bool fcntl_locked; + int i; + + i_assert(mbox->mbox_lock_id == (lock_id & ~1U)); + + if ((lock_id & 1) != 0) { + /* dropping exclusive lock */ + i_assert(mbox->mbox_excl_locks > 0); + if (--mbox->mbox_excl_locks > 0) + return 0; + if (mbox->mbox_shared_locks > 0) { + /* drop to shared lock */ + if (mbox_update_locking(mbox, F_RDLCK, + &fcntl_locked) < 0) + return -1; + return 0; + } + } else { + /* dropping shared lock */ + i_assert(mbox->mbox_shared_locks > 0); + if (--mbox->mbox_shared_locks > 0) + return 0; + if (mbox->mbox_excl_locks > 0) + return 0; + } + /* all locks gone */ + + /* make sure we don't read the stream while unlocked */ + if (mbox->mbox_stream != NULL) + istream_raw_mbox_set_unlocked(mbox->mbox_stream); + + i_zero(&ctx); + ctx.mbox = mbox; + + for (i = 0; i < MBOX_LOCK_COUNT; i++) + ctx.locked_status[i] = TRUE; + + return mbox_unlock_files(&ctx); +} + +unsigned int mbox_get_cur_lock_id(struct mbox_mailbox *mbox) +{ + return mbox->mbox_lock_id + + (mbox->mbox_excl_locks > 0 ? 1 : 0); +} + +void mbox_dotlock_touch(struct mbox_mailbox *mbox) +{ + if (mbox->mbox_dotlock == NULL) + return; + + if (!mbox->mbox_used_privileges) + (void)file_dotlock_touch(mbox->mbox_dotlock); + else { + mbox_dotlock_privileged_op(mbox, NULL, + MBOX_DOTLOCK_OP_TOUCH); + } +} diff --git a/src/lib-storage/index/mbox/mbox-lock.h b/src/lib-storage/index/mbox/mbox-lock.h new file mode 100644 index 0000000..2175908 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-lock.h @@ -0,0 +1,15 @@ +#ifndef MBOX_LOCK_H +#define MBOX_LOCK_H + +/* NOTE: if mbox file is not open, it's opened. if it is open but file has + been overwritten (ie. inode has changed), it's reopened. */ +int mbox_lock(struct mbox_mailbox *mbox, int lock_type, + unsigned int *lock_id_r); +int ATTR_NOWARN_UNUSED_RESULT +mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id); + +unsigned int mbox_get_cur_lock_id(struct mbox_mailbox *mbox); + +void mbox_dotlock_touch(struct mbox_mailbox *mbox); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-mail.c b/src/lib-storage/index/mbox/mbox-mail.c new file mode 100644 index 0000000..09223b0 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-mail.c @@ -0,0 +1,439 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "hex-binary.h" +#include "index-mail.h" +#include "mbox-storage.h" +#include "mbox-file.h" +#include "mbox-lock.h" +#include "mbox-sync-private.h" +#include "istream-raw-mbox.h" +#include "istream-header-filter.h" + +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> + +static void mbox_prepare_resync(struct mail *mail) +{ + struct mbox_transaction_context *t = MBOX_TRANSCTX(mail->transaction); + struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->box); + + if (mbox->mbox_lock_type == F_RDLCK) { + if (mbox->mbox_lock_id == t->read_lock_id) + t->read_lock_id = 0; + mbox_unlock(mbox, mbox->mbox_lock_id); + i_assert(mbox->mbox_lock_type == F_UNLCK); + } +} + +static int mbox_mail_seek(struct index_mail *mail) +{ + struct mail *_mail = &mail->mail.mail; + struct mbox_transaction_context *t = MBOX_TRANSCTX(_mail->transaction); + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + enum mbox_sync_flags sync_flags = 0; + int ret, try; + bool deleted; + + if (_mail->expunged || mbox->syncing) + return -1; + + if (!mail_stream_access_start(_mail)) + return -1; + + if (mbox->mbox_stream != NULL && + istream_raw_mbox_is_corrupted(mbox->mbox_stream)) { + /* clear the corruption by forcing a full resync */ + sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC; + } + + for (try = 0; try < 2; try++) { + if ((sync_flags & MBOX_SYNC_FORCE_SYNC) != 0) { + /* dirty offsets are broken. make sure we can sync. */ + mbox_prepare_resync(_mail); + } + if (mbox->mbox_lock_type == F_UNLCK) { + i_assert(t->read_lock_id == 0); + sync_flags |= MBOX_SYNC_LOCK_READING; + if (mbox_sync(mbox, sync_flags) < 0) + return -1; + t->read_lock_id = mbox_get_cur_lock_id(mbox); + i_assert(t->read_lock_id != 0); + + /* refresh index file after mbox has been locked to + make sure we get only up-to-date mbox offsets. */ + if (mail_index_refresh(mbox->box.index) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + i_assert(mbox->mbox_lock_type != F_UNLCK); + } else if (t->read_lock_id == 0) { + /* file is already locked by another transaction, but + we must keep it locked for the entire transaction, + so increase the lock counter. */ + if (mbox_lock(mbox, mbox->mbox_lock_type, + &t->read_lock_id) < 0) + i_unreached(); + } + + if (mbox_file_open_stream(mbox) < 0) + return -1; + + ret = mbox_file_seek(mbox, _mail->transaction->view, + _mail->seq, &deleted); + if (ret > 0) { + /* success */ + break; + } + if (ret < 0) { + if (deleted) + mail_set_expunged(_mail); + return -1; + } + + /* we'll need to re-sync it completely */ + sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC; + } + if (ret == 0) { + mail_set_critical(_mail, "mbox: Losing sync"); + } + return 0; +} + +static int mbox_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (mbox_mail_seek(mail) < 0) + return -1; + data->received_date = + istream_raw_mbox_get_received_time(mbox->mbox_stream); + if (data->received_date == (time_t)-1) { + /* it's broken and conflicts with our "not found" + return value. change it. */ + data->received_date = 0; + } + + *date_r = data->received_date; + return 0; +} + +static int mbox_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + + if (index_mail_get_save_date(_mail, date_r) > 0) + return 0; + + /* no way to know this. save the current time into cache and use + that from now on. this works only as long as the index files + are permanent */ + data->save_date = ioloop_time; + *date_r = data->save_date; + return 0; +} + +static int +mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r) +{ + struct mail *_mail = &mail->mail.mail; + static uint8_t empty_md5[16] = + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + const void *ext_data; + + if (mail->data.guid != NULL) { + *value_r = mail->data.guid; + return 1; + } + + mail_index_lookup_ext(_mail->transaction->view, _mail->seq, + mbox->md5hdr_ext_idx, &ext_data, NULL); + if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) { + mail->data.guid = p_strdup(mail->mail.data_pool, + binary_to_hex(ext_data, 16)); + *value_r = mail->data.guid; + return 1; + } else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) { + mail_set_expunged(_mail); + return -1; + } else { + return 0; + } +} + +static int +mbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + uoff_t offset; + bool move_offset; + int ret; + + switch (field) { + case MAIL_FETCH_FROM_ENVELOPE: + if (mbox_mail_seek(mail) < 0) + return -1; + + *value_r = istream_raw_mbox_get_sender(mbox->mbox_stream); + return 0; + case MAIL_FETCH_GUID: + case MAIL_FETCH_HEADER_MD5: + if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0) + return ret < 0 ? -1 : 0; + + /* i guess in theory the empty_md5 is valid and can happen, + but it's almost guaranteed that it means the MD5 sum is + missing. recalculate it. */ + if (mbox->mbox_lock_type == F_UNLCK || + mbox->mbox_stream == NULL) { + offset = 0; + move_offset = FALSE; + } else { + offset = istream_raw_mbox_get_start_offset(mbox->mbox_stream); + move_offset = TRUE; + } + mbox->mbox_save_md5 = TRUE; + if (mbox_sync(mbox, MBOX_SYNC_FORCE_SYNC | + MBOX_SYNC_READONLY) < 0) + return -1; + if (move_offset) { + if (istream_raw_mbox_seek(mbox->mbox_stream, + offset) < 0) { + i_error("mbox %s sync lost during MD5 syncing", + _mail->box->name); + return -1; + } + } + + if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) { + i_error("mbox %s resyncing didn't save header MD5 values", + _mail->box->name); + return -1; + } + return ret < 0 ? -1 : 0; + default: + break; + } + + return index_mail_get_special(_mail, field, value_r); +} + +static int +mbox_mail_get_next_offset(struct index_mail *mail, uoff_t *next_offset_r) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->mail.mail.box); + struct mail_index_view *view; + const struct mail_index_header *hdr; + uint32_t seq; + int trailer_size; + int ret = 1; + + *next_offset_r = UOFF_T_MAX; + + hdr = mail_index_get_header(mail->mail.mail.transaction->view); + if (mail->mail.mail.seq > hdr->messages_count) { + /* we're appending a new message */ + return 0; + } + + /* We can't really trust trans_view. The next message may already be + expunged from it. Also hdr.messages_count may be incorrect there. + So refresh the index to get the latest changes and get the next + message's offset using a new view. */ + i_assert(mbox->mbox_lock_type != F_UNLCK); + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + + view = mail_index_view_open(mail->mail.mail.box->index); + hdr = mail_index_get_header(view); + if (!mail_index_lookup_seq(view, mail->mail.mail.uid, &seq)) + i_panic("Message unexpectedly expunged from index"); + + if (seq < hdr->messages_count) { + if (mbox_file_lookup_offset(mbox, view, seq + 1, + next_offset_r) <= 0) + ret = -1; + } else if (mail->mail.mail.box->input != NULL) { + /* opened the mailbox as input stream. we can't trust the + sync_size, since it's wrong with compressed mailboxes */ + ret = 0; + } else { + /* last message, use the synced mbox size */ + trailer_size = + mbox->storage->storage.set->mail_save_crlf ? 2 : 1; + *next_offset_r = mbox->mbox_hdr.sync_size - trailer_size; + } + mail_index_view_close(&view); + return ret; +} + +static int mbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box); + struct istream *input; + struct message_size hdr_size; + uoff_t old_offset, body_offset, body_size, next_offset; + + if (index_mail_get_physical_size(_mail, size_r) == 0) + return 0; + + /* we want to return the header size as seen by mail_get_stream(). */ + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream(_mail, &hdr_size, NULL, &input) < 0) + return -1; + + /* our header size varies, so don't do any caching */ + if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) { + mail_set_critical(_mail, "mbox: Couldn't get body offset"); + return -1; + } + + /* use the next message's offset to avoid reading through the entire + message body to find out its size */ + if (mbox_mail_get_next_offset(mail, &next_offset) > 0) + body_size = next_offset - body_offset; + else + body_size = UOFF_T_MAX; + + /* verify that the calculated body size is correct */ + if (istream_raw_mbox_get_body_size(mbox->mbox_stream, + body_size, &body_size) < 0) { + mail_set_critical(_mail, "mbox: Couldn't get body size"); + return -1; + } + + data->physical_size = hdr_size.physical_size + body_size; + *size_r = data->physical_size; + + i_stream_seek(input, old_offset); + return 0; +} + +static int mbox_mail_init_stream(struct index_mail *mail) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->mail.mail.box); + struct istream *raw_stream; + uoff_t hdr_offset, next_offset; + int ret; + + if (mbox_mail_seek(mail) < 0) + return -1; + + ret = mbox_mail_get_next_offset(mail, &next_offset); + if (ret < 0) { + if (mbox_mail_seek(mail) < 0) + return -1; + ret = mbox_mail_get_next_offset(mail, &next_offset); + if (ret < 0) { + i_warning("mbox %s: Can't find next message offset " + "for uid=%u", mailbox_get_path(&mbox->box), + mail->mail.mail.uid); + } + } + + raw_stream = mbox->mbox_stream; + if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) { + mail_set_critical(&mail->mail.mail, + "mbox: Couldn't get header offset"); + return -1; + } + i_stream_seek(raw_stream, hdr_offset); + + if (next_offset != UOFF_T_MAX) + istream_raw_mbox_set_next_offset(raw_stream, next_offset); + + raw_stream = i_stream_create_limit(raw_stream, UOFF_T_MAX); + mail->data.stream = + i_stream_create_header_filter(raw_stream, + HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, + mbox_hide_headers, mbox_hide_headers_count, + *null_header_filter_callback, NULL); + i_stream_unref(&raw_stream); + return 0; +} + +static int mbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + if (mail->data.stream == NULL) { + if (mbox_mail_init_stream(mail) < 0) + return -1; + } + + return index_mail_init_stream(mail, hdr_size, body_size, stream_r); +} + +static void mbox_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + + index_mail_set_seq(_mail, seq, saving); + mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE; +} + +static bool mbox_mail_set_uid(struct mail *_mail, uint32_t uid) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + bool ret; + + ret = index_mail_set_uid(_mail, uid); + mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE; + return ret; +} + +struct mail_vfuncs mbox_mail_vfuncs = { + index_mail_close, + index_mail_free, + mbox_mail_set_seq, + mbox_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + mbox_mail_get_received_date, + mbox_mail_get_save_date, + index_mail_get_virtual_size, + mbox_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + mbox_mail_get_stream, + index_mail_get_binary_stream, + mbox_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/mbox/mbox-md5-all.c b/src/lib-storage/index/mbox/mbox-md5-all.c new file mode 100644 index 0000000..9a09fb2 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-md5-all.c @@ -0,0 +1,39 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "md5.h" +#include "message-parser.h" +#include "mbox-md5.h" + + +struct mbox_md5_context { + struct md5_context hdr_md5_ctx; +}; + +static struct mbox_md5_context *mbox_md5_all_init(void) +{ + struct mbox_md5_context *ctx; + + ctx = i_new(struct mbox_md5_context, 1); + md5_init(&ctx->hdr_md5_ctx); + return ctx; +} + +static void mbox_md5_all_more(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); +} + +static void mbox_md5_all_finish(struct mbox_md5_context *ctx, + unsigned char result[STATIC_ARRAY 16]) +{ + md5_final(&ctx->hdr_md5_ctx, result); + i_free(ctx); +} + +struct mbox_md5_vfuncs mbox_md5_all = { + mbox_md5_all_init, + mbox_md5_all_more, + mbox_md5_all_finish +}; diff --git a/src/lib-storage/index/mbox/mbox-md5-apop3d.c b/src/lib-storage/index/mbox/mbox-md5-apop3d.c new file mode 100644 index 0000000..56f6f91 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-md5-apop3d.c @@ -0,0 +1,119 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "md5.h" +#include "message-parser.h" +#include "mbox-md5.h" + + +struct mbox_md5_context { + struct md5_context hdr_md5_ctx; + bool seen_received_hdr; +}; + +struct mbox_md5_header_func { + const char *header; + bool (*func)(struct mbox_md5_context *ctx, + struct message_header_line *hdr); +}; + +static bool parse_date(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + if (!ctx->seen_received_hdr) { + /* Received-header contains date too, and more trusted one */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + } + return TRUE; +} + +static bool parse_delivered_to(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + return TRUE; +} + +static bool parse_message_id(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + if (!ctx->seen_received_hdr) { + /* Received-header contains unique ID too, + and more trusted one */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + } + return TRUE; +} + +static bool parse_received(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + if (!ctx->seen_received_hdr) { + /* get only the first received-header */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + if (!hdr->continues) + ctx->seen_received_hdr = TRUE; + } + return TRUE; +} + +static bool parse_x_delivery_id(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + /* Let the local delivery agent help generate unique ID's but don't + blindly trust this header alone as it could just as easily come from + the remote. */ + md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len); + return TRUE; +} + + +static struct mbox_md5_header_func md5_header_funcs[] = { + { "Date", parse_date }, + { "Delivered-To", parse_delivered_to }, + { "Message-ID", parse_message_id }, + { "Received", parse_received }, + { "X-Delivery-ID", parse_x_delivery_id } +}; + +static int bsearch_header_func_cmp(const void *p1, const void *p2) +{ + const char *key = p1; + const struct mbox_md5_header_func *func = p2; + + return strcasecmp(key, func->header); +} + +static struct mbox_md5_context *mbox_md5_apop3d_init(void) +{ + struct mbox_md5_context *ctx; + + ctx = i_new(struct mbox_md5_context, 1); + md5_init(&ctx->hdr_md5_ctx); + return ctx; +} + +static void mbox_md5_apop3d_more(struct mbox_md5_context *ctx, + struct message_header_line *hdr) +{ + struct mbox_md5_header_func *func; + + func = bsearch(hdr->name, md5_header_funcs, + N_ELEMENTS(md5_header_funcs), sizeof(*md5_header_funcs), + bsearch_header_func_cmp); + if (func != NULL) + (void)func->func(ctx, hdr); +} + +static void mbox_md5_apop3d_finish(struct mbox_md5_context *ctx, + unsigned char result[STATIC_ARRAY 16]) +{ + md5_final(&ctx->hdr_md5_ctx, result); + i_free(ctx); +} + +struct mbox_md5_vfuncs mbox_md5_apop3d = { + mbox_md5_apop3d_init, + mbox_md5_apop3d_more, + mbox_md5_apop3d_finish +}; diff --git a/src/lib-storage/index/mbox/mbox-md5.h b/src/lib-storage/index/mbox/mbox-md5.h new file mode 100644 index 0000000..7584052 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-md5.h @@ -0,0 +1,17 @@ +#ifndef MBOX_MD5_H +#define MBOX_MD5_H + +struct message_header_line; + +struct mbox_md5_vfuncs { + struct mbox_md5_context *(*init)(void); + void (*more)(struct mbox_md5_context *ctx, + struct message_header_line *hdr); + void (*finish)(struct mbox_md5_context *ctx, + unsigned char result[STATIC_ARRAY 16]); +}; + +extern struct mbox_md5_vfuncs mbox_md5_apop3d; +extern struct mbox_md5_vfuncs mbox_md5_all; + +#endif diff --git a/src/lib-storage/index/mbox/mbox-save.c b/src/lib-storage/index/mbox/mbox-save.c new file mode 100644 index 0000000..2fb3e19 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-save.c @@ -0,0 +1,833 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "base64.h" +#include "hostpid.h" +#include "randgen.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "write-full.h" +#include "istream-header-filter.h" +#include "istream-crlf.h" +#include "istream-concat.h" +#include "message-parser.h" +#include "mail-user.h" +#include "index-mail.h" +#include "mbox-storage.h" +#include "mbox-file.h" +#include "mbox-from.h" +#include "mbox-lock.h" +#include "mbox-md5.h" +#include "mbox-sync-private.h" + +#include <stddef.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <utime.h> + +#define MBOX_DELIVERY_ID_RAND_BYTES (64/8) + +struct mbox_save_context { + struct mail_save_context ctx; + + struct mbox_mailbox *mbox; + struct mail_index_transaction *trans; + uoff_t append_offset, mail_offset; + time_t orig_atime; + + string_t *headers; + size_t space_end_idx; + uint32_t seq, next_uid, uid_validity; + + struct istream *input; + struct ostream *output; + uoff_t extra_hdr_offset, eoh_offset; + char last_char; + + struct mbox_md5_context *mbox_md5_ctx; + char *x_delivery_id_header; + + bool synced:1; + bool failed:1; + bool finished:1; +}; + +#define MBOX_SAVECTX(s) container_of(s, struct mbox_save_context, ctx) + +static void ostream_error(struct mbox_save_context *ctx, const char *func) +{ + mbox_ostream_set_syscall_error(ctx->mbox, ctx->output, func); + ctx->failed = TRUE; +} + +static void write_stream_error(struct mbox_save_context *ctx) +{ + ostream_error(ctx, "write()"); +} + +static void lseek_stream_error(struct mbox_save_context *ctx) +{ + ostream_error(ctx, "o_stream_seek()"); +} + +static int mbox_seek_to_end(struct mbox_save_context *ctx, uoff_t *offset) +{ + struct stat st; + char ch; + int fd; + + if (ctx->mbox->mbox_writeonly) { + *offset = 0; + return 0; + } + + fd = ctx->mbox->mbox_fd; + if (fstat(fd, &st) < 0) { + mbox_set_syscall_error(ctx->mbox, "fstat()"); + return -1; + } + + ctx->orig_atime = st.st_atime; + + *offset = (uoff_t)st.st_size; + if (st.st_size == 0) + return 0; + + if (lseek(fd, st.st_size-1, SEEK_SET) < 0) { + mbox_set_syscall_error(ctx->mbox, "lseek()"); + return -1; + } + + if (read(fd, &ch, 1) != 1) { + mbox_set_syscall_error(ctx->mbox, "read()"); + return -1; + } + + if (ch != '\n') { + if (write_full(fd, "\n", 1) < 0) { + mbox_set_syscall_error(ctx->mbox, "write()"); + return -1; + } + *offset += 1; + } + + return 0; +} + +static int mbox_append_lf(struct mbox_save_context *ctx) +{ + if (o_stream_send(ctx->output, "\n", 1) < 0) { + write_stream_error(ctx); + return -1; + } + + return 0; +} + +static int write_from_line(struct mbox_save_context *ctx, time_t received_date, + const char *from_envelope) +{ + int ret; + + T_BEGIN { + const char *line; + + if (from_envelope == NULL) { + struct mail_storage *storage = + &ctx->mbox->storage->storage; + + from_envelope = + strchr(storage->user->username, '@') != NULL ? + storage->user->username : + t_strconcat(storage->user->username, + "@", my_hostdomain(), NULL); + } else if (*from_envelope == '\0') { + /* can't write empty envelope */ + from_envelope = "MAILER-DAEMON"; + } + + /* save in local timezone, no matter what it was given with */ + line = mbox_from_create(from_envelope, received_date); + + if ((ret = o_stream_send_str(ctx->output, line)) < 0) + write_stream_error(ctx); + } T_END; + return ret; +} + +static int mbox_write_content_length(struct mbox_save_context *ctx) +{ + uoff_t end_offset; + const char *str; + size_t len; + + i_assert(ctx->eoh_offset != UOFF_T_MAX); + + if (ctx->mbox->mbox_writeonly) { + /* we can't seek, don't set Content-Length */ + return 0; + } + + end_offset = ctx->output->offset; + + /* write Content-Length headers */ + str = t_strdup_printf("\nContent-Length: %s", + dec2str(end_offset - ctx->eoh_offset)); + len = strlen(str); + + /* flush manually here so that we don't confuse seek() errors with + buffer flushing errors */ + if (o_stream_flush(ctx->output) < 0) { + write_stream_error(ctx); + return -1; + } + if (o_stream_seek(ctx->output, ctx->extra_hdr_offset + + ctx->space_end_idx - len) < 0) { + lseek_stream_error(ctx); + return -1; + } + + if (o_stream_send(ctx->output, str, len) < 0 || + o_stream_flush(ctx->output) < 0) { + write_stream_error(ctx); + return -1; + } + + if (o_stream_seek(ctx->output, end_offset) < 0) { + lseek_stream_error(ctx); + return -1; + } + return 0; +} + +static void mbox_save_init_sync(struct mailbox_transaction_context *t) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(t->box); + struct mbox_save_context *ctx = MBOX_SAVECTX(t->save_ctx); + const struct mail_index_header *hdr; + struct mail_index_view *view; + + /* open a new view to get the header. this is required if we just + synced the mailbox so we can get updated next_uid. */ + mail_index_refresh(mbox->box.index); + view = mail_index_view_open(mbox->box.index); + hdr = mail_index_get_header(view); + + ctx->next_uid = hdr->next_uid; + ctx->uid_validity = hdr->uid_validity; + ctx->synced = TRUE; + + mail_index_view_close(&view); +} + +static void status_flags_append(string_t *str, enum mail_flags flags, + const struct mbox_flag_type *flags_list) +{ + int i; + + flags ^= MBOX_NONRECENT_KLUDGE; + for (i = 0; flags_list[i].chr != 0; i++) { + if ((flags & flags_list[i].flag) != 0) + str_append_c(str, flags_list[i].chr); + } +} + +static void mbox_save_append_flag_headers(string_t *str, enum mail_flags flags) +{ + /* write the Status: header always. It always gets added soon anyway. */ + str_append(str, "Status: "); + status_flags_append(str, flags, mbox_status_flags); + str_append_c(str, '\n'); + + if ((flags & XSTATUS_FLAGS_MASK) != 0) { + str_append(str, "X-Status: "); + status_flags_append(str, flags, mbox_xstatus_flags); + str_append_c(str, '\n'); + } +} + +static void +mbox_save_append_keyword_headers(struct mbox_save_context *ctx, + struct mail_keywords *keywords) +{ + unsigned char space[MBOX_HEADER_PADDING+1 + + sizeof("Content-Length: \n")-1 + MAX_INT_STRLEN]; + const ARRAY_TYPE(keywords) *keyword_names_list; + const char *const *keyword_names; + unsigned int i, count, keyword_names_count; + + keyword_names_list = mail_index_get_keywords(ctx->mbox->box.index); + keyword_names = array_get(keyword_names_list, &keyword_names_count); + + str_append(ctx->headers, "X-Keywords:"); + count = keywords == NULL ? 0 : keywords->count; + for (i = 0; i < count; i++) { + i_assert(keywords->idx[i] < keyword_names_count); + + str_append_c(ctx->headers, ' '); + str_append(ctx->headers, keyword_names[keywords->idx[i]]); + } + + memset(space, ' ', sizeof(space)); + str_append_data(ctx->headers, space, sizeof(space)); + ctx->space_end_idx = str_len(ctx->headers); + str_append_c(ctx->headers, '\n'); +} + +static int +mbox_save_init_file(struct mbox_save_context *ctx, + struct mbox_transaction_context *t) +{ + struct mailbox_transaction_context *_t = &t->t; + struct mbox_mailbox *mbox = ctx->mbox; + struct mail_storage *storage = &mbox->storage->storage; + int ret; + + if (mbox_is_backend_readonly(ctx->mbox)) { + mail_storage_set_error(storage, MAIL_ERROR_PERM, + "Read-only mbox"); + return -1; + } + + if (ctx->append_offset == UOFF_T_MAX) { + /* first appended mail in this transaction */ + if (t->write_lock_id == 0) { + if (mbox_lock(mbox, F_WRLCK, &t->write_lock_id) <= 0) + return -1; + } + + if (mbox->mbox_fd == -1) { + if (mbox_file_open(mbox) < 0) + return -1; + } + + /* update mbox_sync_dirty state */ + ret = mbox_sync_has_changed(mbox, TRUE); + if (ret < 0) + return -1; + } + + if (!ctx->synced) { + /* we'll need to assign UID for the mail immediately. */ + if (mbox_sync(mbox, 0) < 0) + return -1; + mbox_save_init_sync(_t); + } + + /* the syncing above could have changed the append offset */ + if (ctx->append_offset == UOFF_T_MAX) { + if (mbox_seek_to_end(ctx, &ctx->append_offset) < 0) + return -1; + + i_assert(mbox->mbox_fd != -1); + ctx->output = o_stream_create_fd_file(mbox->mbox_fd, + ctx->append_offset, + FALSE); + o_stream_cork(ctx->output); + } + return 0; +} + +static void +save_header_callback(struct header_filter_istream *input ATTR_UNUSED, + struct message_header_line *hdr, + bool *matched, struct mbox_save_context *ctx) +{ + if (hdr != NULL) { + if (str_begins(hdr->name, "From ")) { + /* we can't allow From_-lines in headers. there's no + legitimate reason for allowing them in any case, + so just drop them. */ + *matched = TRUE; + return; + } + + if (!*matched && ctx->mbox_md5_ctx != NULL) + ctx->mbox->md5_v.more(ctx->mbox_md5_ctx, hdr); + } +} + +static void mbox_save_x_delivery_id(struct mbox_save_context *ctx) +{ + unsigned char md5_result[MD5_RESULTLEN]; + buffer_t *buf; + string_t *str; + void *randbuf; + + buf = t_buffer_create(256); + buffer_append(buf, &ioloop_time, sizeof(ioloop_time)); + buffer_append(buf, &ioloop_timeval.tv_usec, + sizeof(ioloop_timeval.tv_usec)); + + randbuf = buffer_append_space_unsafe(buf, MBOX_DELIVERY_ID_RAND_BYTES); + random_fill(randbuf, MBOX_DELIVERY_ID_RAND_BYTES); + + md5_get_digest(buf->data, buf->used, md5_result); + + str = t_str_new(128); + str_append(str, "X-Delivery-ID: "); + base64_encode(md5_result, sizeof(md5_result), str); + str_append_c(str, '\n'); + + ctx->x_delivery_id_header = i_strdup(str_c(str)); +} + +static struct istream * +mbox_save_get_input_stream(struct mbox_save_context *ctx, struct istream *input) +{ + struct istream *filter, *ret, *cache_input, *streams[3]; + + /* filter out unwanted headers and keep track of headers' MD5 sum */ + filter = i_stream_create_header_filter(input, HEADER_FILTER_EXCLUDE | + HEADER_FILTER_NO_CR | + HEADER_FILTER_ADD_MISSING_EOH | + HEADER_FILTER_END_BODY_WITH_LF, + mbox_save_drop_headers, + mbox_save_drop_headers_count, + save_header_callback, ctx); + + if ((ctx->mbox->storage->storage.flags & + MAIL_STORAGE_FLAG_KEEP_HEADER_MD5) != 0) { + /* we're using MD5 sums to generate POP3 UIDLs. + clients don't like it much if there are duplicates, + so make sure that there can't be any by appending + our own X-Delivery-ID header. */ + const char *hdr; + + T_BEGIN { + mbox_save_x_delivery_id(ctx); + } T_END; + hdr = ctx->x_delivery_id_header; + + streams[0] = i_stream_create_from_data(hdr, strlen(hdr)); + streams[1] = filter; + streams[2] = NULL; + ret = i_stream_create_concat(streams); + i_stream_unref(&filter); + filter = ret; + } + + /* convert linefeeds to wanted format */ + ret = ctx->mbox->storage->storage.set->mail_save_crlf ? + i_stream_create_crlf(filter) : i_stream_create_lf(filter); + i_stream_unref(&filter); + + /* caching creates a tee stream */ + cache_input = index_mail_cache_parse_init(ctx->ctx.dest_mail, ret); + i_stream_unref(&ret); + ret = cache_input; + return ret; +} + +struct mail_save_context * +mbox_save_alloc(struct mailbox_transaction_context *t) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(t->box); + struct mbox_save_context *ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx == NULL) { + ctx = i_new(struct mbox_save_context, 1); + ctx->ctx.transaction = t; + ctx->mbox = mbox; + ctx->trans = t->itrans; + ctx->append_offset = UOFF_T_MAX; + ctx->headers = str_new(default_pool, 512); + ctx->mail_offset = UOFF_T_MAX; + t->save_ctx = &ctx->ctx; + } + return t->save_ctx; +} + +int mbox_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + struct mail_save_data *mdata = &_ctx->data; + struct mbox_transaction_context *t = MBOX_TRANSCTX(_ctx->transaction); + enum mail_flags save_flags; + uint64_t offset; + + /* FIXME: we could write timezone_offset to From-line.. */ + if (mdata->received_date == (time_t)-1) + mdata->received_date = ioloop_time; + + ctx->failed = FALSE; + ctx->seq = 0; + + if (mbox_save_init_file(ctx, t) < 0) { + ctx->failed = TRUE; + return -1; + } + + save_flags = mdata->flags; + if (mdata->uid == 0) + save_flags |= MAIL_RECENT; + str_truncate(ctx->headers, 0); + if (ctx->synced) { + if (ctx->mbox->mbox_save_md5) + ctx->mbox_md5_ctx = ctx->mbox->md5_v.init(); + if (ctx->next_uid < mdata->uid) { + /* we can use the wanted UID */ + ctx->next_uid = mdata->uid; + } + if (ctx->output->offset == 0) { + /* writing the first mail. Insert X-IMAPbase as well. */ + str_printfa(ctx->headers, "X-IMAPbase: %u %010u\n", + ctx->uid_validity, ctx->next_uid); + } + str_printfa(ctx->headers, "X-UID: %u\n", ctx->next_uid); + + mail_index_append(ctx->trans, ctx->next_uid, &ctx->seq); + mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE, + save_flags & ENUM_NEGATE(MAIL_RECENT)); + if (mdata->keywords != NULL) { + mail_index_update_keywords(ctx->trans, ctx->seq, + MODIFY_REPLACE, + mdata->keywords); + } + if (mdata->min_modseq != 0) { + mail_index_update_modseq(ctx->trans, ctx->seq, + mdata->min_modseq); + } + + offset = ctx->output->offset == 0 ? 0 : + ctx->output->offset - 1; + mail_index_update_ext(ctx->trans, ctx->seq, + ctx->mbox->mbox_ext_idx, &offset, NULL); + ctx->next_uid++; + + /* parse and cache the mail headers as we read it */ + mail_set_seq_saving(_ctx->dest_mail, ctx->seq); + } + mbox_save_append_flag_headers(ctx->headers, save_flags); + mbox_save_append_keyword_headers(ctx, mdata->keywords); + str_append_c(ctx->headers, '\n'); + + i_assert(ctx->mbox->mbox_lock_type == F_WRLCK); + + ctx->mail_offset = ctx->output->offset; + ctx->eoh_offset = UOFF_T_MAX; + ctx->last_char = '\n'; + + if (write_from_line(ctx, mdata->received_date, mdata->from_envelope) < 0) + ctx->failed = TRUE; + else + ctx->input = mbox_save_get_input_stream(ctx, input); + return ctx->failed ? -1 : 0; +} + +static int mbox_save_body_input(struct mbox_save_context *ctx) +{ + const unsigned char *data; + size_t size; + + data = i_stream_get_data(ctx->input, &size); + if (size > 0) { + if (o_stream_send(ctx->output, data, size) < 0) { + write_stream_error(ctx); + return -1; + } + ctx->last_char = data[size-1]; + i_stream_skip(ctx->input, size); + } + return 0; +} + +static int mbox_save_body(struct mbox_save_context *ctx) +{ + ssize_t ret; + + while ((ret = i_stream_read(ctx->input)) != -1) { + if (mbox_save_body_input(ctx) < 0) + return -1; + /* i_stream_read() may have returned 0 at EOF + because of this parser */ + index_mail_cache_parse_continue(ctx->ctx.dest_mail); + if (ret == 0) + return 0; + } + + i_assert(ctx->last_char == '\n'); + return 0; +} + +static int mbox_save_finish_headers(struct mbox_save_context *ctx) +{ + i_assert(ctx->eoh_offset == UOFF_T_MAX); + + /* append our own headers and ending empty line */ + ctx->extra_hdr_offset = ctx->output->offset; + if (o_stream_send(ctx->output, str_data(ctx->headers), + str_len(ctx->headers)) < 0) { + write_stream_error(ctx); + return -1; + } + ctx->eoh_offset = ctx->output->offset; + return 0; +} + +int mbox_save_continue(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + const unsigned char *data; + size_t i, size; + ssize_t ret; + + if (ctx->failed) + return -1; + + if (ctx->eoh_offset != UOFF_T_MAX) { + /* writing body */ + return mbox_save_body(ctx); + } + + while ((ret = i_stream_read_more(ctx->input, &data, &size)) > 0) { + for (i = 0; i < size; i++) { + if (data[i] == '\n' && + ((i == 0 && ctx->last_char == '\n') || + (i > 0 && data[i-1] == '\n'))) { + /* end of headers. we don't need to worry about + CRs because they're dropped */ + break; + } + } + if (i != size) { + /* found end of headers. write the rest of them + (not including the finishing empty line) */ + if (o_stream_send(ctx->output, data, i) < 0) { + write_stream_error(ctx); + return -1; + } + ctx->last_char = '\n'; + i_stream_skip(ctx->input, i + 1); + break; + } + + if (o_stream_send(ctx->output, data, size) < 0) { + write_stream_error(ctx); + return -1; + } + i_assert(size > 0); + ctx->last_char = data[size-1]; + i_stream_skip(ctx->input, size); + index_mail_cache_parse_continue(ctx->ctx.dest_mail); + } + if (ret == 0) + return 0; + if (ctx->input->stream_errno != 0) { + i_error("read(%s) failed: %s", i_stream_get_name(ctx->input), + i_stream_get_error(ctx->input)); + ctx->failed = TRUE; + return -1; + } + + i_assert(ctx->last_char == '\n'); + + if (ctx->mbox_md5_ctx != NULL) { + unsigned char hdr_md5_sum[16]; + + if (ctx->x_delivery_id_header != NULL) { + struct message_header_line hdr; + + i_zero(&hdr); + hdr.name = ctx->x_delivery_id_header; + hdr.name_len = sizeof("X-Delivery-ID")-1; + hdr.middle = (const unsigned char *)hdr.name + + hdr.name_len; + hdr.middle_len = 2; + hdr.value = hdr.full_value = + hdr.middle + hdr.middle_len; + hdr.value_len = strlen((const char *)hdr.value); + ctx->mbox->md5_v.more(ctx->mbox_md5_ctx, &hdr); + } + ctx->mbox->md5_v.finish(ctx->mbox_md5_ctx, hdr_md5_sum); + mail_index_update_ext(ctx->trans, ctx->seq, + ctx->mbox->md5hdr_ext_idx, + hdr_md5_sum, NULL); + } + + if (mbox_save_finish_headers(ctx) < 0) + return -1; + + /* write body */ + if (mbox_save_body_input(ctx) < 0) + return -1; + return ctx->input->eof ? 0 : mbox_save_body(ctx); +} + +int mbox_save_finish(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + if (!ctx->failed && ctx->eoh_offset == UOFF_T_MAX) + (void)mbox_save_finish_headers(ctx); + + if (ctx->output != NULL) { + /* make sure everything is written */ + if (o_stream_flush(ctx->output) < 0) + write_stream_error(ctx); + } + + ctx->finished = TRUE; + if (!ctx->failed) { + i_assert(ctx->output != NULL); + T_BEGIN { + if (mbox_write_content_length(ctx) < 0 || + mbox_append_lf(ctx) < 0) + ctx->failed = TRUE; + } T_END; + } + + index_mail_cache_parse_deinit(ctx->ctx.dest_mail, + ctx->ctx.data.received_date, + !ctx->failed); + if (ctx->input != NULL) + i_stream_destroy(&ctx->input); + + if (ctx->failed && ctx->mail_offset != UOFF_T_MAX) { + /* saving this mail failed - truncate back to beginning of it */ + i_assert(ctx->output != NULL); + (void)o_stream_flush(ctx->output); + if (ftruncate(ctx->mbox->mbox_fd, (off_t)ctx->mail_offset) < 0) + mbox_set_syscall_error(ctx->mbox, "ftruncate()"); + (void)o_stream_seek(ctx->output, ctx->mail_offset); + ctx->mail_offset = UOFF_T_MAX; + } + + if (ctx->seq != 0 && ctx->failed) { + index_storage_save_abort_last(&ctx->ctx, ctx->seq); + } + index_save_context_free(_ctx); + return ctx->failed ? -1 : 0; +} + +void mbox_save_cancel(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)mbox_save_finish(_ctx); +} + +static void mbox_transaction_save_deinit(struct mbox_save_context *ctx) +{ + o_stream_destroy(&ctx->output); + str_free(&ctx->headers); +} + +static void mbox_save_truncate(struct mbox_save_context *ctx) +{ + if (ctx->append_offset == UOFF_T_MAX || ctx->mbox->mbox_fd == -1) + return; + + i_assert(ctx->mbox->mbox_lock_type == F_WRLCK); + + /* failed, truncate file back to original size. output stream needs to + be flushed before truncating so unref() won't write anything. */ + if (ctx->output != NULL) + (void)o_stream_flush(ctx->output); + + if (ftruncate(ctx->mbox->mbox_fd, (off_t)ctx->append_offset) < 0) + mbox_set_syscall_error(ctx->mbox, "ftruncate()"); +} + +int mbox_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + struct mbox_mailbox *mbox = ctx->mbox; + struct stat st; + int ret = 0; + + i_assert(ctx->finished); + i_assert(mbox->mbox_fd != -1); + + if (fstat(mbox->mbox_fd, &st) < 0) { + mbox_set_syscall_error(mbox, "fstat()"); + ret = -1; + } + + if (ctx->synced) { + _t->changes->uid_validity = ctx->uid_validity; + mail_index_append_finish_uids(ctx->trans, 0, + &_t->changes->saved_uids); + + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, next_uid), + &ctx->next_uid, sizeof(ctx->next_uid), FALSE); + + if (ret == 0) { + mbox->mbox_hdr.sync_mtime = st.st_mtime; + mbox->mbox_hdr.sync_size = st.st_size; + mail_index_update_header_ext(ctx->trans, + mbox->mbox_ext_idx, + 0, &mbox->mbox_hdr, + sizeof(mbox->mbox_hdr)); + } + } + + if (ret == 0 && ctx->orig_atime != st.st_atime) { + /* try to set atime back to its original value. + (it'll fail with EPERM for shared mailboxes where we aren't + the file's owner) */ + struct utimbuf buf; + + buf.modtime = st.st_mtime; + buf.actime = ctx->orig_atime; + if (utime(mailbox_get_path(&mbox->box), &buf) < 0 && + errno != EPERM) + mbox_set_syscall_error(mbox, "utime()"); + } + + if (ctx->output != NULL) { + /* flush the final LF */ + if (o_stream_flush(ctx->output) < 0) + write_stream_error(ctx); + } + if (mbox->mbox_fd != -1 && !mbox->mbox_writeonly && + mbox->storage->storage.set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync(mbox->mbox_fd) < 0) { + mbox_set_syscall_error(mbox, "fdatasync()"); + mbox_save_truncate(ctx); + ret = -1; + } + } + + mbox_transaction_save_deinit(ctx); + if (ret < 0) + i_free(ctx); + return ret; +} + +void mbox_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result ATTR_UNUSED) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + i_assert(ctx->mbox->mbox_lock_type == F_WRLCK); + + if (ctx->synced) { + /* after saving mails with UIDs we need to update + the last-uid */ + (void)mbox_sync(ctx->mbox, MBOX_SYNC_HEADER | + MBOX_SYNC_REWRITE); + } + i_free(ctx); +} + +void mbox_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx); + + if (!ctx->finished) + mbox_save_cancel(&ctx->ctx); + + mbox_save_truncate(ctx); + mbox_transaction_save_deinit(ctx); + i_free(ctx); +} diff --git a/src/lib-storage/index/mbox/mbox-settings.c b/src/lib-storage/index/mbox/mbox-settings.c new file mode 100644 index 0000000..1df2452 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-settings.c @@ -0,0 +1,55 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "mbox-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct mbox_settings) + +static const struct setting_define mbox_setting_defines[] = { + DEF(STR, mbox_read_locks), + DEF(STR, mbox_write_locks), + DEF(TIME, mbox_lock_timeout), + DEF(TIME, mbox_dotlock_change_timeout), + DEF(SIZE, mbox_min_index_size), + DEF(BOOL, mbox_dirty_syncs), + DEF(BOOL, mbox_very_dirty_syncs), + DEF(BOOL, mbox_lazy_writes), + DEF(ENUM, mbox_md5), + + SETTING_DEFINE_LIST_END +}; + +static const struct mbox_settings mbox_default_settings = { + .mbox_read_locks = "fcntl", + .mbox_write_locks = "dotlock fcntl", + .mbox_lock_timeout = 5*60, + .mbox_dotlock_change_timeout = 2*60, + .mbox_min_index_size = 0, + .mbox_dirty_syncs = TRUE, + .mbox_very_dirty_syncs = FALSE, + .mbox_lazy_writes = TRUE, + .mbox_md5 = "apop3d:all" +}; + +static const struct setting_parser_info mbox_setting_parser_info = { + .module_name = "mbox", + .defines = mbox_setting_defines, + .defaults = &mbox_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct mbox_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info +}; + +const struct setting_parser_info *mbox_get_setting_parser_info(void) +{ + return &mbox_setting_parser_info; +} diff --git a/src/lib-storage/index/mbox/mbox-settings.h b/src/lib-storage/index/mbox/mbox-settings.h new file mode 100644 index 0000000..eb99d82 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-settings.h @@ -0,0 +1,18 @@ +#ifndef MBOX_SETTINGS_H +#define MBOX_SETTINGS_H + +struct mbox_settings { + const char *mbox_read_locks; + const char *mbox_write_locks; + unsigned int mbox_lock_timeout; + unsigned int mbox_dotlock_change_timeout; + uoff_t mbox_min_index_size; + bool mbox_dirty_syncs; + bool mbox_very_dirty_syncs; + bool mbox_lazy_writes; + const char *mbox_md5; +}; + +const struct setting_parser_info *mbox_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-storage.c b/src/lib-storage/index/mbox/mbox-storage.c new file mode 100644 index 0000000..4b2bb35 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-storage.c @@ -0,0 +1,911 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "restrict-access.h" +#include "master-service.h" +#include "mailbox-list-private.h" +#include "mbox-storage.h" +#include "mbox-lock.h" +#include "mbox-file.h" +#include "mbox-sync-private.h" +#include "istream-raw-mbox.h" +#include "mail-copy.h" +#include "index-mail.h" + +#include <sys/stat.h> + +/* How often to touch the dotlock file when using KEEP_LOCKED flag */ +#define MBOX_LOCK_TOUCH_MSECS (10*1000) + +/* Assume that if atime < mtime, there are new mails. If it's good enough for + UW-IMAP, it's good enough for us. */ +#define STAT_GET_MARKED(st) \ + ((st).st_size == 0 ? MAILBOX_UNMARKED : \ + (st).st_atime < (st).st_mtime ? MAILBOX_MARKED : MAILBOX_UNMARKED) + +#define MBOX_LIST_CONTEXT(obj) \ + MODULE_CONTEXT_REQUIRE(obj, mbox_mailbox_list_module) + +struct mbox_mailbox_list { + union mailbox_list_module_context module_ctx; + const struct mbox_settings *set; +}; + +/* NOTE: must be sorted for istream-header-filter. Note that it's not such + a good idea to change this list, as the messages will then change from + client's point of view. So if you do it, change all mailboxes' UIDVALIDITY + so all caches are reset. */ +const char *mbox_hide_headers[] = { + "Content-Length", + "Status", + "X-IMAP", + "X-IMAPbase", + "X-Keywords", + "X-Status", + "X-UID" +}; +unsigned int mbox_hide_headers_count = N_ELEMENTS(mbox_hide_headers); + +/* A bit ugly duplification of the above list. It's safe to modify this list + without bad side effects, just keep the list sorted. */ +const char *mbox_save_drop_headers[] = { + "Content-Length", + "Status", + "X-Delivery-ID", + "X-IMAP", + "X-IMAPbase", + "X-Keywords", + "X-Status", + "X-UID" +}; +unsigned int mbox_save_drop_headers_count = N_ELEMENTS(mbox_save_drop_headers); + +extern struct mail_storage mbox_storage; +extern struct mailbox mbox_mailbox; + +static struct event_category event_category_mbox = { + .name = "mbox", + .parent = &event_category_storage, +}; + +static MODULE_CONTEXT_DEFINE_INIT(mbox_mailbox_list_module, + &mailbox_list_module_register); + +static void +mbox_set_syscall_error_str(struct mbox_mailbox *mbox, const char *function, + const char *error) +{ + i_assert(function != NULL); + + if (ENOQUOTA(errno)) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + } else { + const char *toobig_error = errno != EFBIG ? "" : + " (process was started with ulimit -f limit)"; + mailbox_set_critical(&mbox->box, "%s failed with mbox: %s%s", + function, error, toobig_error); + } +} + +void mbox_set_syscall_error(struct mbox_mailbox *mbox, const char *function) +{ + mbox_set_syscall_error_str(mbox, function, strerror(errno)); +} + +void mbox_istream_set_syscall_error(struct mbox_mailbox *mbox, + struct istream *input, + const char *function) +{ + errno = input->stream_errno; + mbox_set_syscall_error_str(mbox, function, i_stream_get_error(input)); +} + +void mbox_ostream_set_syscall_error(struct mbox_mailbox *mbox, + struct ostream *output, + const char *function) +{ + errno = output->stream_errno; + mbox_set_syscall_error_str(mbox, function, o_stream_get_error(output)); +} + +static int +mbox_list_get_path(struct mailbox_list *list, const char *name, + enum mailbox_list_path_type type, const char **path_r) +{ + struct mbox_mailbox_list *mlist = MBOX_LIST_CONTEXT(list); + const char *path, *p; + int ret; + + *path_r = NULL; + + ret = mlist->module_ctx.super.get_path(list, name, type, &path); + if (ret <= 0) + return ret; + + switch (type) { + case MAILBOX_LIST_PATH_TYPE_CONTROL: + case MAILBOX_LIST_PATH_TYPE_INDEX: + case MAILBOX_LIST_PATH_TYPE_INDEX_CACHE: + case MAILBOX_LIST_PATH_TYPE_LIST_INDEX: + if (name == NULL && type == MAILBOX_LIST_PATH_TYPE_CONTROL && + list->set.control_dir != NULL) { + /* kind of a kludge for backwards compatibility: + the subscriptions file is in the root control_dir + without .imap/ suffix */ + *path_r = path; + return 1; + } + if (name == NULL) { + *path_r = t_strconcat(path, "/"MBOX_INDEX_DIR_NAME, NULL); + return 1; + } + + p = strrchr(path, '/'); + if (p == NULL) + return 0; + + *path_r = t_strconcat(t_strdup_until(path, p), + "/"MBOX_INDEX_DIR_NAME"/", p+1, NULL); + break; + case MAILBOX_LIST_PATH_TYPE_DIR: + case MAILBOX_LIST_PATH_TYPE_ALT_DIR: + case MAILBOX_LIST_PATH_TYPE_MAILBOX: + case MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX: + case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE: + *path_r = path; + break; + } + return 1; +} + +static struct mail_storage *mbox_storage_alloc(void) +{ + struct mbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mbox storage", 512+256); + storage = p_new(pool, struct mbox_storage, 1); + storage->storage = mbox_storage; + storage->storage.pool = pool; + return &storage->storage; +} + +static int +mbox_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, + const char **error_r) +{ + struct mbox_storage *storage = MBOX_STORAGE(_storage); + struct stat st; + const char *dir; + + if (master_service_get_client_limit(master_service) > 1) { + /* we can't handle locking related problems. */ + *error_r = "mbox requires client_limit=1 for service"; + return -1; + } + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + + if (mailbox_list_get_root_path(ns->list, MAILBOX_LIST_PATH_TYPE_INDEX, &dir)) { + _storage->temp_path_prefix = p_strconcat(_storage->pool, dir, + "/", mailbox_list_get_temp_prefix(ns->list), NULL); + } + if (stat(ns->list->set.root_dir, &st) == 0 && !S_ISDIR(st.st_mode)) { + *error_r = t_strdup_printf( + "mbox root directory can't be a file: %s " + "(http://wiki2.dovecot.org/MailLocation/Mbox)", + ns->list->set.root_dir); + return -1; + } + return 0; +} + +static void mbox_storage_get_list_settings(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_FS; + if (set->subscription_fname == NULL) + set->subscription_fname = MBOX_SUBSCRIPTION_FILE_NAME; + + if (set->inbox_path == NULL && + strcasecmp(set->layout, MAILBOX_LIST_NAME_FS) == 0) { + set->inbox_path = t_strconcat(set->root_dir, "/inbox", NULL); + e_debug(ns->user->event, "mbox: INBOX defaulted to %s", set->inbox_path); + } +} + +static bool mbox_is_file(const char *path, const char *name, bool debug) +{ + struct stat st; + + if (stat(path, &st) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: stat(%s) failed: %m", + name, path); + } + return FALSE; + } + if (S_ISDIR(st.st_mode)) { + if (debug) { + i_debug("mbox autodetect: %s: is a directory (%s)", + name, path); + } + return FALSE; + } + if (access(path, R_OK|W_OK) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: no R/W access (%s)", + name, path); + } + return FALSE; + } + + if (debug) + i_debug("mbox autodetect: %s: yes (%s)", name, path); + return TRUE; +} + +static bool mbox_is_dir(const char *path, const char *name, bool debug) +{ + struct stat st; + + if (stat(path, &st) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: stat(%s) failed: %m", + name, path); + } + return FALSE; + } + if (!S_ISDIR(st.st_mode)) { + if (debug) { + i_debug("mbox autodetect: %s: is not a directory (%s)", + name, path); + } + return FALSE; + } + if (access(path, R_OK|W_OK|X_OK) < 0) { + if (debug) { + i_debug("mbox autodetect: %s: no R/W/X access (%s)", + name, path); + } + return FALSE; + } + + if (debug) + i_debug("mbox autodetect: %s: yes (%s)", name, path); + return TRUE; +} + +static bool mbox_storage_is_root_dir(const char *dir, bool debug) +{ + if (mbox_is_dir(t_strconcat(dir, "/"MBOX_INDEX_DIR_NAME, NULL), + "has "MBOX_INDEX_DIR_NAME"/", debug)) + return TRUE; + if (mbox_is_file(t_strconcat(dir, "/inbox", NULL), "has inbox", debug)) + return TRUE; + if (mbox_is_file(t_strconcat(dir, "/mbox", NULL), "has mbox", debug)) + return TRUE; + return FALSE; +} + +static const char *mbox_storage_find_root_dir(const struct mail_namespace *ns) +{ + bool debug = ns->mail_set->mail_debug; + const char *home, *path; + + if (ns->owner == NULL || + mail_user_get_home(ns->owner, &home) <= 0) { + if (debug) + i_debug("maildir: Home directory not set"); + home = ""; + } + + path = t_strconcat(home, "/mail", NULL); + if (mbox_storage_is_root_dir(path, debug)) + return path; + + path = t_strconcat(home, "/Mail", NULL); + if (mbox_storage_is_root_dir(path, debug)) + return path; + return NULL; +} + +static const char * +mbox_storage_find_inbox_file(const char *user, bool debug) +{ + const char *path; + + path = t_strconcat("/var/mail/", user, NULL); + if (access(path, R_OK|W_OK) == 0) { + if (debug) + i_debug("mbox: INBOX exists (%s)", path); + return path; + } + if (debug) + i_debug("mbox: INBOX: access(%s, rw) failed: %m", path); + + path = t_strconcat("/var/spool/mail/", user, NULL); + if (access(path, R_OK|W_OK) == 0) { + if (debug) + i_debug("mbox: INBOX exists (%s)", path); + return path; + } + if (debug) + i_debug("mbox: INBOX: access(%s, rw) failed: %m", path); + return NULL; +} + +static bool mbox_storage_autodetect(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + bool debug = ns->mail_set->mail_debug; + const char *root_dir, *inbox_path; + + root_dir = set->root_dir; + inbox_path = set->inbox_path; + + if (root_dir != NULL) { + if (inbox_path == NULL && + mbox_is_file(root_dir, "INBOX file", debug)) { + /* using location=<INBOX> */ + inbox_path = root_dir; + root_dir = NULL; + } else if (!mbox_storage_is_root_dir(root_dir, debug)) + return FALSE; + } + if (root_dir == NULL) { + root_dir = mbox_storage_find_root_dir(ns); + if (root_dir == NULL) { + if (debug) + i_debug("mbox: couldn't find root dir"); + return FALSE; + } + } + if (inbox_path == NULL) { + inbox_path = mbox_storage_find_inbox_file(ns->user->username, + debug); + } + set->root_dir = root_dir; + set->inbox_path = inbox_path; + + mbox_storage_get_list_settings(ns, set); + return TRUE; +} + +static bool want_memory_indexes(struct mbox_storage *storage, const char *path) +{ + struct stat st; + + if (storage->set->mbox_min_index_size == 0) + return FALSE; + + if (stat(path, &st) < 0) { + if (errno == ENOENT) + st.st_size = 0; + else { + mail_storage_set_critical(&storage->storage, + "stat(%s) failed: %m", path); + return FALSE; + } + } + return (uoff_t)st.st_size < storage->set->mbox_min_index_size; +} + +static struct mailbox * +mbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mbox_mailbox *mbox; + pool_t pool; + + pool = pool_alloconly_create("mbox mailbox", 1024*3); + mbox = p_new(pool, struct mbox_mailbox, 1); + mbox->box = mbox_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = MBOX_STORAGE(storage); + mbox->mbox_fd = -1; + mbox->mbox_lock_type = F_UNLCK; + mbox->mbox_list_index_ext_id = (uint32_t)-1; + + if (strcmp(mbox->storage->set->mbox_md5, "apop3d") == 0) + mbox->md5_v = mbox_md5_apop3d; + else if (strcmp(mbox->storage->set->mbox_md5, "all") == 0) + mbox->md5_v = mbox_md5_all; + else { + i_fatal("Invalid mbox_md5 setting: %s", + mbox->storage->set->mbox_md5); + } + + if ((storage->flags & MAIL_STORAGE_FLAG_KEEP_HEADER_MD5) != 0) + mbox->mbox_save_md5 = TRUE; + return &mbox->box; +} + +static void mbox_lock_touch_timeout(struct mbox_mailbox *mbox) +{ + mbox_dotlock_touch(mbox); +} + +static int +mbox_mailbox_open_finish(struct mbox_mailbox *mbox, bool move_to_memory) +{ + if (index_storage_mailbox_open(&mbox->box, move_to_memory) < 0) + return -1; + + mbox->mbox_ext_idx = + mail_index_ext_register(mbox->box.index, "mbox", + sizeof(mbox->mbox_hdr), + sizeof(uint64_t), sizeof(uint64_t)); + mbox->md5hdr_ext_idx = + mail_index_ext_register(mbox->box.index, "header-md5", + 0, 16, 1); + return 0; +} + +static int mbox_mailbox_open_existing(struct mbox_mailbox *mbox) +{ + struct mailbox *box = &mbox->box; + const char *rootdir, *box_path = mailbox_get_path(box); + bool move_to_memory; + + move_to_memory = want_memory_indexes(mbox->storage, box_path); + + if (box->inbox_any || strcmp(box->name, "INBOX") == 0) { + /* if INBOX isn't under the root directory, it's probably in + /var/mail and we want to allow privileged dotlocking */ + rootdir = mailbox_list_get_root_forced(box->list, + MAILBOX_LIST_PATH_TYPE_DIR); + if (!str_begins(box_path, rootdir)) + mbox->mbox_privileged_locking = TRUE; + } + if ((box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0) { + if (mbox_lock(mbox, F_WRLCK, &mbox->mbox_global_lock_id) <= 0) + return -1; + + if (mbox->mbox_dotlock != NULL) { + mbox->keep_lock_to = + timeout_add(MBOX_LOCK_TOUCH_MSECS, + mbox_lock_touch_timeout, mbox); + } + } + return mbox_mailbox_open_finish(mbox, move_to_memory); +} + +static bool mbox_storage_is_readonly(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (index_storage_is_readonly(box)) + return TRUE; + + if (mbox_is_backend_readonly(mbox)) { + /* return read-only only if there are no private flags + (that are stored in index files) */ + if (mailbox_get_private_flags_mask(box) == 0) + return TRUE; + } + return FALSE; +} + +static int mbox_mailbox_open(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + struct stat st; + int ret; + + if (box->input != NULL) { + i_stream_ref(box->input); + mbox->mbox_file_stream = box->input; + mbox->backend_readonly = TRUE; + mbox->backend_readonly_set = TRUE; + mbox->no_mbox_file = TRUE; + return mbox_mailbox_open_finish(mbox, FALSE); + } + + ret = stat(mailbox_get_path(box), &st); + if (ret == 0) { + if (!S_ISDIR(st.st_mode)) + return mbox_mailbox_open_existing(mbox); + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + "Mailbox isn't selectable"); + return -1; + } else if (ENOTFOUND(errno)) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } else if (mail_storage_set_error_from_errno(box->storage)) { + return -1; + } else { + mailbox_set_critical(box, + "stat(%s) failed: %m", mailbox_get_path(box)); + return -1; + } +} + +static int +mbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + + if (update->uid_validity != 0 || update->min_next_uid != 0 || + !guid_128_is_empty(update->mailbox_guid)) { + mbox->sync_hdr_update = update; + ret = mbox_sync(mbox, MBOX_SYNC_HEADER | MBOX_SYNC_FORCE_SYNC | + MBOX_SYNC_REWRITE); + mbox->sync_hdr_update = NULL; + } + if (ret == 0) + ret = index_storage_mailbox_update(box, update); + return ret; +} + +static int create_inbox(struct mailbox *box) +{ + const char *inbox_path; + int fd; + + inbox_path = mailbox_get_path(box); + + fd = open(inbox_path, O_RDWR | O_CREAT | O_EXCL, 0660); + if (fd == -1 && errno == EACCES) { + /* try again with increased privileges */ + (void)restrict_access_use_priv_gid(); + fd = open(inbox_path, O_RDWR | O_CREAT | O_EXCL, 0660); + restrict_access_drop_priv_gid(); + } + if (fd != -1) { + i_close_fd(&fd); + return 0; + } else if (errno == EACCES) { + mailbox_set_critical(box, "%s", + mail_error_create_eacces_msg("open", inbox_path)); + return -1; + } else if (errno == EEXIST) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } else { + mailbox_set_critical(box, + "open(%s, O_CREAT) failed: %m", inbox_path); + return -1; + } +} + +static int +mbox_mailbox_create(struct mailbox *box, const struct mailbox_update *update, + bool directory) +{ + int fd, ret; + + if ((ret = index_storage_mailbox_create(box, directory)) <= 0) + return ret; + + if (box->inbox_any) { + if (create_inbox(box) < 0) + return -1; + } else { + /* create the mbox file */ + ret = mailbox_create_fd(box, mailbox_get_path(box), + O_RDWR | O_CREAT | O_EXCL, &fd); + if (ret < 0) + return -1; + if (ret == 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + return -1; + } + i_close_fd(&fd); + } + return update == NULL ? 0 : mbox_mailbox_update(box, update); +} + +static void mbox_mailbox_close(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + const struct mail_index_header *hdr; + enum mbox_sync_flags sync_flags = 0; + + if (mbox->mbox_stream != NULL && + istream_raw_mbox_is_corrupted(mbox->mbox_stream)) { + /* clear the corruption by forcing a full resync */ + sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC; + } + + if (box->view != NULL) { + hdr = mail_index_get_header(box->view); + if ((hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 && + !mbox_is_backend_readonly(mbox)) { + /* we've done changes to mbox which haven't been + written yet. do it now. */ + sync_flags |= MBOX_SYNC_REWRITE; + } + } + if (sync_flags != 0 && !mbox->invalid_mbox_file) + (void)mbox_sync(mbox, sync_flags); + + if (mbox->mbox_global_lock_id != 0) + mbox_unlock(mbox, mbox->mbox_global_lock_id); + timeout_remove(&mbox->keep_lock_to); + + mbox_file_close(mbox); + i_stream_destroy(&mbox->mbox_file_stream); + + index_storage_mailbox_close(box); +} + +static int +mbox_mailbox_get_guid(struct mbox_mailbox *mbox, guid_128_t guid_r) +{ + const char *errstr; + + if (mail_index_is_in_memory(mbox->box.index)) { + errstr = "Mailbox GUIDs are not permanent without index files"; + if (mbox->storage->set->mbox_min_index_size != 0) { + errstr = t_strconcat(errstr, + " (mbox_min_index_size is non-zero)", NULL); + } + mail_storage_set_error(mbox->box.storage, + MAIL_ERROR_NOTPOSSIBLE, errstr); + return -1; + } + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + + if (!guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + /* we have the GUID */ + } else if (mbox_file_open(mbox) < 0) + return -1; + else if (mbox->backend_readonly) { + mail_storage_set_error(mbox->box.storage, MAIL_ERROR_PERM, + "Can't set mailbox GUID to a read-only mailbox"); + return -1; + } else { + /* create another mailbox and sync */ + struct mailbox *box2; + struct mbox_mailbox *mbox2; + int ret; + + i_assert(mbox->mbox_lock_type == F_UNLCK); + box2 = mailbox_alloc(mbox->box.list, mbox->box.vname, 0); + ret = mailbox_sync(box2, 0); + mbox2 = MBOX_MAILBOX(box2); + memcpy(guid_r, mbox2->mbox_hdr.mailbox_guid, GUID_128_SIZE); + mailbox_free(&box2); + return ret; + } + memcpy(guid_r, mbox->mbox_hdr.mailbox_guid, GUID_128_SIZE); + return 0; +} + +static int +mbox_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + if ((items & MAILBOX_METADATA_GUID) != 0) { + if (mbox_mailbox_get_guid(mbox, metadata_r->guid) < 0) + return -1; + } + return 0; +} + +static void mbox_notify_changes(struct mailbox *box) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (box->notify_callback == NULL) + mailbox_watch_remove_all(box); + else if (!mbox->no_mbox_file) + mailbox_watch_add(box, mailbox_get_path(box)); +} + +static bool +mbox_is_internal_name(struct mailbox_list *list ATTR_UNUSED, + const char *name) +{ + size_t len; + + /* don't allow *.lock files/dirs */ + len = strlen(name); + if (len > 5 && strcmp(name+len-5, ".lock") == 0) + return TRUE; + + return strcmp(name, MBOX_INDEX_DIR_NAME) == 0; +} + +static void mbox_storage_add_list(struct mail_storage *storage, + struct mailbox_list *list) +{ + struct mbox_mailbox_list *mlist; + + mlist = p_new(list->pool, struct mbox_mailbox_list, 1); + mlist->module_ctx.super = list->v; + mlist->set = mail_namespace_get_driver_settings(list->ns, storage); + + if (*list->set.maildir_name == '\0') { + /* have to use .imap/ directories */ + list->v.get_path = mbox_list_get_path; + } + list->v.is_internal_name = mbox_is_internal_name; + + MODULE_CONTEXT_SET(list, mbox_mailbox_list_module, mlist); +} + +static struct mailbox_transaction_context * +mbox_transaction_begin(struct mailbox *box, + enum mailbox_transaction_flags flags, + const char *reason) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + struct mbox_transaction_context *mt; + + if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) + mbox->external_transactions++; + + mt = i_new(struct mbox_transaction_context, 1); + index_transaction_init(&mt->t, box, flags, reason); + return &mt->t; +} + +static void +mbox_transaction_unlock(struct mailbox *box, unsigned int lock_id1, + unsigned int lock_id2) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + + if (lock_id1 != 0) + mbox_unlock(mbox, lock_id1); + if (lock_id2 != 0) + mbox_unlock(mbox, lock_id2); + if (mbox->mbox_global_lock_id == 0) { + i_assert(mbox->box.transaction_count > 0); + i_assert(mbox->box.transaction_count > 1 || + mbox->external_transactions > 0 || + mbox->mbox_lock_type == F_UNLCK); + } else { + /* mailbox opened with MAILBOX_FLAG_KEEP_LOCKED */ + i_assert(mbox->mbox_lock_type == F_WRLCK); + } +} + +static int +mbox_transaction_commit(struct mailbox_transaction_context *t, + struct mail_transaction_commit_changes *changes_r) +{ + struct mbox_transaction_context *mt = MBOX_TRANSCTX(t); + struct mailbox *box = t->box; + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + unsigned int read_lock_id = mt->read_lock_id; + unsigned int write_lock_id = mt->write_lock_id; + int ret; + + if ((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) { + i_assert(mbox->external_transactions > 0); + mbox->external_transactions--; + } + + ret = index_transaction_commit(t, changes_r); + mbox_transaction_unlock(box, read_lock_id, write_lock_id); + return ret; +} + +static void +mbox_transaction_rollback(struct mailbox_transaction_context *t) +{ + struct mbox_transaction_context *mt = MBOX_TRANSCTX(t); + struct mailbox *box = t->box; + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + unsigned int read_lock_id = mt->read_lock_id; + unsigned int write_lock_id = mt->write_lock_id; + + if ((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) { + i_assert(mbox->external_transactions > 0); + mbox->external_transactions--; + } + + index_transaction_rollback(t); + mbox_transaction_unlock(box, read_lock_id, write_lock_id); +} + +bool mbox_is_backend_readonly(struct mbox_mailbox *mbox) +{ + if (!mbox->backend_readonly_set) { + mbox->backend_readonly_set = TRUE; + if (access(mailbox_get_path(&mbox->box), R_OK|W_OK) < 0 && + errno == EACCES) + mbox->backend_readonly = TRUE; + } + return mbox->backend_readonly; +} + +struct mail_storage mbox_storage = { + .name = MBOX_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_MAILBOX_IS_FILE | + MAIL_STORAGE_CLASS_FLAG_OPEN_STREAMS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS, + .event_category = &event_category_mbox, + + .v = { + mbox_get_setting_parser_info, + mbox_storage_alloc, + mbox_storage_create, + index_storage_destroy, + mbox_storage_add_list, + mbox_storage_get_list_settings, + mbox_storage_autodetect, + mbox_mailbox_alloc, + NULL, + NULL, + } +}; + +struct mailbox mbox_mailbox = { + .v = { + mbox_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mbox_mailbox_open, + mbox_mailbox_close, + index_storage_mailbox_free, + mbox_mailbox_create, + mbox_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mbox_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + mbox_list_index_has_changed, + mbox_list_index_update_sync, + mbox_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + mbox_notify_changes, + mbox_transaction_begin, + mbox_transaction_commit, + mbox_transaction_rollback, + NULL, + index_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + mbox_save_alloc, + mbox_save_begin, + mbox_save_continue, + mbox_save_finish, + mbox_save_cancel, + mail_storage_copy, + mbox_transaction_save_commit_pre, + mbox_transaction_save_commit_post, + mbox_transaction_save_rollback, + index_storage_is_inconsistent + } +}; diff --git a/src/lib-storage/index/mbox/mbox-storage.h b/src/lib-storage/index/mbox/mbox-storage.h new file mode 100644 index 0000000..e35a795 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-storage.h @@ -0,0 +1,115 @@ +#ifndef MBOX_STORAGE_H +#define MBOX_STORAGE_H + +#include "index-storage.h" +#include "mbox-settings.h" +#include "mbox-md5.h" + +/* Padding to leave in X-Keywords header when rewriting mbox */ +#define MBOX_HEADER_PADDING 50 +/* Don't write Content-Length header unless it's value is larger than this. */ +#define MBOX_MIN_CONTENT_LENGTH_SIZE 1024 + +#define MBOX_STORAGE_NAME "mbox" +#define MBOX_SUBSCRIPTION_FILE_NAME ".subscriptions" +#define MBOX_INDEX_DIR_NAME ".imap" +#define MBOX_UIDVALIDITY_FNAME "dovecot-uidvalidity" + +struct mbox_index_header { + uint64_t sync_size; + uint32_t sync_mtime; + uint8_t dirty_flag; + uint8_t unused[3]; + guid_128_t mailbox_guid; +}; + +struct mbox_list_index_record { + uint32_t mtime; + uint32_t size; +}; + +struct mbox_storage { + struct mail_storage storage; + + const struct mbox_settings *set; + enum mbox_lock_type *read_locks; + enum mbox_lock_type *write_locks; + bool lock_settings_initialized:1; +}; + +struct mbox_mailbox { + struct mailbox box; + struct mbox_storage *storage; + + int mbox_fd; + struct istream *mbox_stream, *mbox_file_stream; + int mbox_lock_type; + dev_t mbox_dev; + ino_t mbox_ino; + unsigned int mbox_excl_locks, mbox_shared_locks; + struct dotlock *mbox_dotlock; + unsigned int mbox_lock_id, mbox_global_lock_id; + struct timeout *keep_lock_to; + bool mbox_writeonly; + unsigned int external_transactions; + + uint32_t mbox_ext_idx, md5hdr_ext_idx, mbox_list_index_ext_id; + struct mbox_index_header mbox_hdr; + const struct mailbox_update *sync_hdr_update; + + struct mbox_md5_vfuncs md5_v; + + bool no_mbox_file:1; + bool invalid_mbox_file:1; + bool mbox_broken_offsets:1; + bool mbox_save_md5:1; + bool mbox_dotlocked:1; + bool mbox_used_privileges:1; + bool mbox_privileged_locking:1; + bool syncing:1; + bool backend_readonly:1; + bool backend_readonly_set:1; +}; + +struct mbox_transaction_context { + struct mailbox_transaction_context t; + union mail_index_transaction_module_context module_ctx; + + unsigned int read_lock_id; + unsigned int write_lock_id; +}; + +#define MBOX_STORAGE(s) container_of(s, struct mbox_storage, storage) +#define MBOX_MAILBOX(s) container_of(s, struct mbox_mailbox, box) +#define MBOX_TRANSCTX(s) container_of(s, struct mbox_transaction_context, t) + +extern struct mail_vfuncs mbox_mail_vfuncs; +extern const char *mbox_hide_headers[], *mbox_save_drop_headers[]; +extern unsigned int mbox_hide_headers_count, mbox_save_drop_headers_count; + +void mbox_set_syscall_error(struct mbox_mailbox *mbox, const char *function); +void mbox_istream_set_syscall_error(struct mbox_mailbox *mbox, + struct istream *input, + const char *function); +void mbox_ostream_set_syscall_error(struct mbox_mailbox *mbox, + struct ostream *output, + const char *function); + +struct mailbox_sync_context * +mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +struct mail_save_context * +mbox_save_alloc(struct mailbox_transaction_context *_t); +int mbox_save_begin(struct mail_save_context *ctx, struct istream *input); +int mbox_save_continue(struct mail_save_context *ctx); +int mbox_save_finish(struct mail_save_context *ctx); +void mbox_save_cancel(struct mail_save_context *ctx); + +int mbox_transaction_save_commit_pre(struct mail_save_context *ctx); +void mbox_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void mbox_transaction_save_rollback(struct mail_save_context *ctx); + +bool mbox_is_backend_readonly(struct mbox_mailbox *mbox); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-sync-list-index.c b/src/lib-storage/index/mbox/mbox-sync-list-index.c new file mode 100644 index 0000000..934b7ea --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-list-index.c @@ -0,0 +1,109 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" + +static unsigned int +mbox_list_get_ext_id(struct mbox_mailbox *mbox, + struct mail_index_view *view) +{ + if (mbox->mbox_list_index_ext_id == (uint32_t)-1) { + mbox->mbox_list_index_ext_id = + mail_index_ext_register(mail_index_view_get_index(view), + "mbox", 0, + sizeof(struct mbox_list_index_record), + sizeof(uint32_t)); + } + return mbox->mbox_list_index_ext_id; +} + +int mbox_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, const char **reason_r) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + const struct mbox_list_index_record *rec; + const void *data; + const char *path; + struct stat st; + uint32_t ext_id; + bool expunged; + int ret; + + ret = index_storage_list_index_has_changed(box, list_view, seq, + quick, reason_r); + if (ret != 0 || box->storage->set->mailbox_list_index_very_dirty_syncs) + return ret; + + ext_id = mbox_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + rec = data; + + if (rec == NULL) { + *reason_r = "mbox record is missing"; + return 1; + } else if (expunged) { + *reason_r = "mbox record is expunged"; + return 1; + } else if (rec->mtime == 0) { + /* not synced */ + *reason_r = "mbox record mtime=0"; + return 1; + } + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path); + if (ret < 0) + return ret; + i_assert(ret > 0); + + if (stat(path, &st) < 0) { + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return -1; + } + if ((time_t)rec->mtime != st.st_mtime) { + *reason_r = t_strdup_printf( + "mbox record mtime changed %u != %"PRIdTIME_T, + rec->mtime, st.st_mtime); + return 1; + } + uint32_t new_size = (uint32_t)(st.st_size & 0xffffffffU); + if (rec->size != new_size) { + *reason_r = t_strdup_printf("mbox record size changed %u != %u", + rec->size, new_size); + return 1; + } + return 0; +} + +void mbox_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + struct mail_index_view *list_view; + const struct mbox_index_header *mhdr = &mbox->mbox_hdr; + const struct mbox_list_index_record *old_rec; + struct mbox_list_index_record new_rec; + const void *data; + uint32_t ext_id; + bool expunged; + + index_storage_list_index_update_sync(box, trans, seq); + + /* get the current record */ + list_view = mail_index_transaction_get_view(trans); + ext_id = mbox_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + if (expunged) + return; + old_rec = data; + + i_zero(&new_rec); + new_rec.mtime = mhdr->sync_mtime; + new_rec.size = mhdr->sync_size & 0xffffffffU; + + if (old_rec == NULL || + memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0) + mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL); +} diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c new file mode 100644 index 0000000..bbc2da2 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c @@ -0,0 +1,616 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* MD5 header summing logic was pretty much copy&pasted from popa3d by + Solar Designer */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "istream.h" +#include "str.h" +#include "write-full.h" +#include "message-parser.h" +#include "mail-index.h" +#include "mbox-storage.h" +#include "mbox-md5.h" +#include "mbox-sync-private.h" + + +#define IS_LWSP_LF(c) (IS_LWSP(c) || (c) == '\n') + +struct mbox_sync_header_func { + const char *header; + bool (*func)(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr); +}; + +struct mbox_flag_type mbox_status_flags[] = { + { 'R', MAIL_SEEN }, + { 'O', MBOX_NONRECENT_KLUDGE }, + { 0, 0 } +}; + +struct mbox_flag_type mbox_xstatus_flags[] = { + { 'A', MAIL_ANSWERED }, + { 'F', MAIL_FLAGGED }, + { 'T', MAIL_DRAFT }, + { 'D', MAIL_DELETED }, + { 0, 0 } +}; + +static void parse_trailing_whitespace(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + size_t i, space = 0; + + /* the value may contain newlines. we can't count whitespace before + and after it as a single contiguous whitespace block, as that may + get us into situation where removing whitespace goes eg. + " \n \n" -> " \n\n" which would then be treated as end of headers. + + that could probably be avoided by being careful, but as newlines + should never be there (we don't generate them), it's not worth the + trouble. */ + + for (i = hdr->full_value_len; i > 0; i--) { + if (!IS_LWSP(hdr->full_value[i-1])) + break; + space++; + } + + if ((ssize_t)space > ctx->mail.space) { + i_assert(space != 0); + ctx->mail.offset = ctx->hdr_offset + str_len(ctx->header) + i; + ctx->mail.space = space; + } +} + +static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr) +{ + int i; + + for (i = 0; flags[i].chr != 0; i++) { + if (flags[i].chr == chr) + return flags[i].flag; + } + + return 0; +} + +static bool parse_status_flags(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr, + struct mbox_flag_type *flags_list) +{ + enum mail_flags flag; + size_t i; + bool duplicates = FALSE; + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + for (i = 0; i < hdr->full_value_len; i++) { + flag = mbox_flag_find(flags_list, hdr->full_value[i]); + if ((ctx->mail.flags & flag) != 0) + duplicates = TRUE; + else + ctx->mail.flags |= flag; + } + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + return duplicates; +} + +static bool parse_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + if (parse_status_flags(ctx, hdr, mbox_status_flags)) + ctx->mail.status_broken = TRUE; + ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); + return TRUE; +} + +static bool parse_x_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + if (parse_status_flags(ctx, hdr, mbox_xstatus_flags)) + ctx->mail.xstatus_broken = TRUE; + ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header); + return TRUE; +} + +static void +parse_imap_keywords_list(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr, size_t pos) +{ + struct mailbox *box = &ctx->sync_ctx->mbox->box; + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const char *keyword, *error; + size_t keyword_start; + unsigned int idx, count; + + count = 0; + while (pos < hdr->full_value_len) { + if (IS_LWSP_LF(hdr->full_value[pos])) { + pos++; + continue; + } + + /* read the keyword */ + keyword_start = pos; + for (; pos < hdr->full_value_len; pos++) { + if (IS_LWSP_LF(hdr->full_value[pos])) + break; + } + + /* add it to index's keyword list if it's not there already */ + keyword = t_strndup(hdr->full_value + keyword_start, + pos - keyword_start); + if (mailbox_keyword_is_valid(&ctx->sync_ctx->mbox->box, + keyword, &error)) { + mail_index_keyword_lookup_or_create(box->index, + keyword, &idx); + } + count++; + } + + if (count != array_count(ibox->keyword_names)) { + /* need to update this list */ + ctx->imapbase_rewrite = TRUE; + ctx->need_rewrite = TRUE; + } +} + +static bool parse_x_imap_base(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + size_t i, j, uid_last_pos; + uint32_t uid_validity, uid_last; + + if (ctx->seq != 1 || ctx->seen_imapbase || + ctx->sync_ctx->renumber_uids) { + /* Valid only in first message */ + return FALSE; + } + + /* <uid-validity> 10x<uid-last> */ + for (i = 0, uid_validity = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') { + if (hdr->full_value[i] != ' ') + return FALSE; + break; + } + uid_validity = uid_validity * 10 + (hdr->full_value[i] - '0'); + } + + if (uid_validity == 0) { + /* broken */ + return FALSE; + } + + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) + break; + } + uid_last_pos = i; + + for (uid_last = 0, j = 0; i < hdr->full_value_len; i++, j++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') { + if (!IS_LWSP_LF(hdr->full_value[i])) + return FALSE; + break; + } + uid_last = uid_last * 10 + (hdr->full_value[i] - '0'); + } + + if (j != 10 || + hdr->full_value_offset != ctx->hdr_offset + str_len(ctx->header)) { + /* uid-last field must be exactly 10 characters to make + rewriting it easier. also don't try to do this if some + headers have been removed */ + ctx->imapbase_rewrite = TRUE; + ctx->need_rewrite = TRUE; + } else { + ctx->last_uid_value_start_pos = uid_last_pos; + ctx->sync_ctx->base_uid_last_offset = + hdr->full_value_offset + uid_last_pos; + } + + if (ctx->sync_ctx->base_uid_validity == 0) { + /* first time parsing this (ie. we're not rewriting). + save the values. */ + ctx->sync_ctx->base_uid_validity = uid_validity; + ctx->sync_ctx->base_uid_last = uid_last; + + if (ctx->sync_ctx->next_uid-1 <= uid_last) { + /* new messages have been added since our last sync. + just update our internal next_uid. */ + ctx->sync_ctx->next_uid = uid_last+1; + } else { + /* we need to rewrite the next-uid */ + ctx->need_rewrite = TRUE; + } + i_assert(ctx->sync_ctx->next_uid > ctx->sync_ctx->prev_msg_uid); + } + + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); + ctx->seen_imapbase = TRUE; + + T_BEGIN { + parse_imap_keywords_list(ctx, hdr, i); + } T_END; + parse_trailing_whitespace(ctx, hdr); + return TRUE; +} + +static bool parse_x_imap(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + if (!parse_x_imap_base(ctx, hdr)) + return FALSE; + + /* this is the c-client style "FOLDER INTERNAL DATA" message. + skip it. */ + ctx->mail.pseudo = TRUE; + return TRUE; +} + +static bool parse_x_keywords_real(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + struct mailbox *box = &ctx->sync_ctx->mbox->box; + ARRAY_TYPE(keyword_indexes) keyword_list; + const unsigned int *list; + string_t *keyword; + size_t keyword_start; + unsigned int i, idx, count; + size_t pos; + + if (array_is_created(&ctx->mail.keywords)) + return FALSE; /* duplicate header, delete */ + + /* read keyword indexes to temporary array first */ + keyword = t_str_new(128); + t_array_init(&keyword_list, 16); + + for (pos = 0; pos < hdr->full_value_len; ) { + if (IS_LWSP_LF(hdr->full_value[pos])) { + pos++; + continue; + } + + /* read the keyword string */ + keyword_start = pos; + for (; pos < hdr->full_value_len; pos++) { + if (IS_LWSP_LF(hdr->full_value[pos])) + break; + } + + str_truncate(keyword, 0); + str_append_data(keyword, hdr->full_value + keyword_start, + pos - keyword_start); + if (!mail_index_keyword_lookup(box->index, str_c(keyword), + &idx)) { + /* keyword wasn't found. that means the sent mail + originally contained X-Keywords header. Delete it. */ + return FALSE; + } + + /* check that the keyword isn't already added there. + we don't want duplicates. */ + list = array_get(&keyword_list, &count); + for (i = 0; i < count; i++) { + if (list[i] == idx) + break; + } + + if (i == count) + array_push_back(&keyword_list, &idx); + } + + /* once we know how many keywords there are, we can allocate the array + from mail_keyword_pool without wasting memory. */ + if (array_count(&keyword_list) > 0) { + p_array_init(&ctx->mail.keywords, + ctx->sync_ctx->mail_keyword_pool, + array_count(&keyword_list)); + array_append_array(&ctx->mail.keywords, &keyword_list); + } + + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); + parse_trailing_whitespace(ctx, hdr); + return TRUE; +} + +static bool parse_x_keywords(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + bool ret; + + T_BEGIN { + ret = parse_x_keywords_real(ctx, hdr); + } T_END; + return ret; +} + +static bool parse_x_uid(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uint32_t value = 0; + size_t i; + + if (ctx->mail.uid != 0) { + /* duplicate */ + return FALSE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) { + /* broken value */ + return FALSE; + } + } + + if (ctx->sync_ctx == NULL) { + /* we're in mbox_sync_parse_match_mail(). + don't do any extra checks. */ + ctx->mail.uid = value; + return TRUE; + } + + if (ctx->seq == 1 && !ctx->seen_imapbase) { + /* Don't bother allowing X-UID before X-IMAPbase + header. c-client doesn't allow it either, and this + way the UID doesn't have to be reset if X-IMAPbase + header isn't what we expect it to be. */ + return FALSE; + } + + if (value == ctx->sync_ctx->next_uid) { + /* X-UID is the next expected one. allow it because + we'd just use this UID anyway. X-IMAPbase header + still needs to be updated for this. */ + ctx->sync_ctx->next_uid++; + } else if (value > ctx->sync_ctx->next_uid) { + /* UID is larger than expected. Don't allow it because + incoming mails can contain untrusted X-UID fields, + causing possibly DoS if the UIDs get large enough. */ + ctx->mail.uid_broken = TRUE; + return FALSE; + } + + if (value <= ctx->sync_ctx->prev_msg_uid) { + /* broken - UIDs must be growing */ + ctx->mail.uid_broken = TRUE; + return FALSE; + } + + ctx->mail.uid = value; + /* if we had multiple X-UID headers, we could have + uid_broken=TRUE here. */ + ctx->mail.uid_broken = FALSE; + + if (ctx->sync_ctx->dest_first_mail && ctx->seq != 1) { + /* if we're expunging the first mail, delete this header since + otherwise X-IMAPbase header would be added after this, which + we don't like */ + return FALSE; + } + + ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); + ctx->parsed_uid = value; + parse_trailing_whitespace(ctx, hdr); + return TRUE; +} + +static bool parse_content_length(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uoff_t value = 0; + size_t i; + + if (ctx->content_length != UOFF_T_MAX) { + /* duplicate */ + return FALSE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) { + /* broken value */ + return FALSE; + } + } + + ctx->content_length = value; + return TRUE; +} + +static struct mbox_sync_header_func header_funcs[] = { + { "Content-Length", parse_content_length }, + { "Status", parse_status }, + { "X-IMAP", parse_x_imap }, + { "X-IMAPbase", parse_x_imap_base }, + { "X-Keywords", parse_x_keywords }, + { "X-Status", parse_x_status }, + { "X-UID", parse_x_uid } +}; + +static int mbox_sync_bsearch_header_func_cmp(const void *p1, const void *p2) +{ + const char *key = p1; + const struct mbox_sync_header_func *func = p2; + + return strcasecmp(key, func->header); +} + +int mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx) +{ + struct mbox_sync_context *sync_ctx = ctx->sync_ctx; + struct message_header_parser_ctx *hdr_ctx; + struct message_header_line *hdr; + struct mbox_sync_header_func *func; + struct mbox_md5_context *mbox_md5_ctx; + size_t line_start_pos; + int i, ret; + + ctx->hdr_offset = ctx->mail.offset; + ctx->mail.flags = MAIL_RECENT; /* default to having recent flag */ + + ctx->header_first_change = SIZE_MAX; + ctx->header_last_change = 0; + + for (i = 0; i < MBOX_HDR_COUNT; i++) + ctx->hdr_pos[i] = SIZE_MAX; + + ctx->content_length = UOFF_T_MAX; + str_truncate(ctx->header, 0); + + mbox_md5_ctx = ctx->sync_ctx->mbox->md5_v.init(); + + line_start_pos = 0; + hdr_ctx = message_parse_header_init(input, NULL, 0); + while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) { + if (hdr->eoh) { + ctx->have_eoh = TRUE; + break; + } + + if (!hdr->continued) { + line_start_pos = str_len(ctx->header); + str_append(ctx->header, hdr->name); + str_append_data(ctx->header, hdr->middle, hdr->middle_len); + } + + func = bsearch(hdr->name, header_funcs, + N_ELEMENTS(header_funcs), sizeof(*header_funcs), + mbox_sync_bsearch_header_func_cmp); + + if (func != NULL) { + if (hdr->continues) { + hdr->use_full_value = TRUE; + continue; + } + + if (!func->func(ctx, hdr)) { + /* this header is broken, remove it */ + ctx->need_rewrite = TRUE; + str_truncate(ctx->header, line_start_pos); + if (ctx->header_first_change == SIZE_MAX) { + ctx->header_first_change = + line_start_pos; + } + continue; + } + buffer_append(ctx->header, hdr->full_value, + hdr->full_value_len); + } else { + ctx->sync_ctx->mbox->md5_v.more(mbox_md5_ctx, hdr); + buffer_append(ctx->header, hdr->value, + hdr->value_len); + } + if (!hdr->no_newline) { + if (hdr->crlf_newline) + str_append_c(ctx->header, '\r'); + str_append_c(ctx->header, '\n'); + } + } + i_assert(ret != 0); + message_parse_header_deinit(&hdr_ctx); + + ctx->sync_ctx->mbox->md5_v.finish(mbox_md5_ctx, ctx->hdr_md5_sum); + + if ((ctx->seq == 1 && !ctx->seen_imapbase) || + (ctx->seq > 1 && sync_ctx->dest_first_mail)) { + /* missing X-IMAPbase */ + ctx->need_rewrite = TRUE; + if (sync_ctx->base_uid_validity == 0) { + /* figure out a new UIDVALIDITY for us. */ + sync_ctx->base_uid_validity = + sync_ctx->hdr->uid_validity != 0 && + !sync_ctx->renumber_uids ? + sync_ctx->hdr->uid_validity : + I_MAX((uint32_t)ioloop_time, 1); + } + } + + ctx->body_offset = input->v_offset; + if (input->stream_errno != 0) { + mbox_sync_set_critical(ctx->sync_ctx, "read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); + return -1; + } + return 0; +} + +bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox, + struct mail_index_view *view, uint32_t seq) +{ + struct mbox_sync_mail_context ctx; + struct message_header_parser_ctx *hdr_ctx; + struct message_header_line *hdr; + struct header_func *func; + struct mbox_md5_context *mbox_md5_ctx; + const void *data; + bool expunged; + uint32_t uid; + int ret; + + /* we only wish to be sure that this mail actually is what we expect + it to be. If there's X-UID header and it matches our UID, we use it. + Otherwise it could mean that the X-UID header is invalid and it's + just not yet been rewritten. In that case use MD5 sum, if it + exists. */ + + mail_index_lookup_uid(view, seq, &uid); + i_zero(&ctx); + mbox_md5_ctx = mbox->md5_v.init(); + + hdr_ctx = message_parse_header_init(mbox->mbox_stream, NULL, 0); + while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) { + if (hdr->eoh) + break; + + func = bsearch(hdr->name, header_funcs, + N_ELEMENTS(header_funcs), sizeof(*header_funcs), + mbox_sync_bsearch_header_func_cmp); + if (func != NULL) { + if (strcasecmp(hdr->name, "X-UID") == 0) { + if (hdr->continues) { + hdr->use_full_value = TRUE; + continue; + } + (void)parse_x_uid(&ctx, hdr); + + if (ctx.mail.uid == uid) + break; + } + } else { + mbox->md5_v.more(mbox_md5_ctx, hdr); + } + } + i_assert(ret != 0); + message_parse_header_deinit(&hdr_ctx); + + mbox->md5_v.finish(mbox_md5_ctx, ctx.hdr_md5_sum); + + if (ctx.mail.uid == uid) + return TRUE; + + /* match by MD5 sum */ + mbox->mbox_save_md5 = TRUE; + + mail_index_lookup_ext(view, seq, mbox->md5hdr_ext_idx, + &data, &expunged); + return data == NULL ? 0 : + memcmp(data, ctx.hdr_md5_sum, 16) == 0; +} diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h new file mode 100644 index 0000000..7585e4d --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-private.h @@ -0,0 +1,192 @@ +#ifndef MBOX_SYNC_PRIVATE_H +#define MBOX_SYNC_PRIVATE_H + +#include "md5.h" +#include "mail-index.h" + +#include <sys/stat.h> + +enum mbox_sync_flags { + MBOX_SYNC_HEADER = 0x02, + MBOX_SYNC_LOCK_READING = 0x04, + MBOX_SYNC_UNDIRTY = 0x08, + MBOX_SYNC_REWRITE = 0x10, + MBOX_SYNC_FORCE_SYNC = 0x20, + MBOX_SYNC_READONLY = 0x40 +}; + +struct mbox_flag_type { + char chr; + enum mail_flags flag; +}; + +enum header_position { + MBOX_HDR_STATUS, + MBOX_HDR_X_IMAPBASE, + MBOX_HDR_X_KEYWORDS, + MBOX_HDR_X_STATUS, + MBOX_HDR_X_UID, + + MBOX_HDR_COUNT +}; + +/* kludgy. swap MAIL_RECENT with MBOX_NONRECENT_KLUDGE when writing Status + header, because 'O' flag means non-recent but internally we want to use + recent flag. */ +#define MBOX_NONRECENT_KLUDGE MAIL_RECENT + +#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT_KLUDGE) +#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED) +extern struct mbox_flag_type mbox_status_flags[]; +extern struct mbox_flag_type mbox_xstatus_flags[]; + +struct mbox_sync_mail { + /* uid=0 can mean that this mail describes an expunged area or that + this is a pseudo message */ + uint32_t uid; + uint32_t idx_seq; + + ARRAY_TYPE(keyword_indexes) keywords; + uint8_t flags; + + bool uid_broken:1; + bool expunged:1; + bool pseudo:1; + bool status_broken:1; + bool xstatus_broken:1; + + uoff_t from_offset; + uoff_t body_size; + + /* following variables have a bit overloaded functionality: + + a) space <= 0 : offset points to beginning of headers. space is the + amount of space missing that is required to be able to rewrite + the headers + b) space > 0 : offset points to beginning of whitespace that can + be removed. space is the amount of data that can be removed from + there. note that the message may contain more whitespace + elsewhere. */ + uoff_t offset; + off_t space; +}; + +struct mbox_sync_mail_context { + struct mbox_sync_context *sync_ctx; + struct mbox_sync_mail mail; + + uint32_t seq; + uoff_t hdr_offset, body_offset; + + size_t header_first_change, header_last_change; + string_t *header; + + unsigned char hdr_md5_sum[16]; + + uoff_t content_length; + + size_t hdr_pos[MBOX_HDR_COUNT]; + uint32_t parsed_uid, last_uid_updated_value; + unsigned int last_uid_value_start_pos; + + bool have_eoh:1; + bool need_rewrite:1; + bool seen_imapbase:1; + bool updated:1; + bool recent:1; + bool dirty:1; + bool imapbase_rewrite:1; + bool imapbase_updated:1; +}; + +struct mbox_sync_context { + struct mbox_mailbox *mbox; + enum mbox_sync_flags flags; + struct istream *input, *file_input; + int write_fd; + + time_t orig_mtime, orig_atime; + uoff_t orig_size; + struct stat last_stat; + + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *t; + + struct mail_index_header reset_hdr; + const struct mail_index_header *hdr; + + string_t *header, *from_line; + + /* header state: */ + uint32_t base_uid_validity, base_uid_last; + uoff_t base_uid_last_offset; + + /* mail state: */ + ARRAY(struct mbox_sync_mail) mails; + struct index_sync_changes_context *sync_changes; + + /* per-mail pool */ + pool_t mail_keyword_pool; + /* used for mails[].keywords */ + pool_t saved_keywords_pool; + + uint32_t prev_msg_uid, next_uid, idx_next_uid; + uint32_t seq, idx_seq, need_space_seq; + uint32_t last_nonrecent_uid; + off_t expunged_space, space_diff; + + bool dest_first_mail:1; + bool first_mail_crlf_expunged:1; + + /* global flags: */ + bool keep_recent:1; + bool readonly:1; + bool delay_writes:1; + bool renumber_uids:1; + bool moved_offsets:1; + bool ext_modified:1; + bool index_reset:1; + bool errors:1; +}; + +int mbox_sync_header_refresh(struct mbox_mailbox *mbox); +int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags); +int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty); +void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx, + const char *fmt, ...) ATTR_FORMAT(2, 3); + +int mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx); +bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox, + struct mail_index_view *view, uint32_t seq); + +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx); +void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail); +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff); +int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + uoff_t end_offset, off_t move_diff, uoff_t extra_space, + uint32_t first_seq, uint32_t last_seq); + +int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset); +void mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx); +void mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty); +int mbox_move(struct mbox_sync_context *sync_ctx, + uoff_t dest, uoff_t source, uoff_t size); +void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx, + size_t pos, size_t need, size_t have); +void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size); +int mbox_sync_get_guid(struct mbox_mailbox *mbox); + +int mbox_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, + const char **reason_r); +void mbox_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq); + +#endif diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c new file mode 100644 index 0000000..eeb4878 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c @@ -0,0 +1,615 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "write-full.h" +#include "message-parser.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" +#include "istream-raw-mbox.h" + +int mbox_move(struct mbox_sync_context *sync_ctx, + uoff_t dest, uoff_t source, uoff_t size) +{ + struct mbox_mailbox *mbox = sync_ctx->mbox; + struct istream *input; + struct ostream *output; + int ret; + + i_assert(source > 0 || (dest != 1 && dest != 2)); + i_assert(size < OFF_T_MAX); + + if (size == 0 || source == dest) + return 0; + + i_stream_sync(sync_ctx->input); + + output = o_stream_create_fd_file(sync_ctx->write_fd, UOFF_T_MAX, FALSE); + i_stream_seek(sync_ctx->file_input, source); + if (o_stream_seek(output, dest) < 0) { + mbox_ostream_set_syscall_error(sync_ctx->mbox, output, + "o_stream_seek()"); + o_stream_unref(&output); + return -1; + } + + /* we're moving data within a file. it really shouldn't be failing at + this point or we're corrupted. */ + input = i_stream_create_limit(sync_ctx->file_input, size); + o_stream_nsend_istream(output, input); + if (input->stream_errno != 0) { + mailbox_set_critical(&mbox->box, + "read() failed with mbox: %s", + i_stream_get_error(input)); + ret = -1; + } else if (output->stream_errno != 0) { + mailbox_set_critical(&mbox->box, + "write() failed with mbox: %s", + o_stream_get_error(output)); + ret = -1; + } else if (input->v_offset != size) { + mbox_sync_set_critical(sync_ctx, + "mbox_move(%"PRIuUOFF_T", %"PRIuUOFF_T", %"PRIuUOFF_T + ") moved only %"PRIuUOFF_T" bytes", + dest, source, size, input->v_offset); + ret = -1; + } else { + ret = 0; + } + i_stream_unref(&input); + + mbox_sync_file_updated(sync_ctx, FALSE); + o_stream_destroy(&output); + return ret; +} + +static int mbox_fill_space(struct mbox_sync_context *sync_ctx, + uoff_t offset, uoff_t size) +{ + unsigned char space[1024]; + + memset(space, ' ', sizeof(space)); + while (size > sizeof(space)) { + if (pwrite_full(sync_ctx->write_fd, space, + sizeof(space), offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + size -= sizeof(space); + } + + if (pwrite_full(sync_ctx->write_fd, space, size, offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + mbox_sync_file_updated(sync_ctx, TRUE); + return 0; +} + +void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + size_t data_size, pos, start_pos; + const unsigned char *data; + void *p; + + i_assert(size < SSIZE_T_MAX); + + if (ctx->mail.pseudo) + start_pos = ctx->hdr_pos[MBOX_HDR_X_IMAPBASE]; + else if (ctx->mail.space > 0) { + /* update the header using the existing offset. + otherwise we might chose wrong header and just decrease + the available space */ + start_pos = ctx->mail.offset - ctx->hdr_offset; + } else { + /* Append at the end of X-Keywords header, + or X-UID if it doesn't exist */ + start_pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != SIZE_MAX ? + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] : + ctx->hdr_pos[MBOX_HDR_X_UID]; + } + + data = str_data(ctx->header); + data_size = str_len(ctx->header); + i_assert(start_pos < data_size); + + for (pos = start_pos; pos < data_size; pos++) { + if (data[pos] == '\n') { + /* possibly continues in next line */ + if (pos+1 == data_size || !IS_LWSP(data[pos+1])) + break; + start_pos = pos+1; + } else if (!IS_LWSP(data[pos]) && data[pos] != '\r') { + start_pos = pos+1; + } + } + + /* pos points to end of header now, and start_pos to beginning + of whitespace. */ + mbox_sync_move_buffer(ctx, pos, size, 0); + + p = buffer_get_space_unsafe(ctx->header, pos, size); + memset(p, ' ', size); + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + ctx->header_last_change = SIZE_MAX; + + ctx->mail.space = (pos - start_pos) + size; + ctx->mail.offset = ctx->hdr_offset; + if (ctx->mail.space > 0) + ctx->mail.offset += start_pos; +} + +static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx, + size_t start_pos, size_t *size) +{ + const unsigned char *data; + size_t data_size, pos, last_line_pos; + + /* find the end of the LWSP */ + data = str_data(ctx->header); + data_size = str_len(ctx->header); + + for (pos = last_line_pos = start_pos; pos < data_size; pos++) { + if (data[pos] == '\n') { + /* possibly continues in next line */ + if (pos+1 == data_size || !IS_LWSP(data[pos+1])) { + data_size = pos; + break; + } + last_line_pos = pos+1; + } else if (!IS_LWSP(data[pos]) && data[pos] != '\r') { + start_pos = last_line_pos = pos+1; + } + } + + if (start_pos == data_size) + return; + + /* and remove what we can */ + if (ctx->header_first_change > start_pos) + ctx->header_first_change = start_pos; + ctx->header_last_change = SIZE_MAX; + + if (data_size - start_pos <= *size) { + /* remove it all */ + mbox_sync_move_buffer(ctx, start_pos, 0, data_size - start_pos); + *size -= data_size - start_pos; + return; + } + + /* we have more space than needed. since we're removing from + the beginning of header instead of end, we don't have to + worry about multiline-headers. */ + mbox_sync_move_buffer(ctx, start_pos, 0, *size); + if (last_line_pos <= start_pos + *size) + last_line_pos = start_pos; + else + last_line_pos -= *size; + data_size -= *size; + + *size = 0; + + if (ctx->mail.space < (off_t)(data_size - last_line_pos)) { + ctx->mail.space = data_size - last_line_pos; + ctx->mail.offset = ctx->hdr_offset; + if (ctx->mail.space > 0) + ctx->mail.offset += last_line_pos; + } +} + +static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + static enum header_position space_positions[] = { + MBOX_HDR_X_UID, + MBOX_HDR_X_KEYWORDS, + MBOX_HDR_X_IMAPBASE + }; + enum header_position pos; + int i; + + ctx->mail.space = 0; + ctx->mail.offset = ctx->hdr_offset; + + for (i = 0; i < 3 && size > 0; i++) { + pos = space_positions[i]; + if (ctx->hdr_pos[pos] != SIZE_MAX) { + mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos], + &size); + } + } + + /* FIXME: see if we could remove X-Keywords header completely */ +} + +static void mbox_sync_first_mail_written(struct mbox_sync_mail_context *ctx, + uoff_t hdr_offset) +{ + /* we wrote the first mail. update last-uid offset so we can find + it later */ + i_assert(ctx->last_uid_value_start_pos != 0); + i_assert(ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] != SIZE_MAX); + + ctx->sync_ctx->base_uid_last_offset = hdr_offset + + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] + + ctx->last_uid_value_start_pos; + + if (ctx->imapbase_updated) { + /* update so a) we don't try to update it later needlessly, + b) if we do actually update it, we see the correct value */ + ctx->sync_ctx->base_uid_last = ctx->last_uid_updated_value; + } +} + +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff) +{ + struct mbox_sync_context *sync_ctx = ctx->sync_ctx; + size_t old_hdr_size, new_hdr_size; + + i_assert(sync_ctx->mbox->mbox_lock_type == F_WRLCK); + + old_hdr_size = ctx->body_offset - ctx->hdr_offset; + new_hdr_size = str_len(ctx->header); + + if (new_hdr_size <= old_hdr_size) { + /* add space. note that we must call add_space() even if we're + not adding anything so mail.offset gets fixed. */ + mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size); + } else if (new_hdr_size > old_hdr_size) { + /* try removing the space where we can */ + mbox_sync_headers_remove_space(ctx, + new_hdr_size - old_hdr_size); + new_hdr_size = str_len(ctx->header); + + if (new_hdr_size <= old_hdr_size) { + /* good, we removed enough. */ + i_assert(new_hdr_size == old_hdr_size); + } else if (move_diff < 0 && + new_hdr_size - old_hdr_size <= (uoff_t)-move_diff) { + /* moving backwards - we can use the extra space from + it, just update expunged_space accordingly */ + i_assert(ctx->mail.space == 0); + i_assert(sync_ctx->expunged_space >= + (off_t)(new_hdr_size - old_hdr_size)); + sync_ctx->expunged_space -= new_hdr_size - old_hdr_size; + } else { + /* couldn't get enough space */ + i_assert(ctx->mail.space == 0); + ctx->mail.space = + -(ssize_t)(new_hdr_size - old_hdr_size); + return 0; + } + } + + i_assert(ctx->mail.space >= 0); + + if (ctx->header_first_change == SIZE_MAX && move_diff == 0) { + /* no changes actually. we get here if index sync record told + us to do something that was already there */ + return 1; + } + + if (move_diff != 0) { + /* forget about partial write optimizations */ + ctx->header_first_change = 0; + ctx->header_last_change = 0; + } + + if (ctx->header_last_change != SIZE_MAX && + ctx->header_last_change != 0) + str_truncate(ctx->header, ctx->header_last_change); + + if (pwrite_full(sync_ctx->write_fd, + str_data(ctx->header) + ctx->header_first_change, + str_len(ctx->header) - ctx->header_first_change, + (off_t)ctx->hdr_offset + (off_t)ctx->header_first_change + + move_diff) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + + if (sync_ctx->dest_first_mail && + (ctx->imapbase_updated || ctx->sync_ctx->base_uid_last != 0)) { + /* the position might have moved as a result of moving + whitespace */ + mbox_sync_first_mail_written(ctx, (off_t)ctx->hdr_offset + move_diff); + } + + mbox_sync_file_updated(sync_ctx, FALSE); + return 1; +} + +static int mbox_sync_read_next(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + struct mbox_sync_mail *mails, + uint32_t seq, uint32_t idx, + uoff_t expunged_space) +{ + unsigned int first_mail_expunge_extra; + uint32_t orig_next_uid; + + i_zero(mail_ctx); + mail_ctx->sync_ctx = sync_ctx; + mail_ctx->seq = seq; + mail_ctx->header = sync_ctx->header; + + if (istream_raw_mbox_get_header_offset(sync_ctx->input, + &mail_ctx->mail.offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get header offset for seq=%u", seq); + return -1; + } + mail_ctx->mail.body_size = mails[idx].body_size; + + orig_next_uid = sync_ctx->next_uid; + if (mails[idx].uid != 0) { + /* This will force the UID to be the one that we originally + assigned to it, regardless of whether it's broken or not in + the file. */ + sync_ctx->next_uid = mails[idx].uid; + sync_ctx->prev_msg_uid = mails[idx].uid - 1; + } else { + /* Pseudo mail shouldn't have X-UID header at all */ + i_assert(mails[idx].pseudo); + sync_ctx->prev_msg_uid = 0; + } + + first_mail_expunge_extra = 1 + + (sync_ctx->first_mail_crlf_expunged ? 1 : 0); + if (mails[idx].from_offset + + first_mail_expunge_extra - expunged_space != 0) { + sync_ctx->dest_first_mail = mails[idx].from_offset == 0; + } else { + /* we need to skip over the initial \n (it's already counted in + expunged_space) */ + sync_ctx->dest_first_mail = TRUE; + mails[idx].from_offset += first_mail_expunge_extra; + } + + if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0) + return -1; + i_assert(mail_ctx->mail.pseudo == mails[idx].pseudo); + + /* set next_uid back before updating the headers. this is important + if we're updating the first message to make X-IMAP[base] header + have the correct value. */ + sync_ctx->next_uid = orig_next_uid; + + if (mails[idx].space != 0) { + if (mails[idx].space < 0) { + /* remove all possible spacing before updating */ + mbox_sync_headers_remove_space(mail_ctx, SIZE_MAX); + } + mbox_sync_update_header_from(mail_ctx, &mails[idx]); + } else { + /* updating might just try to add headers and mess up our + calculations completely. so only add the EOH here. */ + if (mail_ctx->have_eoh) + str_append_c(mail_ctx->header, '\n'); + } + return 0; +} + +static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + struct mbox_sync_mail *mails, + uint32_t seq, uint32_t idx, uint32_t padding, + off_t move_diff, uoff_t expunged_space, + uoff_t end_offset, bool first_nonexpunged) +{ + struct mbox_sync_mail_context new_mail_ctx; + uoff_t offset, dest_offset; + size_t need_space; + + if (mail_ctx == NULL) { + if (mbox_sync_seek(sync_ctx, mails[idx].from_offset) < 0) + return -1; + + if (mbox_sync_read_next(sync_ctx, &new_mail_ctx, mails, seq, idx, + expunged_space) < 0) + return -1; + mail_ctx = &new_mail_ctx; + } else { + i_assert(seq == mail_ctx->seq); + if (mail_ctx->mail.space < 0) + mail_ctx->mail.space = 0; + i_stream_seek(sync_ctx->input, mail_ctx->body_offset); + } + + if (mail_ctx->mail.space <= 0) { + need_space = str_len(mail_ctx->header) - mail_ctx->mail.space - + (mail_ctx->body_offset - mail_ctx->hdr_offset); + if (need_space != (uoff_t)-mails[idx].space) { + /* this check works only if we're doing the first + write, or if the file size was changed externally */ + mbox_sync_file_update_ext_modified(sync_ctx); + + mbox_sync_set_critical(sync_ctx, + "seq=%u uid=%u uid_broken=%d " + "originally needed %"PRIuUOFF_T + " bytes, now needs %zu bytes", + seq, mails[idx].uid, mails[idx].uid_broken ? 1 : 0, + (uoff_t)-mails[idx].space, need_space); + return -1; + } + } + + if (first_nonexpunged && expunged_space > 0) { + /* move From-line (after parsing headers so we don't + overwrite them) */ + i_assert(mails[idx].from_offset >= expunged_space); + if (mbox_move(sync_ctx, mails[idx].from_offset - expunged_space, + mails[idx].from_offset, + mails[idx].offset - mails[idx].from_offset) < 0) + return -1; + } + + if (mails[idx].space == 0) { + /* don't touch spacing */ + } else if (padding < (uoff_t)mail_ctx->mail.space) { + mbox_sync_headers_remove_space(mail_ctx, mail_ctx->mail.space - + padding); + } else { + mbox_sync_headers_add_space(mail_ctx, padding - + mail_ctx->mail.space); + } + + /* move the body of this message and headers of next message forward, + then write the headers */ + offset = sync_ctx->input->v_offset; + dest_offset = offset + move_diff; + i_assert(offset <= end_offset); + if (mbox_move(sync_ctx, dest_offset, offset, end_offset - offset) < 0) + return -1; + + /* the header may actually be moved backwards if there was expunged + space which we wanted to remove */ + i_assert(dest_offset >= str_len(mail_ctx->header)); + dest_offset -= str_len(mail_ctx->header); + i_assert(dest_offset >= mails[idx].from_offset - expunged_space); + if (pwrite_full(sync_ctx->write_fd, str_data(mail_ctx->header), + str_len(mail_ctx->header), dest_offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + mbox_sync_file_updated(sync_ctx, TRUE); + + if (sync_ctx->dest_first_mail) { + mbox_sync_first_mail_written(mail_ctx, dest_offset); + sync_ctx->dest_first_mail = FALSE; + } + + mails[idx].offset = dest_offset + + (mail_ctx->mail.offset - mail_ctx->hdr_offset); + mails[idx].space = mail_ctx->mail.space; + return 0; +} + +int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + uoff_t end_offset, off_t move_diff, uoff_t extra_space, + uint32_t first_seq, uint32_t last_seq) +{ + struct mbox_sync_mail *mails; + uoff_t offset, dest_offset, next_end_offset, next_move_diff; + uoff_t start_offset, expunged_space; + uint32_t idx, first_nonexpunged_idx, padding_per_mail; + uint32_t orig_prev_msg_uid; + unsigned int count; + int ret = 0; + + i_assert(extra_space < OFF_T_MAX); + i_assert(sync_ctx->mbox->mbox_lock_type == F_WRLCK); + + mails = array_get_modifiable(&sync_ctx->mails, &count); + i_assert(count == last_seq - first_seq + 1); + + /* if there's expunges in mails[], we would get more correct balancing + by counting only them here. however, that might make us overwrite + data which hasn't yet been copied backwards. to avoid too much + complexity, we just leave all the rest of the extra space to first + mail */ + idx = last_seq - first_seq + 1; + padding_per_mail = extra_space / idx; + + /* after expunge the next mail must have been missing space, or we + would have moved it backwards already */ + expunged_space = 0; + start_offset = mails[0].from_offset; + for (first_nonexpunged_idx = 0;; first_nonexpunged_idx++) { + i_assert(first_nonexpunged_idx != idx); + if (!mails[first_nonexpunged_idx].expunged) + break; + expunged_space += mails[first_nonexpunged_idx].space; + } + i_assert(mails[first_nonexpunged_idx].space < 0); + + orig_prev_msg_uid = sync_ctx->prev_msg_uid; + + /* start moving backwards. */ + while (idx > first_nonexpunged_idx) { + idx--; + if (idx == first_nonexpunged_idx) { + /* give the rest of the extra space to first mail. + we might also have to move the mail backwards to + fill the expunged space */ + padding_per_mail = move_diff + (off_t)expunged_space + + (off_t)mails[idx].space; + } + + next_end_offset = mails[idx].offset; + + if (mails[idx].space <= 0 && !mails[idx].expunged) { + /* give space to this mail. end_offset is left to + contain this message's From-line (ie. below we + move only headers + body). */ + bool first_nonexpunged = idx == first_nonexpunged_idx; + + next_move_diff = -mails[idx].space; + if (mbox_sync_read_and_move(sync_ctx, mail_ctx, mails, + first_seq + idx, idx, + padding_per_mail, + move_diff, expunged_space, + end_offset, + first_nonexpunged) < 0) { + ret = -1; + break; + } + move_diff -= next_move_diff + mails[idx].space; + } else { + /* this mail provides more space. just move it forward + from the extra space offset and set end_offset to + point to beginning of extra space. that way the + header will be moved along with previous mail's + body. + + if this is expunged mail, we're moving following + mail's From-line and maybe headers. */ + offset = mails[idx].offset + mails[idx].space; + dest_offset = offset + move_diff; + i_assert(offset <= end_offset); + if (mbox_move(sync_ctx, dest_offset, offset, + end_offset - offset) < 0) { + ret = -1; + break; + } + + move_diff += mails[idx].space; + if (!mails[idx].expunged) { + move_diff -= padding_per_mail; + mails[idx].space = padding_per_mail; + + if (mbox_fill_space(sync_ctx, move_diff + + mails[idx].offset, + padding_per_mail) < 0) { + ret = -1; + break; + } + } + mails[idx].offset += move_diff; + } + mail_ctx = NULL; + + i_assert(move_diff >= 0 || idx == first_nonexpunged_idx); + i_assert(next_end_offset <= end_offset); + + end_offset = next_end_offset; + mails[idx].from_offset += move_diff; + } + + if (ret == 0) { + i_assert(mails[idx].from_offset == start_offset); + i_assert(move_diff + (off_t)expunged_space >= 0); + } + + mbox_sync_file_updated(sync_ctx, FALSE); + sync_ctx->prev_msg_uid = orig_prev_msg_uid; + return ret; +} diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c new file mode 100644 index 0000000..8442013 --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync-update.c @@ -0,0 +1,466 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "str.h" +#include "message-parser.h" +#include "index-storage.h" +#include "index-sync-changes.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" + +/* Line length when to wrap X-IMAP, X-IMAPbase and X-Keywords headers to next + line. Keep this pretty long, as after we wrap we lose compatibility with + UW-IMAP */ +#define KEYWORD_WRAP_LINE_LENGTH 1024 + +static void status_flags_append(struct mbox_sync_mail_context *ctx, + const struct mbox_flag_type *flags_list) +{ + int i; + + for (i = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + str_append_c(ctx->header, flags_list[i].chr); + } +} + +void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx, + size_t pos, size_t need, size_t have) +{ + ssize_t diff = (ssize_t)need - (ssize_t)have; + int i; + + i_assert(have < SSIZE_T_MAX); + + if (diff == 0) { + if (ctx->header_last_change < pos + have || + ctx->header_last_change == SIZE_MAX) + ctx->header_last_change = pos + have; + } else { + /* FIXME: if (diff < ctx->space && pos < ctx->offset) then + move the data only up to space offset and give/take the + space from there. update header_last_change accordingly. + (except pos and offset can't be compared directly) */ + ctx->header_last_change = SIZE_MAX; + for (i = 0; i < MBOX_HDR_COUNT; i++) { + if (ctx->hdr_pos[i] > pos && + ctx->hdr_pos[i] != SIZE_MAX) + ctx->hdr_pos[i] = (ssize_t)ctx->hdr_pos[i] + diff; + } + + if (ctx->mail.space > 0) { + i_assert(ctx->mail.offset + ctx->mail.space <= + ctx->hdr_offset + pos || + ctx->mail.offset > ctx->hdr_offset + pos + have); + if (ctx->mail.offset > ctx->hdr_offset + pos) { + /* free space offset moves */ + ctx->mail.offset = (ssize_t)ctx->mail.offset + diff; + } + } + + if (diff < 0) + str_delete(ctx->header, pos, -diff); + else { + ctx->header_last_change = SIZE_MAX; + buffer_copy(ctx->header, pos + diff, + ctx->header, pos, SIZE_MAX); + } + } +} + +static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos, + const struct mbox_flag_type *flags_list) +{ + unsigned char *data; + size_t size; + int i, need, have; + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + + /* how many bytes do we need? */ + for (i = 0, need = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + need++; + } + + /* how many bytes do we have now? */ + data = buffer_get_modifiable_data(ctx->header, &size); + for (have = 0; pos < size; pos++) { + if (data[pos] == '\n' || data[pos] == '\r') + break; + + /* see if this is unknown flag for us */ + for (i = 0; flags_list[i].chr != 0; i++) { + if (flags_list[i].chr == (char)data[pos]) + break; + } + + if (flags_list[i].chr != 0) + have++; + else { + /* save this one */ + data[pos-have] = data[pos]; + } + } + pos -= have; + mbox_sync_move_buffer(ctx, pos, need, have); + + /* @UNSAFE */ + data = buffer_get_space_unsafe(ctx->header, pos, need); + for (i = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + *data++ = flags_list[i].chr; + } + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; +} + +static void +keywords_append(struct mbox_sync_context *sync_ctx, string_t *dest, + const ARRAY_TYPE(keyword_indexes) *keyword_indexes_arr) +{ + struct index_mailbox_context *ibox = + INDEX_STORAGE_CONTEXT(&sync_ctx->mbox->box); + const char *const *keyword_names; + const unsigned int *keyword_indexes; + unsigned int i, idx_count, keywords_count; + size_t last_break; + + keyword_names = array_get(ibox->keyword_names, + &keywords_count); + keyword_indexes = array_get(keyword_indexes_arr, &idx_count); + + for (i = 0, last_break = str_len(dest); i < idx_count; i++) { + i_assert(keyword_indexes[i] < keywords_count); + + /* wrap the line whenever it gets too long */ + if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH) { + if (i > 0) + str_append_c(dest, ' '); + } else { + str_append(dest, "\n\t"); + last_break = str_len(dest); + } + str_append(dest, keyword_names[keyword_indexes[i]]); + } +} + +static void +keywords_append_all(struct mbox_sync_mail_context *ctx, string_t *dest, + size_t startpos) +{ + struct index_mailbox_context *ibox = + INDEX_STORAGE_CONTEXT(&ctx->sync_ctx->mbox->box); + const char *const *names; + const unsigned char *p; + unsigned int i, count; + size_t last_break; + + p = str_data(dest); + if (str_len(dest) - startpos < KEYWORD_WRAP_LINE_LENGTH) + last_break = startpos; + else { + /* set last_break to beginning of line */ + for (last_break = str_len(dest); last_break > 0; last_break--) { + if (p[last_break-1] == '\n') + break; + } + } + + names = array_get(ibox->keyword_names, &count); + for (i = 0; i < count; i++) { + /* wrap the line whenever it gets too long */ + if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH) + str_append_c(dest, ' '); + else { + str_append(dest, "\n\t"); + last_break = str_len(dest); + } + str_append(dest, names[i]); + } +} + +static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) +{ + size_t new_hdr_size, startpos; + + new_hdr_size = str_len(ctx->header); + if (new_hdr_size > 0 && + str_data(ctx->header)[new_hdr_size-1] != '\n') { + /* broken header - doesn't end with \n. fix it. */ + str_append_c(ctx->header, '\n'); + } + + if (ctx->sync_ctx->dest_first_mail && + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX) { + i_assert(ctx->sync_ctx->base_uid_validity != 0); + + str_append(ctx->header, "X-IMAPbase: "); + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); + /* startpos must start from identical position as when + updating */ + startpos = str_len(ctx->header); + str_printfa(ctx->header, "%u ", + ctx->sync_ctx->base_uid_validity); + + ctx->last_uid_updated_value = ctx->sync_ctx->next_uid-1; + ctx->last_uid_value_start_pos = str_len(ctx->header) - + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE]; + ctx->imapbase_updated = TRUE; + str_printfa(ctx->header, "%010u", ctx->last_uid_updated_value); + + keywords_append_all(ctx, ctx->header, startpos); + str_append_c(ctx->header, '\n'); + } + + if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX && !ctx->mail.pseudo) { + str_append(ctx->header, "X-UID: "); + ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); + str_printfa(ctx->header, "%u\n", ctx->mail.uid); + } + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + + if (ctx->hdr_pos[MBOX_HDR_STATUS] == SIZE_MAX && + (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) { + str_append(ctx->header, "Status: "); + ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); + status_flags_append(ctx, mbox_status_flags); + str_append_c(ctx->header, '\n'); + } + + if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == SIZE_MAX && + (ctx->mail.flags & XSTATUS_FLAGS_MASK) != 0) { + str_append(ctx->header, "X-Status: "); + ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header); + status_flags_append(ctx, mbox_xstatus_flags); + str_append_c(ctx->header, '\n'); + } + + ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE; + + if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX && + array_is_created(&ctx->mail.keywords) && + array_count(&ctx->mail.keywords) > 0) { + str_append(ctx->header, "X-Keywords: "); + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); + keywords_append(ctx->sync_ctx, ctx->header, + &ctx->mail.keywords); + str_append_c(ctx->header, '\n'); + } + + if (ctx->content_length == UOFF_T_MAX && + ctx->mail.body_size >= MBOX_MIN_CONTENT_LENGTH_SIZE) { + str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n", + ctx->mail.body_size); + } + + if (str_len(ctx->header) != new_hdr_size) { + if (ctx->header_first_change == SIZE_MAX) + ctx->header_first_change = new_hdr_size; + ctx->header_last_change = SIZE_MAX; + } + + if (ctx->have_eoh) + str_append_c(ctx->header, '\n'); +} + +static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx) +{ + if (ctx->hdr_pos[MBOX_HDR_STATUS] != SIZE_MAX) { + status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_STATUS], + mbox_status_flags); + } +} + +static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx) +{ + if (ctx->hdr_pos[MBOX_HDR_X_STATUS] != SIZE_MAX) { + status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_X_STATUS], + mbox_xstatus_flags); + } +} + +static void mbox_sync_update_line(struct mbox_sync_mail_context *ctx, + size_t pos, string_t *new_line) +{ + const char *hdr, *p; + uoff_t file_pos; + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + + /* set p = end of header, handle also wrapped headers */ + hdr = p = str_c(ctx->header) + pos; + for (;;) { + p = strchr(p, '\n'); + if (p == NULL) { + /* shouldn't really happen, but allow anyway.. */ + p = hdr + strlen(hdr); + break; + } + if (p[1] != '\t' && p[1] != ' ') + break; + p += 2; + } + + file_pos = pos + ctx->hdr_offset; + if (ctx->mail.space > 0 && ctx->mail.offset >= file_pos && + ctx->mail.offset < file_pos + (p - hdr)) { + /* extra space points to this line. remove it. */ + ctx->mail.offset = ctx->hdr_offset; + ctx->mail.space = 0; + } + + mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1); + buffer_copy(ctx->header, pos, new_line, 0, SIZE_MAX); +} + +static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx) +{ + string_t *str; + + if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX) + return; + + str = t_str_new(256); + if (array_is_created(&ctx->mail.keywords)) + keywords_append(ctx->sync_ctx, str, &ctx->mail.keywords); + str_append_c(str, '\n'); + mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_KEYWORDS], str); +} + +static void mbox_sync_update_x_imap_base(struct mbox_sync_mail_context *ctx) +{ + struct mbox_sync_context *sync_ctx = ctx->sync_ctx; + string_t *str; + + i_assert(sync_ctx->base_uid_validity != 0); + + if (!sync_ctx->dest_first_mail || + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX) + return; + + if (!ctx->imapbase_rewrite) { + /* uid-last might need updating, but we'll do it later by + writing it directly where needed. */ + return; + } + + /* a) keyword list changed, b) uid-last didn't use 10 digits */ + str = t_str_new(200); + str_printfa(str, "%u ", sync_ctx->base_uid_validity); + + ctx->last_uid_updated_value = sync_ctx->next_uid-1; + ctx->last_uid_value_start_pos = str_len(str); + ctx->imapbase_updated = TRUE; + str_printfa(str, "%010u", ctx->last_uid_updated_value); + + keywords_append_all(ctx, str, 0); + str_append_c(str, '\n'); + + mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_IMAPBASE], str); +} + +static void mbox_sync_update_x_uid(struct mbox_sync_mail_context *ctx) +{ + string_t *str; + + if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX || + ctx->mail.uid == ctx->parsed_uid) + return; + + str = t_str_new(64); + str_printfa(str, "%u\n", ctx->mail.uid); + mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_UID], str); +} + +static void mbox_sync_update_header_real(struct mbox_sync_mail_context *ctx) +{ + i_assert(ctx->mail.uid != 0 || ctx->mail.pseudo); + + if (!ctx->sync_ctx->keep_recent) + ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT); + + mbox_sync_update_status(ctx); + mbox_sync_update_xstatus(ctx); + mbox_sync_update_xkeywords(ctx); + + mbox_sync_update_x_imap_base(ctx); + mbox_sync_update_x_uid(ctx); + + mbox_sync_add_missing_headers(ctx); + ctx->updated = TRUE; +} + +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx) +{ + T_BEGIN { + mbox_sync_update_header_real(ctx); + } T_END; +} + +static void +mbox_sync_update_header_from_real(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail) +{ + if (mail->status_broken || + (ctx->mail.flags & STATUS_FLAGS_MASK) != + (mail->flags & STATUS_FLAGS_MASK) || + (ctx->mail.flags & MAIL_RECENT) != 0) { + ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(STATUS_FLAGS_MASK)) | + (mail->flags & STATUS_FLAGS_MASK); + if (!ctx->sync_ctx->keep_recent) + ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT); + mbox_sync_update_status(ctx); + } + if (mail->xstatus_broken || + (ctx->mail.flags & XSTATUS_FLAGS_MASK) != + (mail->flags & XSTATUS_FLAGS_MASK)) { + ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(XSTATUS_FLAGS_MASK)) | + (mail->flags & XSTATUS_FLAGS_MASK); + mbox_sync_update_xstatus(ctx); + } + if (!array_is_created(&mail->keywords) || + array_count(&mail->keywords) == 0) { + /* no keywords for this mail */ + if (array_is_created(&ctx->mail.keywords)) { + array_clear(&ctx->mail.keywords); + mbox_sync_update_xkeywords(ctx); + } + } else if (!array_is_created(&ctx->mail.keywords)) { + /* adding first keywords */ + p_array_init(&ctx->mail.keywords, + ctx->sync_ctx->mail_keyword_pool, + array_count(&mail->keywords)); + array_append_array(&ctx->mail.keywords, + &mail->keywords); + mbox_sync_update_xkeywords(ctx); + } else if (!array_cmp(&ctx->mail.keywords, &mail->keywords)) { + /* keywords changed. */ + array_clear(&ctx->mail.keywords); + array_append_array(&ctx->mail.keywords, + &mail->keywords); + mbox_sync_update_xkeywords(ctx); + } + + i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid); + ctx->mail.uid = mail->uid; + + mbox_sync_update_x_imap_base(ctx); + mbox_sync_update_x_uid(ctx); + mbox_sync_add_missing_headers(ctx); +} + +void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail) +{ + T_BEGIN { + mbox_sync_update_header_from_real(ctx, mail); + } T_END; +} diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c new file mode 100644 index 0000000..0d2aa7f --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -0,0 +1,2066 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* + Modifying mbox can be slow, so we try to do it all at once minimizing the + required disk I/O. We may need to: + + - Update message flags in Status, X-Status and X-Keywords headers + - Write missing X-UID and X-IMAPbase headers + - Write missing or broken Content-Length header if there's space + - Expunge specified messages + + Here's how we do it: + + - Start reading the mails from the beginning + - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end + of them, remember how much each message has and offset to beginning of the + padding + - If header needs to be rewritten and there's enough space, do it + - If we didn't have enough space, remember how much was missing + - Continue reading and counting the padding in each message. If available + padding is enough to rewrite all the previous messages needing it, do it + - When we encounter expunged message, treat all of it as padding and + rewrite previous messages if needed (and there's enough space). + Afterwards keep moving messages backwards to fill the expunged space. + Moving is done by rewriting each message's headers, with possibly adding + missing Content-Length header and padding. Message bodies are moved + without modifications. + - If we encounter end of file, grow the file and rewrite needed messages + - Rewriting is done by moving message body forward, rewriting message's + header and doing the same for previous message, until all of them are + rewritten. +*/ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "hostpid.h" +#include "istream.h" +#include "file-set-size.h" +#include "str.h" +#include "read-full.h" +#include "write-full.h" +#include "sleep.h" +#include "message-date.h" +#include "istream-raw-mbox.h" +#include "mbox-storage.h" +#include "index-sync-changes.h" +#include "mailbox-uidvalidity.h" +#include "mailbox-recent-flags.h" +#include "mbox-from.h" +#include "mbox-file.h" +#include "mbox-lock.h" +#include "mbox-sync-private.h" + +#include <stddef.h> +#include <utime.h> +#include <sys/stat.h> + +/* The text below was taken exactly as c-client wrote it to my mailbox, + so it's probably copyrighted by University of Washington. */ +#define PSEUDO_MESSAGE_BODY \ +"This text is part of the internal format of your mail folder, and is not\n" \ +"a real message. It is created automatically by the mail system software.\n" \ +"If deleted, important folder data will be lost, and it will be re-created\n" \ +"with the data reset to initial values.\n" + +void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx, + const char *fmt, ...) +{ + va_list va; + + sync_ctx->errors = TRUE; + if (sync_ctx->ext_modified) { + mailbox_set_critical(&sync_ctx->mbox->box, + "mbox was modified while we were syncing, " + "check your locking settings"); + } + + va_start(va, fmt); + mailbox_set_critical(&sync_ctx->mbox->box, + "Sync failed for mbox: %s", + t_strdup_vprintf(fmt, va)); + va_end(va); +} + +int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset) +{ + if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Unexpectedly lost From-line at offset %"PRIuUOFF_T, + from_offset); + return -1; + } + return 0; +} + +void mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx) +{ + struct stat st; + + /* Do this even if ext_modified is already set. Expunging code relies + on last_stat being updated. */ + if (fstat(sync_ctx->write_fd, &st) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "fstat()"); + return; + } + + if (st.st_size != sync_ctx->last_stat.st_size || + (sync_ctx->last_stat.st_mtime != 0 && + !CMP_ST_MTIME(&st, &sync_ctx->last_stat))) + sync_ctx->ext_modified = TRUE; + + sync_ctx->last_stat = st; +} + +void mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty) +{ + if (dirty) { + /* just mark the stat as dirty. */ + sync_ctx->last_stat.st_mtime = 0; + return; + } + if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "fstat()"); + i_stream_sync(sync_ctx->input); +} + +static int +mbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + uoff_t offset; + + /* get EOF */ + (void)istream_raw_mbox_get_header_offset(sync_ctx->input, &offset); + if (istream_raw_mbox_is_eof(sync_ctx->input)) + return 0; + + p_clear(sync_ctx->mail_keyword_pool); + i_zero(mail_ctx); + mail_ctx->sync_ctx = sync_ctx; + mail_ctx->seq = ++sync_ctx->seq; + mail_ctx->header = sync_ctx->header; + + mail_ctx->mail.from_offset = + istream_raw_mbox_get_start_offset(sync_ctx->input); + if (istream_raw_mbox_get_header_offset(sync_ctx->input, &mail_ctx->mail.offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get header offset for seq=%u", mail_ctx->seq); + return -1; + } + + if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0) + return -1; + if (istream_raw_mbox_is_corrupted(sync_ctx->input)) + return -1; + + i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset || + sync_ctx->input->eof); + + if (istream_raw_mbox_get_body_size(sync_ctx->input, + mail_ctx->content_length, + &mail_ctx->mail.body_size) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get body size for seq=%u", mail_ctx->seq); + return -1; + } + i_assert(mail_ctx->mail.body_size < OFF_T_MAX); + + if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 && + !mail_ctx->mail.pseudo) { + if (!sync_ctx->keep_recent) { + /* need to add 'O' flag to Status-header */ + mail_ctx->need_rewrite = TRUE; + } + mail_ctx->recent = TRUE; + } + return 1; +} + +static void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx, + uint32_t uid, bool *sync_expunge_r) +{ + guid_128_t expunged_guid_128; + + if (uid == 0 || sync_ctx->index_reset) { + /* nothing for this or the future ones */ + uid = (uint32_t)-1; + } + + index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r, + expunged_guid_128); + if (sync_ctx->readonly) { + /* we can't expunge anything from read-only mboxes */ + *sync_expunge_r = FALSE; + } +} + +static bool +mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx, + uint32_t uid, const struct mail_index_record **rec_r) +{ + const struct mail_index_record *rec = NULL; + uint32_t messages_count; + bool ret = FALSE; + + if (sync_ctx->index_reset) { + *rec_r = NULL; + return TRUE; + } + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + while (sync_ctx->idx_seq <= messages_count) { + rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq); + if (uid <= rec->uid) + break; + + /* externally expunged message, remove from index */ + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq); + sync_ctx->idx_seq++; + rec = NULL; + } + + if (rec == NULL && uid < sync_ctx->idx_next_uid) { + /* this UID was already in index and it was expunged */ + mbox_sync_set_critical(sync_ctx, + "Expunged message reappeared to mailbox " + "(UID %u < %u, seq=%u, idx_msgs=%u)", + uid, sync_ctx->idx_next_uid, + sync_ctx->seq, messages_count); + ret = FALSE; rec = NULL; + } else if (rec != NULL && rec->uid != uid) { + /* new UID in the middle of the mailbox - shouldn't happen */ + mbox_sync_set_critical(sync_ctx, + "UID inserted in the middle of mailbox " + "(%u > %u, seq=%u, idx_msgs=%u)", + rec->uid, uid, sync_ctx->seq, messages_count); + ret = FALSE; rec = NULL; + } else { + ret = TRUE; + } + + *rec_r = rec; + return ret; +} + +static void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx, + unsigned char hdr_md5_sum[], + const struct mail_index_record **rec_r) +{ + const struct mail_index_record *rec = NULL; + uint32_t messages_count; + const void *data; + + if (sync_ctx->index_reset) { + *rec_r = NULL; + return; + } + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + while (sync_ctx->idx_seq <= messages_count) { + rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq); + mail_index_lookup_ext(sync_ctx->sync_view, + sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + &data, NULL); + if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0) + break; + + /* externally expunged message, remove from index */ + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq); + sync_ctx->idx_seq++; + rec = NULL; + } + + *rec_r = rec; +} + +static void +mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail *mail, + bool nocheck) +{ + const void *data; + uint64_t offset; + + if (!nocheck) { + /* see if from_offset needs updating */ + mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq, + sync_ctx->mbox->mbox_ext_idx, + &data, NULL); + if (data != NULL && + *((const uint64_t *)data) == mail->from_offset) + return; + } + + offset = mail->from_offset; + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->mbox_ext_idx, &offset, NULL); +} + +static void +mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mail_index *index = sync_ctx->mbox->box.index; + struct mail_keywords *keywords; + + keywords = !array_is_created(&mail_ctx->mail.keywords) ? + mail_index_keywords_create(index, NULL) : + mail_index_keywords_create_from_indexes(index, + &mail_ctx->mail.keywords); + mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, keywords); + mail_index_keywords_unref(&keywords); +} + +static void +mbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + const void *ext_data; + + mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, &ext_data, NULL); + if (ext_data == NULL || + memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) { + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + mail_ctx->hdr_md5_sum, NULL); + } +} + +static void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + ARRAY_TYPE(keyword_indexes) idx_keywords; + uint8_t idx_flags, mbox_flags; + + /* default to undirtying the message. it gets added back if + flags/keywords don't match what is in the index. */ + mail_ctx->mail.flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY); + + /* replace flags */ + idx_flags = rec->flags & MAIL_FLAGS_NONRECENT; + mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT; + if (idx_flags != mbox_flags) { + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) | + idx_flags | MAIL_INDEX_MAIL_FLAG_DIRTY; + } + + /* replace keywords */ + t_array_init(&idx_keywords, 32); + mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq, + &idx_keywords); + if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) { + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + + if (!array_is_created(&mail_ctx->mail.keywords)) { + p_array_init(&mail_ctx->mail.keywords, + sync_ctx->mail_keyword_pool, + array_count(&idx_keywords)); + } + array_clear(&mail_ctx->mail.keywords); + array_append_array(&mail_ctx->mail.keywords, &idx_keywords); + } +} + +static void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mailbox *box = &sync_ctx->mbox->box; + struct mbox_sync_mail *mail = &mail_ctx->mail; + enum mail_index_sync_type sync_type; + ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT; + uint8_t flags, orig_flags; + + if (rec != NULL) { + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* flags and keywords are dirty. replace the current + ones from the flags in index file. */ + mbox_sync_get_dirty_flags(mail_ctx, rec); + } + } + + flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT; + if (array_is_created(&mail->keywords)) { + t_array_init(&orig_keywords, 32); + array_append_array(&orig_keywords, &mail->keywords); + } + + /* apply new changes */ + index_sync_changes_apply(sync_ctx->sync_changes, + sync_ctx->mail_keyword_pool, + &flags, &mail->keywords, &sync_type); + if (flags != orig_flags || + !index_keyword_array_cmp(&mail->keywords, &orig_keywords)) { + mail_ctx->need_rewrite = TRUE; + mail->flags = flags | (mail->flags & MAIL_RECENT) | + MAIL_INDEX_MAIL_FLAG_DIRTY; + } + if (sync_type != 0) { + mailbox_sync_notify(box, mail_ctx->mail.uid, + index_sync_type_convert(sync_type)); + } +} + +static void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mbox_sync_mail *mail = &mail_ctx->mail; + ARRAY_TYPE(keyword_indexes) idx_keywords; + uint8_t mbox_flags; + + mbox_flags = mail->flags & ENUM_NEGATE(MAIL_RECENT); + if (!sync_ctx->delay_writes) { + /* changes are written to the mbox file */ + mbox_flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY); + } else if (mail_ctx->need_rewrite) { + /* make sure this message gets written later */ + mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + } + + if (rec == NULL) { + /* new message */ + mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq); + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, mbox_flags); + mbox_sync_update_index_keywords(mail_ctx); + + if (sync_ctx->mbox->mbox_save_md5) { + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + mail_ctx->hdr_md5_sum, NULL); + } + } else { + if ((rec->flags & MAIL_FLAGS_NONRECENT) != + (mbox_flags & MAIL_FLAGS_NONRECENT)) { + /* flags other than recent/dirty have changed */ + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, mbox_flags); + } else if (((rec->flags ^ mbox_flags) & + MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* only dirty flag state changed */ + bool dirty; + + dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0; + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + dirty ? MODIFY_ADD : MODIFY_REMOVE, + (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY); + } + + /* see if keywords changed */ + t_array_init(&idx_keywords, 32); + mail_index_lookup_keywords(sync_ctx->sync_view, + sync_ctx->idx_seq, &idx_keywords); + if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords)) + mbox_sync_update_index_keywords(mail_ctx); + + /* see if we need to update md5 sum. */ + if (sync_ctx->mbox->mbox_save_md5) + mbox_sync_update_md5_if_changed(mail_ctx); + } + + if (!mail_ctx->recent) { + /* Mail has "Status: O" header. No messages before this + can be recent. */ + sync_ctx->last_nonrecent_uid = mail->uid; + } + + /* update from_offsets, but not if we're going to rewrite this message. + rewriting would just move it anyway. */ + if (sync_ctx->need_space_seq == 0) { + bool nocheck = rec == NULL || sync_ctx->expunged_space > 0; + mbox_sync_update_from_offset(sync_ctx, mail, nocheck); + } +} + +static int mbox_read_from_line(struct mbox_sync_mail_context *ctx) +{ + struct istream *input = ctx->sync_ctx->file_input; + const unsigned char *data; + size_t size, from_line_size; + + buffer_set_used_size(ctx->sync_ctx->from_line, 0); + from_line_size = ctx->hdr_offset - ctx->mail.from_offset; + + i_stream_seek(input, ctx->mail.from_offset); + for (;;) { + data = i_stream_get_data(input, &size); + if (size >= from_line_size) + size = from_line_size; + + buffer_append(ctx->sync_ctx->from_line, data, size); + i_stream_skip(input, size); + from_line_size -= size; + + if (from_line_size == 0) + break; + + if (i_stream_read(input) < 0) + return -1; + } + + return 0; +} + +static int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx) +{ + unsigned char buf[10]; + const char *str; + uint32_t uid_last; + unsigned int i; + int ret; + + i_assert(sync_ctx->base_uid_last_offset != 0); + + /* first check that the 10 bytes are there and they're exactly as + expected. just an extra safety check to make sure we never write + to wrong location in the mbox file. */ + ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf), + sync_ctx->base_uid_last_offset); + if (ret < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pread_full()"); + return -1; + } + if (ret == 0) { + mbox_sync_set_critical(sync_ctx, + "X-IMAPbase uid-last offset unexpectedly outside mbox"); + return -1; + } + + for (i = 0, uid_last = 0; i < sizeof(buf); i++) { + if (buf[i] < '0' || buf[i] > '9') { + uid_last = (uint32_t)-1; + break; + } + uid_last = uid_last * 10 + (buf[i] - '0'); + } + + if (uid_last != sync_ctx->base_uid_last) { + mbox_sync_set_critical(sync_ctx, + "X-IMAPbase uid-last unexpectedly lost"); + return -1; + } + + /* and write it */ + str = t_strdup_printf("%010u", sync_ctx->next_uid - 1); + if (pwrite_full(sync_ctx->write_fd, str, 10, + sync_ctx->base_uid_last_offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + mbox_sync_file_updated(sync_ctx, FALSE); + + sync_ctx->base_uid_last = sync_ctx->next_uid - 1; + return 0; +} + +static int +mbox_write_from_line(struct mbox_sync_mail_context *ctx) +{ + string_t *str = ctx->sync_ctx->from_line; + + if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str), + ctx->mail.from_offset) < 0) { + mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()"); + return -1; + } + + mbox_sync_file_updated(ctx->sync_ctx, FALSE); + return 0; +} + +static void update_from_offsets(struct mbox_sync_context *sync_ctx) +{ + const struct mbox_sync_mail *mails; + unsigned int i, count; + uint32_t ext_idx; + uint64_t offset; + + ext_idx = sync_ctx->mbox->mbox_ext_idx; + + mails = array_get(&sync_ctx->mails, &count); + for (i = 0; i < count; i++) { + if (mails[i].idx_seq == 0 || mails[i].expunged) + continue; + + sync_ctx->moved_offsets = TRUE; + offset = mails[i].from_offset; + mail_index_update_ext(sync_ctx->t, mails[i].idx_seq, + ext_idx, &offset, NULL); + } +} + +static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mailbox *box = &sync_ctx->mbox->box; + + mailbox_sync_notify(box, mail_ctx->mail.uid, + MAILBOX_SYNC_TYPE_EXPUNGE); + mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq); + + mail_ctx->mail.expunged = TRUE; + mail_ctx->mail.offset = mail_ctx->mail.from_offset; + mail_ctx->mail.space = + mail_ctx->body_offset - mail_ctx->mail.from_offset + + mail_ctx->mail.body_size; + mail_ctx->mail.body_size = 0; + mail_ctx->mail.uid = 0; + + if (sync_ctx->seq == 1) { + /* expunging first message, fix space to contain next + message's \n header too since it will be removed. */ + mail_ctx->mail.space++; + if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) { + mail_ctx->mail.space++; + sync_ctx->first_mail_crlf_expunged = TRUE; + } + + /* uid-last offset is invalid now */ + sync_ctx->base_uid_last_offset = 0; + } + + sync_ctx->expunged_space += mail_ctx->mail.space; +} + +static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + uoff_t orig_from_offset, postlf_from_offset = UOFF_T_MAX; + off_t move_diff; + int ret; + + if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) { + /* move the header backwards to fill expunged space */ + move_diff = -sync_ctx->expunged_space; + + orig_from_offset = mail_ctx->mail.from_offset; + if (sync_ctx->dest_first_mail) { + /* we're moving this mail to beginning of file. + skip the initial \n (it's already counted in + expunged_space) */ + mail_ctx->mail.from_offset++; + if (sync_ctx->first_mail_crlf_expunged) + mail_ctx->mail.from_offset++; + } + postlf_from_offset = mail_ctx->mail.from_offset; + + /* read the From-line before rewriting overwrites it */ + if (mbox_read_from_line(mail_ctx) < 0) + return -1; + i_assert((off_t)mail_ctx->mail.from_offset + move_diff != 1 && + (off_t)mail_ctx->mail.from_offset + move_diff != 2); + + mbox_sync_update_header(mail_ctx); + ret = mbox_sync_try_rewrite(mail_ctx, move_diff); + if (ret < 0) + return -1; + + if (ret > 0) { + /* rewrite successful, write From-line to + new location */ + i_assert((off_t)mail_ctx->mail.from_offset >= + -move_diff); + mail_ctx->mail.from_offset = (off_t)mail_ctx->mail.from_offset + move_diff; + mail_ctx->mail.offset = (off_t)mail_ctx->mail.offset + move_diff; + if (mbox_write_from_line(mail_ctx) < 0) + return -1; + } else { + if (sync_ctx->dest_first_mail) { + /* didn't have enough space, move the offset + back so seeking into it doesn't fail */ + mail_ctx->mail.from_offset = orig_from_offset; + } + } + } else if (mail_ctx->need_rewrite) { + mbox_sync_update_header(mail_ctx); + if (sync_ctx->delay_writes && sync_ctx->need_space_seq == 0) { + /* mark it dirty and do it later. we can't do this + if we're in the middle of rewriting acquiring more + space. */ + mail_ctx->dirty = TRUE; + return 0; + } + + if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0) + return -1; + } else { + /* nothing to do */ + return 0; + } + + if (ret == 0 && sync_ctx->need_space_seq == 0) { + /* first mail with no space to write it */ + sync_ctx->need_space_seq = sync_ctx->seq; + sync_ctx->space_diff = 0; + + if (sync_ctx->expunged_space > 0) { + /* create dummy message to describe the expunged data */ + struct mbox_sync_mail mail; + + /* if this is going to be the first mail, increase the + from_offset to point to the beginning of the + From-line, because the previous [CR]LF is already + covered by expunged_space. */ + i_assert(postlf_from_offset != UOFF_T_MAX); + mail_ctx->mail.from_offset = postlf_from_offset; + + i_zero(&mail); + mail.expunged = TRUE; + mail.offset = mail.from_offset = + mail_ctx->mail.from_offset - + sync_ctx->expunged_space; + mail.space = sync_ctx->expunged_space; + + sync_ctx->space_diff = sync_ctx->expunged_space; + sync_ctx->expunged_space = 0; + i_assert(sync_ctx->space_diff < -mail_ctx->mail.space); + + sync_ctx->need_space_seq--; + array_push_back(&sync_ctx->mails, &mail); + } + } + return 0; +} + +static int +mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + uoff_t end_offset, move_diff, extra_space, needed_space; + uint32_t last_seq; + ARRAY_TYPE(keyword_indexes) keywords_copy; + + i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 || + mail_ctx->mail.offset == mail_ctx->hdr_offset); + + if (array_is_created(&mail_ctx->mail.keywords)) { + /* mail's keywords are allocated from a pool that's cleared + for each mail. we'll need to copy it to something more + permanent. */ + p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool, + array_count(&mail_ctx->mail.keywords)); + array_append_array(&keywords_copy, &mail_ctx->mail.keywords); + mail_ctx->mail.keywords = keywords_copy; + } + array_push_back(&sync_ctx->mails, &mail_ctx->mail); + + sync_ctx->space_diff += mail_ctx->mail.space; + if (sync_ctx->space_diff < 0) { + if (sync_ctx->expunged_space > 0) { + i_assert(sync_ctx->expunged_space == + mail_ctx->mail.space); + sync_ctx->expunged_space = 0; + } + return 0; + } + + /* we have enough space now */ + if (mail_ctx->mail.uid == 0) { + /* this message was expunged. fill more or less of the space. + space_diff now consists of a negative "bytes needed" sum, + plus the expunged space of this message. so it contains how + many bytes of _extra_ space we have. */ + i_assert(mail_ctx->mail.space >= sync_ctx->space_diff); + extra_space = MBOX_HEADER_PADDING * + (sync_ctx->seq - sync_ctx->need_space_seq + 1); + needed_space = mail_ctx->mail.space - sync_ctx->space_diff; + if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) { + /* don't waste too much on padding */ + move_diff = needed_space + extra_space; + sync_ctx->expunged_space = + mail_ctx->mail.space - move_diff; + } else { + move_diff = mail_ctx->mail.space; + extra_space = sync_ctx->space_diff; + sync_ctx->expunged_space = 0; + } + last_seq = sync_ctx->seq - 1; + array_pop_back(&sync_ctx->mails); + end_offset = mail_ctx->mail.from_offset; + } else { + /* this message gave enough space from headers. rewriting stops + at the end of this message's headers. */ + sync_ctx->expunged_space = 0; + last_seq = sync_ctx->seq; + end_offset = mail_ctx->body_offset; + + move_diff = 0; + extra_space = sync_ctx->space_diff; + } + + mbox_sync_file_update_ext_modified(sync_ctx); + if (mbox_sync_rewrite(sync_ctx, + last_seq == sync_ctx->seq ? mail_ctx : NULL, + end_offset, move_diff, extra_space, + sync_ctx->need_space_seq, last_seq) < 0) + return -1; + + update_from_offsets(sync_ctx); + + /* mail_ctx may contain wrong data after rewrite, so make sure we + don't try to access it */ + i_zero(mail_ctx); + + sync_ctx->need_space_seq = 0; + sync_ctx->space_diff = 0; + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + return 0; +} + +static int +mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq) +{ + struct mbox_mailbox *mbox = sync_ctx->mbox; + uoff_t old_offset, offset; + uint32_t uid; + int ret; + bool deleted; + + if (seq == 0) { + if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) { + mbox->invalid_mbox_file = TRUE; + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOTPOSSIBLE, + "Mailbox isn't a valid mbox file"); + return -1; + } + seq++; + } else { + old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + + ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted); + if (ret < 0) { + if (deleted) { + mbox_sync_set_critical(sync_ctx, + "Message was expunged unexpectedly"); + } + return -1; + } + if (ret == 0) { + if (istream_raw_mbox_seek(mbox->mbox_stream, + old_offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Error seeking back to original " + "offset %s", dec2str(old_offset)); + return -1; + } + return 0; + } + } + + if (seq <= 1) + uid = 0; + else + mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid); + + sync_ctx->prev_msg_uid = uid; + + /* set to -1, since it's always increased later */ + sync_ctx->seq = seq-1; + if (sync_ctx->seq == 0 && + istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) { + /* this mbox has pseudo mail which contains the X-IMAP header */ + sync_ctx->seq++; + } + + sync_ctx->idx_seq = seq; + sync_ctx->dest_first_mail = sync_ctx->seq == 0; + if (istream_raw_mbox_get_body_offset(sync_ctx->input, &offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Message body offset lookup failed"); + return -1; + } + return 1; +} + +static int +mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid) +{ + struct mail_index_view *sync_view = sync_ctx->sync_view; + uint32_t seq1, seq2; + uoff_t size; + int ret; + + i_assert(!sync_ctx->index_reset); + + if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1, + &seq1, &seq2)) { + /* doesn't exist anymore, seek to end of file */ + ret = i_stream_get_size(sync_ctx->file_input, TRUE, &size); + if (ret < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_get_size()"); + return -1; + } + i_assert(ret != 0); + + if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream, + size) < 0) { + mbox_sync_set_critical(sync_ctx, + "Error seeking to end of mbox"); + return -1; + } + sync_ctx->idx_seq = + mail_index_view_get_messages_count(sync_view) + 1; + return 1; + } + + return mbox_sync_seek_to_seq(sync_ctx, seq1); +} + +static int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx, + uint32_t next_uid, bool *partial, + bool *skipped_mails) +{ + uint32_t messages_count, uid; + int ret; + + i_assert(!sync_ctx->index_reset); + + /* delete sync records up to next message. so if there's still + something left in array, it means the next message needs modifying */ + index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid); + if (index_sync_changes_have(sync_ctx->sync_changes)) + return 1; + + if (sync_ctx->hdr->first_recent_uid <= next_uid && + !sync_ctx->keep_recent) { + /* we'll need to rewrite Status: O headers */ + return 1; + } + + uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes); + + if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid && + (uid > sync_ctx->hdr->first_recent_uid || uid == 0) && + !sync_ctx->keep_recent) { + /* we'll need to rewrite Status: O headers */ + uid = sync_ctx->hdr->first_recent_uid; + } + + if (uid != 0) { + /* we can skip forward to next record which needs updating. */ + if (uid != next_uid) { + *skipped_mails = TRUE; + next_uid = uid; + } + ret = mbox_sync_seek_to_uid(sync_ctx, next_uid); + } else { + /* if there's no sync records left, we can stop. except if + this is a dirty sync, check if there are new messages. */ + if (sync_ctx->mbox->mbox_hdr.dirty_flag == 0) + return 0; + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + if (sync_ctx->seq + 1 != messages_count) { + ret = mbox_sync_seek_to_seq(sync_ctx, messages_count); + *skipped_mails = TRUE; + } else { + ret = 1; + } + *partial = FALSE; + } + + if (ret == 0) { + /* seek failed because the offset is dirty. just ignore and + continue from where we are now. */ + *partial = FALSE; + ret = 1; + } + return ret; +} + +static void mbox_sync_hdr_update(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update; + + if (update->uid_validity != 0) { + sync_ctx->base_uid_validity = update->uid_validity; + mail_ctx->imapbase_rewrite = TRUE; + mail_ctx->need_rewrite = TRUE; + } + if (update->min_next_uid != 0 && + sync_ctx->base_uid_last+1 < update->min_next_uid) { + i_assert(sync_ctx->next_uid <= update->min_next_uid); + sync_ctx->base_uid_last = update->min_next_uid-1; + sync_ctx->next_uid = update->min_next_uid; + mail_ctx->imapbase_rewrite = TRUE; + mail_ctx->need_rewrite = TRUE; + } +} + +static bool mbox_sync_imapbase(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + if (sync_ctx->base_uid_validity != 0 && + sync_ctx->hdr->uid_validity != 0 && + sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) { + i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s", + sync_ctx->hdr->uid_validity, + sync_ctx->base_uid_validity, + mailbox_get_path(&sync_ctx->mbox->box)); + sync_ctx->index_reset = TRUE; + return TRUE; + } + if (sync_ctx->mbox->sync_hdr_update != NULL) + mbox_sync_hdr_update(sync_ctx, mail_ctx); + return FALSE; +} + +static int mbox_sync_loop(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + bool partial) +{ + const struct mail_index_record *rec; + uint32_t uid, messages_count; + uoff_t offset; + int ret; + bool expunged, skipped_mails, uids_broken; + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + + /* always start from first message so we can read X-IMAP or + X-IMAPbase header */ + ret = mbox_sync_seek_to_seq(sync_ctx, 0); + if (ret <= 0) + return ret; + + if (sync_ctx->renumber_uids) { + /* expunge everything */ + while (sync_ctx->idx_seq <= messages_count) { + mail_index_expunge(sync_ctx->t, + sync_ctx->idx_seq++); + } + } + + skipped_mails = uids_broken = FALSE; + while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) { + uid = mail_ctx->mail.uid; + + if (mail_ctx->seq == 1) { + if (mbox_sync_imapbase(sync_ctx, mail_ctx)) { + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + return 0; + } + } + + if (mail_ctx->mail.uid_broken && partial) { + /* UID ordering problems, resync everything to make + sure we get everything right */ + if (sync_ctx->mbox->mbox_hdr.dirty_flag != 0) + return 0; + + mbox_sync_set_critical(sync_ctx, + "UIDs broken with partial sync"); + + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + return 0; + } + if (mail_ctx->mail.uid_broken) + uids_broken = TRUE; + + if (mail_ctx->mail.pseudo) + uid = 0; + + rec = NULL; ret = 1; + if (uid != 0) { + if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec)) + ret = 0; + } + + if (ret == 0) { + /* UID found but it's broken */ + uid = 0; + } else if (uid == 0 && + !mail_ctx->mail.pseudo && + (sync_ctx->delay_writes || + sync_ctx->idx_seq <= messages_count)) { + /* If we can't use/store X-UID header, use MD5 sum. + Also check for existing MD5 sums when we're actually + able to write X-UIDs. */ + sync_ctx->mbox->mbox_save_md5 = TRUE; + + mbox_sync_find_index_md5(sync_ctx, + mail_ctx->hdr_md5_sum, &rec); + if (rec != NULL) + uid = mail_ctx->mail.uid = rec->uid; + } + + /* get all sync records related to this message. with pseudo + message just get the first sync record so we can jump to + it with partial seeking. */ + mbox_sync_read_index_syncs(sync_ctx, + mail_ctx->mail.pseudo ? 1 : uid, + &expunged); + + if (mail_ctx->mail.pseudo) { + /* if it was set, it was for the next message */ + expunged = FALSE; + } else { + if (rec == NULL) { + /* message wasn't found from index. we have to + read everything from now on, no skipping */ + partial = FALSE; + } + } + + if (uid == 0 && !mail_ctx->mail.pseudo) { + /* missing/broken X-UID. all the rest of the mails + need new UIDs. */ + while (sync_ctx->idx_seq <= messages_count) { + mail_index_expunge(sync_ctx->t, + sync_ctx->idx_seq++); + } + + if (sync_ctx->next_uid == (uint32_t)-1) { + /* oh no, we're out of UIDs. this shouldn't + happen normally, so just try to get it fixed + without crashing. */ + mailbox_set_critical(&sync_ctx->mbox->box, + "Out of UIDs, renumbering them in mbox"); + sync_ctx->renumber_uids = TRUE; + return 0; + } + + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.uid = sync_ctx->next_uid++; + } + sync_ctx->prev_msg_uid = mail_ctx->mail.uid; + + if (!mail_ctx->mail.pseudo) + mail_ctx->mail.idx_seq = sync_ctx->idx_seq; + + if (!expunged) { + if (!mail_ctx->mail.pseudo) T_BEGIN { + mbox_sync_update_flags(mail_ctx, rec); + } T_END; + if (mbox_sync_handle_header(mail_ctx) < 0) + return -1; + sync_ctx->dest_first_mail = FALSE; + } else { + mbox_sync_handle_expunge(mail_ctx); + } + + if (!mail_ctx->mail.pseudo) { + if (!expunged) T_BEGIN { + mbox_sync_update_index(mail_ctx, rec); + } T_END; + sync_ctx->idx_seq++; + } + + if (istream_raw_mbox_next(sync_ctx->input, + mail_ctx->mail.body_size) < 0) + return -1; + offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + + if (sync_ctx->need_space_seq != 0) { + if (mbox_sync_handle_missing_space(mail_ctx) < 0) + return -1; + if (mbox_sync_seek(sync_ctx, offset) < 0) + return -1; + } else if (sync_ctx->expunged_space > 0) { + if (!expunged) { + /* move the body */ + mbox_sync_file_update_ext_modified(sync_ctx); + if (mbox_move(sync_ctx, + mail_ctx->body_offset - + sync_ctx->expunged_space, + mail_ctx->body_offset, + mail_ctx->mail.body_size) < 0) + return -1; + if (mbox_sync_seek(sync_ctx, offset) < 0) + return -1; + } + } else if (partial) { + ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1, + &partial, + &skipped_mails); + if (ret <= 0) + break; + } + } + if (ret < 0) + return -1; + + if (istream_raw_mbox_is_eof(sync_ctx->input)) { + /* rest of the messages in index don't exist -> expunge them */ + while (sync_ctx->idx_seq <= messages_count) + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++); + } + + if (!skipped_mails) + sync_ctx->mbox->mbox_hdr.dirty_flag = 0; + sync_ctx->mbox->mbox_broken_offsets = FALSE; + + if (uids_broken && sync_ctx->delay_writes) { + /* once we get around to writing the changes, we'll need to do + a full sync to avoid the "UIDs broken in partial sync" + error */ + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } + return 1; +} + +static int mbox_write_pseudo(struct mbox_sync_context *sync_ctx, bool force) +{ + string_t *str; + unsigned int uid_validity; + + i_assert(sync_ctx->write_fd != -1); + + if (sync_ctx->mbox->sync_hdr_update != NULL) { + const struct mailbox_update *update = + sync_ctx->mbox->sync_hdr_update; + bool change = FALSE; + + if (update->uid_validity != 0) { + sync_ctx->base_uid_validity = update->uid_validity; + change = TRUE; + } + if (update->min_next_uid != 0) { + sync_ctx->base_uid_last = update->min_next_uid-1; + change = TRUE; + } + if (!change && !force) + return 0; + } + + uid_validity = sync_ctx->base_uid_validity != 0 ? + sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity; + i_assert(uid_validity != 0); + + str = t_str_new(1024); + str_printfa(str, "%sDate: %s\n" + "From: Mail System Internal Data <MAILER-DAEMON@%s>\n" + "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA" + "\nMessage-ID: <%s@%s>\n" + "X-IMAP: %u %010u\n" + "Status: RO\n" + "\n" + PSEUDO_MESSAGE_BODY + "\n", + mbox_from_create("MAILER_DAEMON", ioloop_time), + message_date_create(ioloop_time), + my_hostname, dec2str(ioloop_time), my_hostname, + uid_validity, sync_ctx->next_uid-1); + + if (pwrite_full(sync_ctx->write_fd, + str_data(str), str_len(str), 0) < 0) { + if (!ENOSPACE(errno)) { + mbox_set_syscall_error(sync_ctx->mbox, + "pwrite_full()"); + return -1; + } + + /* out of disk space, truncate to empty */ + if (ftruncate(sync_ctx->write_fd, 0) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + } + + sync_ctx->base_uid_validity = uid_validity; + sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */ + sync_ctx->base_uid_last = sync_ctx->next_uid-1; + return 0; +} + +static int mbox_append_zero(struct mbox_sync_context *sync_ctx, + uoff_t orig_file_size, uoff_t count) +{ + char block[IO_BLOCK_SIZE]; + uoff_t offset = orig_file_size; + ssize_t ret = 0; + + memset(block, 0, I_MIN(sizeof(block), count)); + while (count > 0) { + ret = pwrite(sync_ctx->write_fd, block, + I_MIN(sizeof(block), count), offset); + if (ret < 0) + break; + offset += ret; + count -= ret; + } + + if (ret < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite()"); + if (ftruncate(sync_ctx->write_fd, orig_file_size) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + return -1; + } + return 0; +} + +static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + uoff_t file_size, offset, padding, trailer_size; + int ret; + + if (!istream_raw_mbox_is_eof(sync_ctx->input)) { + i_assert(sync_ctx->need_space_seq == 0); + i_assert(sync_ctx->expunged_space == 0); + return 0; + } + + ret = i_stream_get_size(sync_ctx->file_input, TRUE, &file_size); + if (ret < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_get_size()"); + return -1; + } + if (ret == 0) { + /* Not a file - allow anyway */ + return 0; + } + + if (file_size < sync_ctx->file_input->v_offset) { + mbox_sync_set_critical(sync_ctx, + "file size unexpectedly shrank " + "(%"PRIuUOFF_T" vs %"PRIuUOFF_T")", file_size, + sync_ctx->file_input->v_offset); + return -1; + } + trailer_size = file_size - sync_ctx->file_input->v_offset; + i_assert(trailer_size <= 2); + + if (sync_ctx->need_space_seq != 0) { + i_assert(sync_ctx->write_fd != -1); + + i_assert(sync_ctx->space_diff < 0); + padding = MBOX_HEADER_PADDING * + (sync_ctx->seq - sync_ctx->need_space_seq + 1); + sync_ctx->space_diff -= padding; + + i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff); + sync_ctx->space_diff += sync_ctx->expunged_space; + sync_ctx->expunged_space = 0; + + if (mail_ctx->have_eoh && !mail_ctx->updated) + str_append_c(mail_ctx->header, '\n'); + + i_assert(sync_ctx->space_diff < 0); + + if (mbox_append_zero(sync_ctx, file_size, + -sync_ctx->space_diff) < 0) + return -1; + mbox_sync_file_updated(sync_ctx, FALSE); + + if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size, + -sync_ctx->space_diff, padding, + sync_ctx->need_space_seq, + sync_ctx->seq) < 0) + return -1; + + update_from_offsets(sync_ctx); + + sync_ctx->need_space_seq = 0; + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + } + + if (sync_ctx->expunged_space > 0) { + i_assert(sync_ctx->write_fd != -1); + + mbox_sync_file_update_ext_modified(sync_ctx); + + /* copy trailer, then truncate the file */ + file_size = sync_ctx->last_stat.st_size; + if (file_size == (uoff_t)sync_ctx->expunged_space) { + /* everything deleted, the trailer_size still contains + the \n trailer though */ + trailer_size = 0; + } else if (sync_ctx->expunged_space == (off_t)file_size + 1 || + sync_ctx->expunged_space == (off_t)file_size + 2) { + /* everything deleted and we didn't have a proper + trailer. */ + trailer_size = 0; + sync_ctx->expunged_space = file_size; + } + + i_assert(file_size >= sync_ctx->expunged_space + trailer_size); + offset = file_size - sync_ctx->expunged_space - trailer_size; + i_assert(offset == 0 || offset > 31); + + if (mbox_move(sync_ctx, offset, + offset + sync_ctx->expunged_space, + trailer_size) < 0) + return -1; + if (ftruncate(sync_ctx->write_fd, + offset + trailer_size) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + return -1; + } + + if (offset == 0) { + if (mbox_write_pseudo(sync_ctx, TRUE) < 0) + return -1; + } + + sync_ctx->expunged_space = 0; + mbox_sync_file_updated(sync_ctx, FALSE); + } else { + if (file_size == 0 && sync_ctx->mbox->sync_hdr_update != NULL) { + if (mbox_write_pseudo(sync_ctx, FALSE) < 0) + return -1; + } + } + return 0; +} + +static void +mbox_sync_index_update_ext_header(struct mbox_mailbox *mbox, + struct mail_index_transaction *trans) +{ + const struct mailbox_update *update = mbox->sync_hdr_update; + const void *data; + size_t data_size; + + if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) { + memcpy(mbox->mbox_hdr.mailbox_guid, update->mailbox_guid, + sizeof(mbox->mbox_hdr.mailbox_guid)); + } else if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + guid_128_generate(mbox->mbox_hdr.mailbox_guid); + } + + mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx, + &data, &data_size); + if (data_size != sizeof(mbox->mbox_hdr) || + memcmp(data, &mbox->mbox_hdr, data_size) != 0) { + if (data_size != sizeof(mbox->mbox_hdr)) { + /* upgrading from v1.x */ + mail_index_ext_resize(trans, mbox->mbox_ext_idx, + sizeof(mbox->mbox_hdr), + sizeof(uint64_t), + sizeof(uint64_t)); + } + mail_index_update_header_ext(trans, mbox->mbox_ext_idx, + 0, &mbox->mbox_hdr, + sizeof(mbox->mbox_hdr)); + } +} + +static uint32_t mbox_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"MBOX_UIDVALIDITY_FNAME, NULL); + return mailbox_uidvalidity_next(list, path); +} + +static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx) +{ + struct mail_index_view *view; + const struct stat *st; + uint32_t first_recent_uid, seq, seq2; + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + + if (sync_ctx->moved_offsets && + ((uint64_t)st->st_size == sync_ctx->mbox->mbox_hdr.sync_size || + (uint64_t)st->st_size == sync_ctx->orig_size)) { + /* We moved messages inside the mbox file without changing + the file's size. If mtime doesn't change, another process + not using the same index file as us can't know that the file + was changed. So make sure the mtime changes. This should + happen rarely enough that the sleeping doesn't become a + performance problem. + + Note that to do this perfectly safe we should do this wait + whenever mails are moved or expunged, regardless of whether + the file's size changed. That however could become a + performance problem and the consequences of being wrong are + quite minimal (an extra logged error message). */ + while (sync_ctx->orig_mtime == st->st_mtime) { + i_sleep_msecs(500); + if (utime(mailbox_get_path(&sync_ctx->mbox->box), NULL) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, + "utime()"); + return -1; + } + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + } + } + + sync_ctx->mbox->mbox_hdr.sync_mtime = st->st_mtime; + sync_ctx->mbox->mbox_hdr.sync_size = st->st_size; + mbox_sync_index_update_ext_header(sync_ctx->mbox, sync_ctx->t); + + /* only reason not to have UID validity at this point is if the file + is entirely empty. In that case just make up a new one if needed. */ + i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0); + + if (sync_ctx->base_uid_validity == 0) { + sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ? + sync_ctx->hdr->uid_validity : + mbox_get_uidvalidity_next(sync_ctx->mbox->box.list); + } + if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) { + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, uid_validity), + &sync_ctx->base_uid_validity, + sizeof(sync_ctx->base_uid_validity), TRUE); + } + + if (istream_raw_mbox_is_eof(sync_ctx->input) && + sync_ctx->next_uid != sync_ctx->hdr->next_uid) { + i_assert(sync_ctx->next_uid != 0); + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, next_uid), + &sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE); + } + + if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) { + /* other sessions have already marked more messages as + recent. */ + sync_ctx->last_nonrecent_uid = + sync_ctx->hdr->first_recent_uid - 1; + } + + /* mark recent messages */ + view = mail_index_transaction_open_updated_view(sync_ctx->t); + if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1, + (uint32_t)-1, &seq, &seq2)) { + mailbox_recent_flags_set_seqs(&sync_ctx->mbox->box, + view, seq, seq2); + } + mail_index_view_close(&view); + + first_recent_uid = !sync_ctx->keep_recent ? + sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1; + if (sync_ctx->hdr->first_recent_uid < first_recent_uid) { + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + return 0; +} + +static void mbox_sync_restart(struct mbox_sync_context *sync_ctx) +{ + sync_ctx->base_uid_validity = 0; + sync_ctx->base_uid_last = 0; + sync_ctx->base_uid_last_offset = 0; + + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + + index_sync_changes_reset(sync_ctx->sync_changes); + mail_index_sync_reset(sync_ctx->index_sync_ctx); + mail_index_transaction_reset(sync_ctx->t); + + if (sync_ctx->index_reset) { + mail_index_reset(sync_ctx->t); + sync_ctx->reset_hdr.next_uid = 1; + sync_ctx->hdr = &sync_ctx->reset_hdr; + mailbox_recent_flags_reset(&sync_ctx->mbox->box); + } + + sync_ctx->prev_msg_uid = 0; + sync_ctx->next_uid = sync_ctx->hdr->next_uid; + sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid; + sync_ctx->seq = 0; + sync_ctx->idx_seq = 1; + sync_ctx->need_space_seq = 0; + sync_ctx->expunged_space = 0; + sync_ctx->space_diff = 0; + + sync_ctx->dest_first_mail = TRUE; + sync_ctx->ext_modified = FALSE; + sync_ctx->errors = FALSE; +} + +static int mbox_sync_do(struct mbox_sync_context *sync_ctx, + enum mbox_sync_flags flags) +{ + struct mbox_index_header *mbox_hdr = &sync_ctx->mbox->mbox_hdr; + struct mbox_sync_mail_context mail_ctx; + const struct stat *st; + unsigned int i; + bool partial; + int ret; + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + sync_ctx->last_stat = *st; + sync_ctx->orig_size = st->st_size; + sync_ctx->orig_atime = st->st_atime; + sync_ctx->orig_mtime = st->st_mtime; + + if ((flags & MBOX_SYNC_FORCE_SYNC) != 0) { + /* forcing a full sync. assume file has changed. */ + partial = FALSE; + mbox_hdr->dirty_flag = 1; + } else if ((uint32_t)st->st_mtime == mbox_hdr->sync_mtime && + (uint64_t)st->st_size == mbox_hdr->sync_size) { + /* file is fully synced */ + if (mbox_hdr->dirty_flag != 0 && (flags & MBOX_SYNC_UNDIRTY) != 0) + partial = FALSE; + else + partial = TRUE; + } else if ((flags & MBOX_SYNC_UNDIRTY) != 0 || + (uint64_t)st->st_size == mbox_hdr->sync_size) { + /* we want to do full syncing. always do this if + file size hasn't changed but timestamp has. it most + likely means that someone had modified some header + and we probably want to know about it */ + partial = FALSE; + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } else { + /* see if we can delay syncing the whole file. + normally we only notice expunges and appends + in partial syncing. */ + partial = TRUE; + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } + + mbox_sync_restart(sync_ctx); + for (i = 0;;) { + ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial); + if (ret > 0 && !sync_ctx->errors) + break; + if (ret < 0) + return -1; + + /* a) partial sync didn't work + b) we ran out of UIDs + c) syncing had errors */ + if (sync_ctx->delay_writes && + (sync_ctx->errors || sync_ctx->renumber_uids)) { + /* fixing a broken mbox state, be sure to write + the changes (except if we're readonly). */ + if (!sync_ctx->readonly) + sync_ctx->delay_writes = FALSE; + } + if (++i == 3) + break; + + mbox_sync_restart(sync_ctx); + partial = FALSE; + } + + if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0) + return -1; + + /* only syncs left should be just appends (and their updates) + which weren't synced yet for some reason (crash). we'll just + ignore them, as we've overwritten them above. */ + index_sync_changes_reset(sync_ctx->sync_changes); + + if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 && + ret > 0 && !sync_ctx->delay_writes && + sync_ctx->base_uid_last_offset != 0) { + /* Rewrite uid_last in X-IMAPbase header if we've seen it + (ie. the file isn't empty) */ + ret = mbox_rewrite_base_uid_last(sync_ctx); + } else { + ret = 0; + } + + if (mbox_sync_update_index_header(sync_ctx) < 0) + return -1; + return ret; +} + +int mbox_sync_header_refresh(struct mbox_mailbox *mbox) +{ + const void *data; + size_t data_size; + + if (mail_index_refresh(mbox->box.index) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx, + &data, &data_size); + if (data_size == 0) { + /* doesn't exist yet. */ + i_zero(&mbox->mbox_hdr); + return 0; + } + + memcpy(&mbox->mbox_hdr, data, I_MIN(sizeof(mbox->mbox_hdr), data_size)); + if (mbox->mbox_broken_offsets) + mbox->mbox_hdr.dirty_flag = 1; + return 0; +} + +int mbox_sync_get_guid(struct mbox_mailbox *mbox) +{ + struct mail_index_transaction *trans; + unsigned int lock_id; + int ret; + + if (mbox_lock(mbox, F_WRLCK, &lock_id) <= 0) + return -1; + + ret = mbox_sync_header_refresh(mbox); + if (ret == 0) { + trans = mail_index_transaction_begin(mbox->box.view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mbox_sync_index_update_ext_header(mbox, trans); + ret = mail_index_transaction_commit(&trans); + } + mbox_unlock(mbox, lock_id); + return ret; +} + +int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty) +{ + const struct stat *st; + struct stat statbuf; + + if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) { + /* read-only stream */ + if (i_stream_stat(mbox->mbox_file_stream, FALSE, &st) < 0) { + if (errno == ENOENT) { + mailbox_set_deleted(&mbox->box); + return 0; + } + mbox_istream_set_syscall_error(mbox, + mbox->mbox_file_stream, "i_stream_stat()"); + return -1; + } + } else { + if (stat(mailbox_get_path(&mbox->box), &statbuf) < 0) { + if (errno == ENOENT) { + mailbox_set_deleted(&mbox->box); + return 0; + } + mbox_set_syscall_error(mbox, "stat()"); + return -1; + } + st = &statbuf; + } + + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + + if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + /* need to assign mailbox GUID */ + return 1; + } + + if ((uint32_t)st->st_mtime == mbox->mbox_hdr.sync_mtime && + (uint64_t)st->st_size == mbox->mbox_hdr.sync_size) { + /* fully synced */ + if (mbox->mbox_hdr.dirty_flag != 0 || leave_dirty) + return 0; + /* flushing dirtiness */ + } + + /* file changed */ + return 1; +} + +static void mbox_sync_context_free(struct mbox_sync_context *sync_ctx) +{ + index_sync_changes_deinit(&sync_ctx->sync_changes); + index_storage_expunging_deinit(&sync_ctx->mbox->box); + if (sync_ctx->index_sync_ctx != NULL) + mail_index_sync_rollback(&sync_ctx->index_sync_ctx); + pool_unref(&sync_ctx->mail_keyword_pool); + pool_unref(&sync_ctx->saved_keywords_pool); + str_free(&sync_ctx->header); + str_free(&sync_ctx->from_line); + array_free(&sync_ctx->mails); +} + +static int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags, + unsigned int *lock_id) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mbox_sync_context sync_ctx; + enum mail_index_sync_flags sync_flags; + int ret; + bool changed, delay_writes, readonly; + + readonly = mbox_is_backend_readonly(mbox) || + (flags & MBOX_SYNC_READONLY) != 0; + delay_writes = readonly || + ((flags & MBOX_SYNC_REWRITE) == 0 && + mbox->storage->set->mbox_lazy_writes); + + if (!mbox->storage->set->mbox_dirty_syncs && + !mbox->storage->set->mbox_very_dirty_syncs) + flags |= MBOX_SYNC_UNDIRTY; + + if ((flags & MBOX_SYNC_LOCK_READING) != 0) { + if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0) + return -1; + } + + if ((flags & MBOX_SYNC_HEADER) != 0 || + (flags & MBOX_SYNC_FORCE_SYNC) != 0) { + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + changed = TRUE; + } else { + bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0; + if ((ret = mbox_sync_has_changed(mbox, leave_dirty)) < 0) + return -1; + changed = ret > 0; + } + + if ((flags & MBOX_SYNC_LOCK_READING) != 0) { + /* we just want to lock it for reading. if mbox hasn't been + modified don't do any syncing. */ + if (!changed) + return 0; + + /* have to sync to make sure offsets have stayed the same */ + mbox_unlock(mbox, *lock_id); + *lock_id = 0; + } + + /* flush input streams' buffers */ + if (mbox->mbox_stream != NULL) + i_stream_sync(mbox->mbox_stream); + if (mbox->mbox_file_stream != NULL) + i_stream_sync(mbox->mbox_file_stream); + +again: + if (changed) { + /* we're most likely modifying the mbox while syncing, just + lock it for writing immediately. the mbox must be locked + before index syncing is started to avoid deadlocks, so we + don't have much choice either (well, easy ones anyway). */ + int lock_type = readonly ? F_RDLCK : F_WRLCK; + + if ((ret = mbox_lock(mbox, lock_type, lock_id)) <= 0) { + if (ret == 0 || lock_type == F_RDLCK) + return -1; + + /* try as read-only */ + if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0) + return -1; + mbox->backend_readonly = readonly = TRUE; + mbox->backend_readonly_set = TRUE; + delay_writes = TRUE; + } + } + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if ((flags & MBOX_SYNC_REWRITE) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY; + + ret = index_storage_expunged_sync_begin(&mbox->box, &index_sync_ctx, + &sync_view, &trans, sync_flags); + if (ret <= 0) + return ret; + + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) { + /* see if we need to drop recent flags */ + sync_ctx.hdr = mail_index_get_header(sync_view); + if (sync_ctx.hdr->first_recent_uid < sync_ctx.hdr->next_uid) + changed = TRUE; + } + + if (!changed && !mail_index_sync_have_more(index_sync_ctx)) { + /* nothing to do */ + nothing_to_do: + /* index may need to do internal syncing though, so commit + instead of rolling back. */ + index_storage_expunging_deinit(&mbox->box); + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + return 0; + } + + i_zero(&sync_ctx); + sync_ctx.mbox = mbox; + sync_ctx.keep_recent = + (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0; + + sync_ctx.hdr = mail_index_get_header(sync_view); + sync_ctx.from_line = str_new(default_pool, 256); + sync_ctx.header = str_new(default_pool, 4096); + + sync_ctx.index_sync_ctx = index_sync_ctx; + sync_ctx.sync_view = sync_view; + sync_ctx.t = trans; + sync_ctx.mail_keyword_pool = + pool_alloconly_create("mbox keywords", 512); + sync_ctx.saved_keywords_pool = + pool_alloconly_create("mbox saved keywords", 4096); + + /* make sure we've read the latest keywords in index */ + (void)mail_index_get_keywords(mbox->box.index); + + i_array_init(&sync_ctx.mails, 64); + + sync_ctx.flags = flags; + sync_ctx.readonly = readonly; + sync_ctx.delay_writes = delay_writes; + + sync_ctx.sync_changes = + index_sync_changes_init(index_sync_ctx, sync_view, trans, + sync_ctx.delay_writes); + + if (!changed && delay_writes) { + /* if we have only flag changes, we don't need to open the + mbox file */ + bool expunged; + uint32_t uid; + + mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged); + uid = expunged ? 1 : + index_sync_changes_get_next_uid(sync_ctx.sync_changes); + if (uid == 0) { + sync_ctx.index_sync_ctx = NULL; + mbox_sync_context_free(&sync_ctx); + goto nothing_to_do; + } + } + + if (*lock_id == 0) { + /* ok, we have something to do but no locks. we'll have to + restart syncing to avoid deadlocking. */ + mbox_sync_context_free(&sync_ctx); + changed = TRUE; + goto again; + } + + if (mbox_file_open_stream(mbox) < 0) { + mbox_sync_context_free(&sync_ctx); + return -1; + } + + sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream; + sync_ctx.input = sync_ctx.mbox->mbox_stream; + sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 : + sync_ctx.mbox->mbox_fd; + + ret = mbox_sync_do(&sync_ctx, flags); + + if (ret < 0) + mail_index_sync_rollback(&index_sync_ctx); + else if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } + sync_ctx.t = NULL; + sync_ctx.index_sync_ctx = NULL; + + if (ret == 0 && mbox->mbox_fd != -1 && sync_ctx.keep_recent && + !readonly) { + /* try to set atime back to its original value. + (it'll fail with EPERM for shared mailboxes where we aren't + the file's owner) */ + struct utimbuf buf; + struct stat st; + + if (fstat(mbox->mbox_fd, &st) < 0) + mbox_set_syscall_error(mbox, "fstat()"); + else { + buf.modtime = st.st_mtime; + buf.actime = sync_ctx.orig_atime; + if (utime(mailbox_get_path(&mbox->box), &buf) < 0 && + errno != EPERM) + mbox_set_syscall_error(mbox, "utime()"); + } + } + + i_assert(*lock_id != 0); + + if (mbox->storage->storage.set->mail_nfs_storage && + mbox->mbox_fd != -1) { + if (fdatasync(mbox->mbox_fd) < 0) { + mbox_set_syscall_error(mbox, "fdatasync()"); + ret = -1; + } + } + + mbox_sync_context_free(&sync_ctx); + return ret; +} + +int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags) +{ + unsigned int lock_id = 0; + int ret; + + i_assert(mbox->mbox_lock_type != F_RDLCK || + (flags & MBOX_SYNC_READONLY) != 0); + + mbox->syncing = TRUE; + ret = mbox_sync_int(mbox, flags, &lock_id); + mbox->syncing = FALSE; + + if (lock_id != 0) { + if (ret < 0) { + /* syncing failed, don't leave it locked */ + mbox_unlock(mbox, lock_id); + } else if ((flags & MBOX_SYNC_LOCK_READING) == 0) { + if (mbox_unlock(mbox, lock_id) < 0) + ret = -1; + } else if (mbox->mbox_lock_type != F_RDLCK) { + /* drop to read lock */ + unsigned int read_lock_id = 0; + + if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0) + ret = -1; + if (mbox_unlock(mbox, lock_id) < 0) + ret = -1; + } + } + + mailbox_sync_notify(&mbox->box, 0, 0); + return ret; +} + +struct mailbox_sync_context * +mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + enum mbox_sync_flags mbox_sync_flags = 0; + int ret = 0; + + if (index_mailbox_want_full_sync(&mbox->box, flags)) { + if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 && + !mbox->storage->set->mbox_very_dirty_syncs) + mbox_sync_flags |= MBOX_SYNC_UNDIRTY; + if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0) + mbox_sync_flags |= MBOX_SYNC_REWRITE; + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) { + mbox_sync_flags |= MBOX_SYNC_UNDIRTY | + MBOX_SYNC_REWRITE | MBOX_SYNC_FORCE_SYNC; + } + + ret = mbox_sync(mbox, mbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +} |