diff options
Diffstat (limited to 'src/lib-storage/index/maildir')
21 files changed, 9435 insertions, 0 deletions
diff --git a/src/lib-storage/index/maildir/Makefile.am b/src/lib-storage/index/maildir/Makefile.am new file mode 100644 index 0000000..7acf249 --- /dev/null +++ b/src/lib-storage/index/maildir/Makefile.am @@ -0,0 +1,36 @@ +noinst_LTLIBRARIES = libstorage_maildir.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_maildir_la_SOURCES = \ + maildir-copy.c \ + maildir-filename.c \ + maildir-filename-flags.c \ + maildir-keywords.c \ + maildir-mail.c \ + maildir-save.c \ + maildir-settings.c \ + maildir-storage.c \ + maildir-sync.c \ + maildir-sync-index.c \ + maildir-uidlist.c \ + maildir-util.c + +headers = \ + maildir-filename.h \ + maildir-filename-flags.h \ + maildir-keywords.h \ + maildir-storage.h \ + maildir-settings.h \ + maildir-sync.h \ + maildir-uidlist.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/maildir/Makefile.in b/src/lib-storage/index/maildir/Makefile.in new file mode 100644 index 0000000..7210c54 --- /dev/null +++ b/src/lib-storage/index/maildir/Makefile.in @@ -0,0 +1,875 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/maildir +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_maildir_la_LIBADD = +am_libstorage_maildir_la_OBJECTS = maildir-copy.lo maildir-filename.lo \ + maildir-filename-flags.lo maildir-keywords.lo maildir-mail.lo \ + maildir-save.lo maildir-settings.lo maildir-storage.lo \ + maildir-sync.lo maildir-sync-index.lo maildir-uidlist.lo \ + maildir-util.lo +libstorage_maildir_la_OBJECTS = $(am_libstorage_maildir_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/maildir-copy.Plo \ + ./$(DEPDIR)/maildir-filename-flags.Plo \ + ./$(DEPDIR)/maildir-filename.Plo \ + ./$(DEPDIR)/maildir-keywords.Plo ./$(DEPDIR)/maildir-mail.Plo \ + ./$(DEPDIR)/maildir-save.Plo ./$(DEPDIR)/maildir-settings.Plo \ + ./$(DEPDIR)/maildir-storage.Plo \ + ./$(DEPDIR)/maildir-sync-index.Plo \ + ./$(DEPDIR)/maildir-sync.Plo ./$(DEPDIR)/maildir-uidlist.Plo \ + ./$(DEPDIR)/maildir-util.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_maildir_la_SOURCES) +DIST_SOURCES = $(libstorage_maildir_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_maildir.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index + +libstorage_maildir_la_SOURCES = \ + maildir-copy.c \ + maildir-filename.c \ + maildir-filename-flags.c \ + maildir-keywords.c \ + maildir-mail.c \ + maildir-save.c \ + maildir-settings.c \ + maildir-storage.c \ + maildir-sync.c \ + maildir-sync-index.c \ + maildir-uidlist.c \ + maildir-util.c + +headers = \ + maildir-filename.h \ + maildir-filename-flags.h \ + maildir-keywords.h \ + maildir-storage.h \ + maildir-settings.h \ + maildir-sync.h \ + maildir-uidlist.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/maildir/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/maildir/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_maildir.la: $(libstorage_maildir_la_OBJECTS) $(libstorage_maildir_la_DEPENDENCIES) $(EXTRA_libstorage_maildir_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_maildir_la_OBJECTS) $(libstorage_maildir_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-copy.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-filename-flags.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-filename.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-keywords.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-sync-index.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-uidlist.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-util.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/maildir-copy.Plo + -rm -f ./$(DEPDIR)/maildir-filename-flags.Plo + -rm -f ./$(DEPDIR)/maildir-filename.Plo + -rm -f ./$(DEPDIR)/maildir-keywords.Plo + -rm -f ./$(DEPDIR)/maildir-mail.Plo + -rm -f ./$(DEPDIR)/maildir-save.Plo + -rm -f ./$(DEPDIR)/maildir-settings.Plo + -rm -f ./$(DEPDIR)/maildir-storage.Plo + -rm -f ./$(DEPDIR)/maildir-sync-index.Plo + -rm -f ./$(DEPDIR)/maildir-sync.Plo + -rm -f ./$(DEPDIR)/maildir-uidlist.Plo + -rm -f ./$(DEPDIR)/maildir-util.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/maildir-copy.Plo + -rm -f ./$(DEPDIR)/maildir-filename-flags.Plo + -rm -f ./$(DEPDIR)/maildir-filename.Plo + -rm -f ./$(DEPDIR)/maildir-keywords.Plo + -rm -f ./$(DEPDIR)/maildir-mail.Plo + -rm -f ./$(DEPDIR)/maildir-save.Plo + -rm -f ./$(DEPDIR)/maildir-settings.Plo + -rm -f ./$(DEPDIR)/maildir-storage.Plo + -rm -f ./$(DEPDIR)/maildir-sync-index.Plo + -rm -f ./$(DEPDIR)/maildir-sync.Plo + -rm -f ./$(DEPDIR)/maildir-uidlist.Plo + -rm -f ./$(DEPDIR)/maildir-util.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/maildir/maildir-copy.c b/src/lib-storage/index/maildir/maildir-copy.c new file mode 100644 index 0000000..1645ddb --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-copy.c @@ -0,0 +1,149 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "nfs-workarounds.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-filename.h" +#include "maildir-keywords.h" +#include "maildir-sync.h" +#include "index-mail.h" +#include "mail-copy.h" + +#include <unistd.h> +#include <sys/stat.h> + +struct hardlink_ctx { + const char *dest_path; + bool success:1; +}; + +static int do_hardlink(struct maildir_mailbox *mbox, const char *path, + struct hardlink_ctx *ctx) +{ + int ret; + + if (mbox->storage->storage.set->mail_nfs_storage) + ret = nfs_safe_link(path, ctx->dest_path, FALSE); + else + ret = link(path, ctx->dest_path); + if (ret < 0) { + if (errno == ENOENT) + return 0; + + if (ENOQUOTA(errno)) { + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + return -1; + } + + /* we could handle the EEXIST condition by changing the + filename, but it practically never happens so just fallback + to standard copying for the rare cases when it does. */ + if (errno == EACCES || ECANTLINK(errno) || errno == EEXIST) + return 1; + + mailbox_set_critical(&mbox->box, "link(%s, %s) failed: %m", + path, ctx->dest_path); + return -1; + } + + ctx->success = TRUE; + return 1; +} + +static int +maildir_copy_hardlink(struct mail_save_context *ctx, struct mail *mail) +{ + struct maildir_mailbox *dest_mbox = MAILDIR_MAILBOX(ctx->transaction->box); + struct maildir_mailbox *src_mbox; + struct maildir_filename *mf; + struct hardlink_ctx do_ctx; + const char *path, *guid, *dest_fname; + uoff_t vsize, size; + enum mail_lookup_abort old_abort; + + if (strcmp(mail->box->storage->name, MAILDIR_STORAGE_NAME) == 0) + src_mbox = MAILDIR_MAILBOX(mail->box); + else if (strcmp(mail->box->storage->name, "raw") == 0) { + /* lda uses raw format */ + src_mbox = NULL; + } else { + /* Can't hard link files from the source storage */ + return 0; + } + + /* hard link to tmp/ with a newly generated filename and later when we + have uidlist locked, move it to new/cur. */ + dest_fname = maildir_filename_generate(); + i_zero(&do_ctx); + do_ctx.dest_path = + t_strdup_printf("%s/tmp/%s", mailbox_get_path(&dest_mbox->box), + dest_fname); + if (src_mbox != NULL) { + /* maildir */ + if (maildir_file_do(src_mbox, mail->uid, + do_hardlink, &do_ctx) < 0) + return -1; + } else { + /* raw / lda */ + if (mail_get_special(mail, MAIL_FETCH_STORAGE_ID, + &path) < 0 || *path == '\0') + return 0; + if (do_hardlink(dest_mbox, path, &do_ctx) < 0) + return -1; + } + + if (!do_ctx.success) { + /* couldn't copy with hardlinking, fallback to copying */ + return 0; + } + + /* hardlinked to tmp/, treat as normal copied mail */ + mf = maildir_save_add(ctx, dest_fname, mail); + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) == 0) { + if (*guid != '\0') + maildir_save_set_dest_basename(ctx, mf, guid); + } + + /* finish copying keywords */ + maildir_save_finish_keywords(ctx); + + /* remember size/vsize if possible */ + old_abort = mail->lookup_abort; + mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + if (mail_get_physical_size(mail, &size) < 0) + size = UOFF_T_MAX; + if (mail_get_virtual_size(mail, &vsize) < 0) + vsize = UOFF_T_MAX; + maildir_save_set_sizes(mf, size, vsize); + mail->lookup_abort = old_abort; + return 1; +} + +int maildir_copy(struct mail_save_context *ctx, struct mail *mail) +{ + struct mailbox_transaction_context *_t = ctx->transaction; + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_t->box); + int ret; + + i_assert((_t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (mbox->storage->set->maildir_copy_with_hardlinks && + mail_storage_copy_can_use_hardlink(mail->box, &mbox->box)) { + T_BEGIN { + ret = maildir_copy_hardlink(ctx, mail); + } T_END; + + if (ret != 0) { + index_save_context_free(ctx); + return ret > 0 ? 0 : -1; + } + + /* non-fatal hardlinking failure, try the slow way */ + } + + return mail_storage_copy(ctx, mail); +} diff --git a/src/lib-storage/index/maildir/maildir-filename-flags.c b/src/lib-storage/index/maildir/maildir-filename-flags.c new file mode 100644 index 0000000..ac68908 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename-flags.c @@ -0,0 +1,185 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "maildir-storage.h" +#include "maildir-keywords.h" +#include "maildir-filename-flags.h" + + +void maildir_filename_flags_get(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags *flags_r, + ARRAY_TYPE(keyword_indexes) *keywords_r) +{ + const char *info; + + array_clear(keywords_r); + *flags_r = 0; + + info = strrchr(fname, MAILDIR_INFO_SEP); + if (info == NULL || info[1] != '2' || info[2] != MAILDIR_FLAGS_SEP) + return; + + for (info += 3; *info != '\0' && *info != MAILDIR_FLAGS_SEP; info++) { + switch (*info) { + case 'R': /* replied */ + *flags_r |= MAIL_ANSWERED; + break; + case 'S': /* seen */ + *flags_r |= MAIL_SEEN; + break; + case 'T': /* trashed */ + *flags_r |= MAIL_DELETED; + break; + case 'D': /* draft */ + *flags_r |= MAIL_DRAFT; + break; + case 'F': /* flagged */ + *flags_r |= MAIL_FLAGGED; + break; + default: + if (*info >= MAILDIR_KEYWORD_FIRST && + *info <= MAILDIR_KEYWORD_LAST) { + int idx; + + idx = maildir_keywords_char_idx(ctx, *info); + if (idx < 0) { + /* unknown keyword. */ + break; + } + + array_push_back(keywords_r, + (unsigned int *)&idx); + break; + } + + /* unknown flag - ignore */ + break; + } + } +} + +static int char_cmp(const void *p1, const void *p2) +{ + const unsigned char *c1 = p1, *c2 = p2; + + return *c1 - *c2; +} + +static void +maildir_filename_append_keywords(struct maildir_keywords_sync_ctx *ctx, + ARRAY_TYPE(keyword_indexes) *keywords, + string_t *fname) +{ + const unsigned int *indexes; + unsigned int i, count; + size_t start = str_len(fname); + char chr; + + indexes = array_get(keywords, &count); + for (i = 0; i < count; i++) { + chr = maildir_keywords_idx_char(ctx, indexes[i]); + if (chr != '\0') + str_append_c(fname, chr); + } + + qsort(str_c_modifiable(fname) + start, str_len(fname) - start, 1, + char_cmp); +} + +static const char * ATTR_NULL(1, 4) +maildir_filename_flags_full_set(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags flags, + ARRAY_TYPE(keyword_indexes) *keywords) +{ + string_t *flags_str; + enum mail_flags flags_left; + const char *info, *oldflags; + int nextflag; + + /* remove the old :info from file name, and get the old flags */ + info = strrchr(fname, MAILDIR_INFO_SEP); + if (info != NULL && strrchr(fname, '/') > info) + info = NULL; + + oldflags = ""; + if (info != NULL) { + fname = t_strdup_until(fname, info); + if (info[1] == '2' && info[2] == MAILDIR_FLAGS_SEP) + oldflags = info+3; + } + + /* insert the new flags between old flags. flags must be sorted by + their ASCII code. unknown flags are kept. */ + flags_str = t_str_new(256); + str_append(flags_str, fname); + str_append(flags_str, MAILDIR_FLAGS_FULL_SEP); + flags_left = flags; + for (;;) { + /* skip all known flags */ + while (*oldflags == 'D' || *oldflags == 'F' || + *oldflags == 'R' || *oldflags == 'S' || + *oldflags == 'T' || + (*oldflags >= MAILDIR_KEYWORD_FIRST && + *oldflags <= MAILDIR_KEYWORD_LAST)) + oldflags++; + + nextflag = *oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP ? + 256 : (unsigned char) *oldflags; + + if ((flags_left & MAIL_DRAFT) != 0 && nextflag > 'D') { + str_append_c(flags_str, 'D'); + flags_left &= ENUM_NEGATE(MAIL_DRAFT); + } + if ((flags_left & MAIL_FLAGGED) != 0 && nextflag > 'F') { + str_append_c(flags_str, 'F'); + flags_left &= ENUM_NEGATE(MAIL_FLAGGED); + } + if ((flags_left & MAIL_ANSWERED) != 0 && nextflag > 'R') { + str_append_c(flags_str, 'R'); + flags_left &= ENUM_NEGATE(MAIL_ANSWERED); + } + if ((flags_left & MAIL_SEEN) != 0 && nextflag > 'S') { + str_append_c(flags_str, 'S'); + flags_left &= ENUM_NEGATE(MAIL_SEEN); + } + if ((flags_left & MAIL_DELETED) != 0 && nextflag > 'T') { + str_append_c(flags_str, 'T'); + flags_left &= ENUM_NEGATE(MAIL_DELETED); + } + + if (keywords != NULL && array_is_created(keywords) && + nextflag > MAILDIR_KEYWORD_FIRST) { + maildir_filename_append_keywords(ctx, keywords, + flags_str); + keywords = NULL; + } + + if (*oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP) + break; + + str_append_c(flags_str, *oldflags); + oldflags++; + } + + if (*oldflags == MAILDIR_FLAGS_SEP) { + /* another flagset, we don't know about these, just keep them */ + while (*oldflags != '\0') + str_append_c(flags_str, *oldflags++); + } + + return str_c(flags_str); +} + +const char *maildir_filename_flags_set(const char *fname, enum mail_flags flags) +{ + return maildir_filename_flags_full_set(NULL, fname, flags, NULL); +} + +const char *maildir_filename_flags_kw_set(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags flags, + ARRAY_TYPE(keyword_indexes) *keywords) +{ + return maildir_filename_flags_full_set(ctx, fname, flags, keywords); +} diff --git a/src/lib-storage/index/maildir/maildir-filename-flags.h b/src/lib-storage/index/maildir/maildir-filename-flags.h new file mode 100644 index 0000000..2676e5c --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename-flags.h @@ -0,0 +1,13 @@ +#ifndef MAILDIR_FILENAME_FLAGS_H +#define MAILDIR_FILENAME_FLAGS_H + +void maildir_filename_flags_get(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags *flags_r, + ARRAY_TYPE(keyword_indexes) *keywords_r); + +const char *maildir_filename_flags_set(const char *fname, enum mail_flags flags); +const char *maildir_filename_flags_kw_set(struct maildir_keywords_sync_ctx *ctx, + const char *fname, enum mail_flags flags, + ARRAY_TYPE(keyword_indexes) *keywords); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-filename.c b/src/lib-storage/index/maildir/maildir-filename.c new file mode 100644 index 0000000..9deba82 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename.c @@ -0,0 +1,143 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "time-util.h" +#include "hostpid.h" +#include "maildir-storage.h" +#include "maildir-filename.h" + +const char *maildir_filename_generate(void) +{ + static struct timeval last_tv = { 0, 0 }; + struct timeval tv; + + /* use secs + usecs to guarantee uniqueness within this process. */ + if (timeval_cmp(&ioloop_timeval, &last_tv) > 0) + tv = ioloop_timeval; + else { + tv = last_tv; + if (++tv.tv_usec == 1000000) { + tv.tv_sec++; + tv.tv_usec = 0; + } + } + last_tv = tv; + + return t_strdup_printf("%s.M%sP%s.%s", + dec2str(tv.tv_sec), dec2str(tv.tv_usec), + my_pid, my_hostname); +} + +bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r) +{ + uoff_t size = 0; + + for (; *fname != '\0'; fname++) { + i_assert(*fname != '/'); + if (*fname == ',' && fname[1] == type && fname[2] == '=') { + fname += 3; + break; + } + } + + if (*fname == '\0') + return FALSE; + + while (*fname >= '0' && *fname <= '9') { + size = size * 10 + (*fname - '0'); + fname++; + } + + if (*fname != MAILDIR_INFO_SEP && + *fname != MAILDIR_EXTRA_SEP && + *fname != '\0') + return FALSE; + + *size_r = size; + return TRUE; +} + +/* a char* hash function from ASU -- from glib */ +unsigned int ATTR_NO_SANITIZE_INTEGER +maildir_filename_base_hash(const char *s) +{ + unsigned int g, h = 0; + + while (*s != MAILDIR_INFO_SEP && *s != '\0') { + i_assert(*s != '/'); + h = (h << 4) + *s; + if ((g = h & 0xf0000000UL) != 0) { + h = h ^ (g >> 24); + h = h ^ g; + } + + s++; + } + + return h; +} + +int maildir_filename_base_cmp(const char *fname1, const char *fname2) +{ + while (*fname1 == *fname2 && *fname1 != MAILDIR_INFO_SEP && + *fname1 != '\0') { + i_assert(*fname1 != '/'); + fname1++; fname2++; + } + + if ((*fname1 == '\0' || *fname1 == MAILDIR_INFO_SEP) && + (*fname2 == '\0' || *fname2 == MAILDIR_INFO_SEP)) + return 0; + return *fname1 - *fname2; +} + +static bool maildir_fname_get_usecs(const char *fname, int *usecs_r) +{ + int usecs = 0; + + /* Assume we already read the timestamp. Next up is + ".<uniqueness>.<host>". Find usecs inside the uniqueness. */ + if (*fname != '.') + return FALSE; + + fname++; + while (*fname != '\0' && *fname != '.' && *fname != MAILDIR_INFO_SEP) { + if (*fname++ == 'M') { + while (*fname >= '0' && *fname <= '9') { + usecs = usecs * 10 + (*fname - '0'); + fname++; + } + *usecs_r = usecs; + return TRUE; + } + } + return FALSE; +} + +int maildir_filename_sort_cmp(const char *fname1, const char *fname2) +{ + const char *s1, *s2; + time_t secs1 = 0, secs2 = 0; + int ret, usecs1, usecs2; + + /* sort primarily by the timestamp in file name */ + for (s1 = fname1; *s1 >= '0' && *s1 <= '9'; s1++) + secs1 = secs1 * 10 + (*s1 - '0'); + for (s2 = fname2; *s2 >= '0' && *s2 <= '9'; s2++) + secs2 = secs2 * 10 + (*s2 - '0'); + + ret = (int)((long)secs1 - (long)secs2); + if (ret == 0) { + /* sort secondarily by microseconds, if they exist */ + if (maildir_fname_get_usecs(s1, &usecs1) && + maildir_fname_get_usecs(s2, &usecs2)) + ret = usecs1 - usecs2; + + if (ret == 0) { + /* fallback to comparing the base file name */ + ret = maildir_filename_base_cmp(s1, s2); + } + } + return ret; +} diff --git a/src/lib-storage/index/maildir/maildir-filename.h b/src/lib-storage/index/maildir/maildir-filename.h new file mode 100644 index 0000000..a691dea --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-filename.h @@ -0,0 +1,14 @@ +#ifndef MAILDIR_FILENAME_H +#define MAILDIR_FILENAME_H + +struct maildir_keywords_sync_ctx; + +const char *maildir_filename_generate(void); + +bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r); + +unsigned int maildir_filename_base_hash(const char *fname); +int maildir_filename_base_cmp(const char *fname1, const char *fname2); +int maildir_filename_sort_cmp(const char *fname1, const char *fname2); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-keywords.c b/src/lib-storage/index/maildir/maildir-keywords.c new file mode 100644 index 0000000..a25d112 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-keywords.c @@ -0,0 +1,499 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +/* note that everything here depends on uidlist file being locked the whole + time. that's why we don't have any locking of our own, or that we do things + that would be racy otherwise. */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "hash.h" +#include "str.h" +#include "istream.h" +#include "eacces-error.h" +#include "file-dotlock.h" +#include "write-full.h" +#include "nfs-workarounds.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" + +#include <sys/stat.h> +#include <utime.h> + +/* how many seconds to wait before overriding dovecot-keywords.lock */ +#define KEYWORDS_LOCK_STALE_TIMEOUT (60*2) + +struct maildir_keywords { + struct maildir_mailbox *mbox; + struct mail_storage *storage; + char *path; + + pool_t pool; + ARRAY_TYPE(keywords) list; + HASH_TABLE(char *, void *) hash; /* name -> idx+1 */ + + struct dotlock_settings dotlock_settings; + + time_t synced_mtime; + bool synced:1; + bool changed:1; +}; + +struct maildir_keywords_sync_ctx { + struct maildir_keywords *mk; + struct mail_index *index; + + const ARRAY_TYPE(keywords) *keywords; + ARRAY(char) idx_to_chr; + unsigned int chridx_to_idx[MAILDIR_MAX_KEYWORDS]; + bool readonly; +}; + +struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox) +{ + struct maildir_keywords *mk; + + mk = maildir_keywords_init_readonly(&mbox->box); + mk->mbox = mbox; + return mk; +} + +struct maildir_keywords * +maildir_keywords_init_readonly(struct mailbox *box) +{ + struct maildir_keywords *mk; + const char *dir; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, &dir) <= 0) + i_unreached(); + + mk = i_new(struct maildir_keywords, 1); + mk->storage = box->storage; + mk->path = i_strconcat(dir, "/" MAILDIR_KEYWORDS_NAME, NULL); + mk->pool = pool_alloconly_create("maildir keywords", 512); + i_array_init(&mk->list, MAILDIR_MAX_KEYWORDS); + hash_table_create(&mk->hash, mk->pool, 0, strcase_hash, strcasecmp); + + mk->dotlock_settings.use_excl_lock = + box->storage->set->dotlock_use_excl; + mk->dotlock_settings.nfs_flush = + box->storage->set->mail_nfs_storage; + mk->dotlock_settings.timeout = + mail_storage_get_lock_timeout(box->storage, + KEYWORDS_LOCK_STALE_TIMEOUT + 2); + mk->dotlock_settings.stale_timeout = KEYWORDS_LOCK_STALE_TIMEOUT; + mk->dotlock_settings.temp_prefix = + mailbox_list_get_temp_prefix(box->list); + return mk; +} + +void maildir_keywords_deinit(struct maildir_keywords **_mk) +{ + struct maildir_keywords *mk = *_mk; + + *_mk = NULL; + hash_table_destroy(&mk->hash); + array_free(&mk->list); + pool_unref(&mk->pool); + i_free(mk->path); + i_free(mk); +} + +static void maildir_keywords_clear(struct maildir_keywords *mk) +{ + array_clear(&mk->list); + hash_table_clear(mk->hash, TRUE); + p_clear(mk->pool); +} + +static int maildir_keywords_sync(struct maildir_keywords *mk) +{ + struct istream *input; + struct stat st; + char *line, *p, *new_name; + const char **strp; + unsigned int idx; + int fd; + + /* Remember that we rely on uidlist file locking in here. That's why + we rely on stat()'s timestamp and don't bother handling ESTALE + errors. */ + + if (mk->storage->set->mail_nfs_storage) { + /* file is updated only by replacing it, no need to flush + attribute cache */ + nfs_flush_file_handle_cache(mk->path); + } + + if (nfs_safe_stat(mk->path, &st) < 0) { + if (errno == ENOENT) { + maildir_keywords_clear(mk); + mk->synced = TRUE; + return 0; + } + mailbox_set_critical(&mk->mbox->box, + "stat(%s) failed: %m", mk->path); + return -1; + } + + if (st.st_mtime == mk->synced_mtime) { + /* hasn't changed */ + mk->synced = TRUE; + return 0; + } + mk->synced_mtime = st.st_mtime; + + fd = open(mk->path, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) { + maildir_keywords_clear(mk); + mk->synced = TRUE; + return 0; + } + mailbox_set_critical(&mk->mbox->box, + "open(%s) failed: %m", mk->path); + return -1; + } + + maildir_keywords_clear(mk); + input = i_stream_create_fd(fd, 1024); + while ((line = i_stream_read_next_line(input)) != NULL) { + p = strchr(line, ' '); + if (p == NULL) { + /* note that when converting .customflags file this + case happens in the first line. */ + continue; + } + *p++ = '\0'; + + if (str_to_uint(line, &idx) < 0 || + idx >= MAILDIR_MAX_KEYWORDS || *p == '\0' || + hash_table_lookup(mk->hash, p) != NULL) { + /* shouldn't happen */ + continue; + } + + /* save it */ + new_name = p_strdup(mk->pool, p); + hash_table_insert(mk->hash, new_name, POINTER_CAST(idx + 1)); + + strp = array_idx_get_space(&mk->list, idx); + *strp = new_name; + } + i_stream_destroy(&input); + + if (close(fd) < 0) { + mailbox_set_critical(&mk->mbox->box, + "close(%s) failed: %m", mk->path); + return -1; + } + + mk->synced = TRUE; + return 0; +} + +static int +maildir_keywords_lookup(struct maildir_keywords *mk, const char *name, + unsigned int *chridx_r) +{ + void *value; + + value = hash_table_lookup(mk->hash, name); + if (value == NULL) { + if (mk->synced) + return 0; + + if (maildir_keywords_sync(mk) < 0) + return -1; + i_assert(mk->synced); + + value = hash_table_lookup(mk->hash, name); + if (value == NULL) + return 0; + } + + *chridx_r = POINTER_CAST_TO(value, unsigned int)-1; + return 1; +} + +static void +maildir_keywords_create(struct maildir_keywords *mk, const char *name, + unsigned int chridx) +{ + const char **strp; + char *new_name; + + i_assert(chridx < MAILDIR_MAX_KEYWORDS); + + new_name = p_strdup(mk->pool, name); + hash_table_insert(mk->hash, new_name, POINTER_CAST(chridx + 1)); + + strp = array_idx_get_space(&mk->list, chridx); + *strp = new_name; + + mk->changed = TRUE; +} + +static int +maildir_keywords_lookup_or_create(struct maildir_keywords *mk, const char *name, + unsigned int *chridx_r) +{ + const char *const *keywords; + unsigned int i, count; + int ret; + + if ((ret = maildir_keywords_lookup(mk, name, chridx_r)) != 0) + return ret; + + /* see if we are full */ + keywords = array_get(&mk->list, &count); + for (i = 0; i < count; i++) { + if (keywords[i] == NULL) + break; + } + + if (i == count && count >= MAILDIR_MAX_KEYWORDS) + return -1; + + if (!maildir_uidlist_is_locked(mk->mbox->uidlist)) + return -1; + + maildir_keywords_create(mk, name, i); + *chridx_r = i; + return 1; +} + +static const char * +maildir_keywords_idx(struct maildir_keywords *mk, unsigned int idx) +{ + const char *const *keywords; + unsigned int count; + + keywords = array_get(&mk->list, &count); + if (idx >= count) { + if (mk->synced) + return NULL; + + if (maildir_keywords_sync(mk) < 0) + return NULL; + i_assert(mk->synced); + + keywords = array_get(&mk->list, &count); + } + return idx >= count ? NULL : keywords[idx]; +} + +static int maildir_keywords_write_fd(struct maildir_keywords *mk, + const char *path, int fd) +{ + struct maildir_mailbox *mbox = mk->mbox; + struct mailbox *box = &mbox->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *const *keywords; + unsigned int i, count; + string_t *str; + struct stat st; + + str = t_str_new(256); + keywords = array_get(&mk->list, &count); + for (i = 0; i < count; i++) { + if (keywords[i] != NULL) + str_printfa(str, "%u %s\n", i, keywords[i]); + } + if (write_full(fd, str_data(str), str_len(str)) < 0) { + mailbox_set_critical(&mk->mbox->box, + "write_full(%s) failed: %m", path); + return -1; + } + + if (fstat(fd, &st) < 0) { + mailbox_set_critical(&mk->mbox->box, + "fstat(%s) failed: %m", path); + return -1; + } + + if (st.st_gid != perm->file_create_gid && + perm->file_create_gid != (gid_t)-1) { + if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(&mk->mbox->box, "%s", + eperm_error_get_chgrp("fchown", path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(&mk->mbox->box, + "fchown(%s) failed: %m", path); + } + } + } + + /* mtime must grow every time */ + if (st.st_mtime <= mk->synced_mtime) { + struct utimbuf ut; + + mk->synced_mtime = ioloop_time <= mk->synced_mtime ? + mk->synced_mtime + 1 : ioloop_time; + ut.actime = ioloop_time; + ut.modtime = mk->synced_mtime; + if (utime(path, &ut) < 0) { + mailbox_set_critical(&mk->mbox->box, + "utime(%s) failed: %m", path); + return -1; + } + } + + if (fsync(fd) < 0) { + mailbox_set_critical(&mk->mbox->box, + "fsync(%s) failed: %m", path); + return -1; + } + return 0; +} + +static int maildir_keywords_commit(struct maildir_keywords *mk) +{ + const struct mailbox_permissions *perm; + struct dotlock *dotlock; + const char *lock_path; + mode_t old_mask; + int i, fd; + + mk->synced = FALSE; + + if (!mk->changed || mk->mbox == NULL) + return 0; + + lock_path = t_strconcat(mk->path, ".lock", NULL); + i_unlink_if_exists(lock_path); + + perm = mailbox_get_permissions(&mk->mbox->box); + for (i = 0;; i++) { + /* we could just create the temp file directly, but doing it + this ways avoids potential problems with overwriting + contents in malicious symlinks */ + old_mask = umask(0777 & ~perm->file_create_mode); + fd = file_dotlock_open(&mk->dotlock_settings, mk->path, + DOTLOCK_CREATE_FLAG_NONBLOCK, &dotlock); + umask(old_mask); + if (fd != -1) + break; + + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + mailbox_set_critical(&mk->mbox->box, + "file_dotlock_open(%s) failed: %m", mk->path); + return -1; + } + /* the control dir doesn't exist. create it unless the whole + mailbox was just deleted. */ + if (!maildir_set_deleted(&mk->mbox->box)) + return -1; + } + + if (maildir_keywords_write_fd(mk, lock_path, fd) < 0) { + file_dotlock_delete(&dotlock); + return -1; + } + + if (file_dotlock_replace(&dotlock, 0) < 0) { + mailbox_set_critical(&mk->mbox->box, + "file_dotlock_replace(%s) failed: %m", mk->path); + return -1; + } + + mk->changed = FALSE; + return 0; +} + +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init(struct maildir_keywords *mk, + struct mail_index *index) +{ + struct maildir_keywords_sync_ctx *ctx; + + ctx = i_new(struct maildir_keywords_sync_ctx, 1); + ctx->mk = mk; + ctx->index = index; + ctx->keywords = mail_index_get_keywords(index); + i_array_init(&ctx->idx_to_chr, MAILDIR_MAX_KEYWORDS); + return ctx; +} + +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init_readonly(struct maildir_keywords *mk, + struct mail_index *index) +{ + struct maildir_keywords_sync_ctx *ctx; + + ctx = maildir_keywords_sync_init(mk, index); + ctx->readonly = TRUE; + return ctx; +} + +void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **_ctx) +{ + struct maildir_keywords_sync_ctx *ctx = *_ctx; + + *_ctx = NULL; + + T_BEGIN { + (void)maildir_keywords_commit(ctx->mk); + } T_END; + + array_free(&ctx->idx_to_chr); + i_free(ctx); +} + +unsigned int maildir_keywords_char_idx(struct maildir_keywords_sync_ctx *ctx, + char keyword) +{ + const char *name; + unsigned int chridx, idx; + + i_assert(keyword >= MAILDIR_KEYWORD_FIRST && + keyword <= MAILDIR_KEYWORD_LAST); + chridx = keyword - MAILDIR_KEYWORD_FIRST; + + if (ctx->chridx_to_idx[chridx] != 0) + return ctx->chridx_to_idx[chridx]; + + /* lookup / create */ + name = maildir_keywords_idx(ctx->mk, chridx); + if (name == NULL) { + /* name is lost. just generate one ourself. */ + name = t_strdup_printf("unknown-%u", chridx); + while (maildir_keywords_lookup(ctx->mk, name, &idx) > 0) { + /* don't create a duplicate name. + keep changing the name until it doesn't exist */ + name = t_strconcat(name, "?", NULL); + } + maildir_keywords_create(ctx->mk, name, chridx); + } + + mail_index_keyword_lookup_or_create(ctx->index, name, &idx); + ctx->chridx_to_idx[chridx] = idx; + return idx; +} + +char maildir_keywords_idx_char(struct maildir_keywords_sync_ctx *ctx, + unsigned int idx) +{ + const char *name; + char *chr_p; + unsigned int chridx; + int ret; + + chr_p = array_idx_get_space(&ctx->idx_to_chr, idx); + if (*chr_p != '\0') + return *chr_p; + + name = array_idx_elem(ctx->keywords, idx); + ret = !ctx->readonly ? + maildir_keywords_lookup_or_create(ctx->mk, name, &chridx) : + maildir_keywords_lookup(ctx->mk, name, &chridx); + if (ret <= 0) + return '\0'; + + *chr_p = chridx + MAILDIR_KEYWORD_FIRST; + return *chr_p; +} diff --git a/src/lib-storage/index/maildir/maildir-keywords.h b/src/lib-storage/index/maildir/maildir-keywords.h new file mode 100644 index 0000000..43e47d7 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-keywords.h @@ -0,0 +1,36 @@ +#ifndef MAILDIR_KEYWORDS_H +#define MAILDIR_KEYWORDS_H + +#define MAILDIR_KEYWORDS_NAME "dovecot-keywords" + +struct maildir_mailbox; +struct maildir_keywords; +struct maildir_keywords_sync_ctx; + +struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox); +void maildir_keywords_deinit(struct maildir_keywords **mk); + +/* Initialize a read-only maildir_keywords instance. Mailbox needs to contain + the dovecot-keywords file, but otherwise it doesn't have to be in maildir + format. */ +struct maildir_keywords * +maildir_keywords_init_readonly(struct mailbox *box); + +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init(struct maildir_keywords *mk, + struct mail_index *index); +/* Don't try to add any nonexistent keywords */ +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init_readonly(struct maildir_keywords *mk, + struct mail_index *index); +void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **ctx); + +/* Returns keyword index. */ +unsigned int maildir_keywords_char_idx(struct maildir_keywords_sync_ctx *ctx, + char keyword); +/* Returns keyword character for given index, or \0 if keyword couldn't be + added. */ +char maildir_keywords_idx_char(struct maildir_keywords_sync_ctx *ctx, + unsigned int idx); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-mail.c b/src/lib-storage/index/maildir/maildir-mail.c new file mode 100644 index 0000000..c3abbd3 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-mail.c @@ -0,0 +1,809 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "nfs-workarounds.h" +#include "index-mail.h" +#include "maildir-storage.h" +#include "maildir-filename.h" +#include "maildir-uidlist.h" +#include "maildir-sync.h" + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> + +struct maildir_open_context { + int fd; + char *path; +}; + +static int +do_open(struct maildir_mailbox *mbox, const char *path, + struct maildir_open_context *ctx) +{ + ctx->fd = nfs_safe_open(path, O_RDONLY); + if (ctx->fd != -1) { + ctx->path = i_strdup(path); + return 1; + } + if (errno == ENOENT) + return 0; + + if (errno == EACCES) { + mailbox_set_critical(&mbox->box, "%s", + mail_error_eacces_msg("open", path)); + } else { + mailbox_set_critical(&mbox->box, + "open(%s) failed: %m", path); + } + return -1; +} + +static int +do_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st) +{ + if (stat(path, st) == 0) + return 1; + if (errno == ENOENT) + return 0; + + if (errno == EACCES) { + mailbox_set_critical(&mbox->box, "%s", + mail_error_eacces_msg("stat", path)); + } else { + mailbox_set_critical(&mbox->box, "stat(%s) failed: %m", path); + } + return -1; +} + +static struct istream * +maildir_open_mail(struct maildir_mailbox *mbox, struct mail *mail, + bool *deleted_r) +{ + struct istream *input; + const char *path; + struct maildir_open_context ctx; + + *deleted_r = FALSE; + + if (!mail_stream_access_start(mail)) + return NULL; + + ctx.fd = -1; + ctx.path = NULL; + + mail->transaction->stats.open_lookup_count++; + if (!mail->saving) { + if (maildir_file_do(mbox, mail->uid, do_open, &ctx) < 0) + return NULL; + } else { + path = maildir_save_file_get_path(mail->transaction, mail->seq); + if (do_open(mbox, path, &ctx) <= 0) + return NULL; + } + + if (ctx.fd == -1) { + *deleted_r = TRUE; + return NULL; + } + + input = i_stream_create_fd_autoclose(&ctx.fd, 0); + if (input->stream_errno == EISDIR) { + i_stream_destroy(&input); + if (maildir_lose_unexpected_dir(&mbox->storage->storage, + ctx.path) >= 0) + *deleted_r = TRUE; + } else { + i_stream_set_name(input, ctx.path); + index_mail_set_read_buffer_size(mail, input); + } + i_free(ctx.path); + return input; +} + +static int maildir_mail_stat(struct mail *mail, struct stat *st_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box); + struct index_mail *imail = INDEX_MAIL(mail); + const char *path; + int fd, ret; + + if (!mail_metadata_access_start(mail)) + return -1; + + if (imail->data.access_part != 0 && + imail->data.stream == NULL) { + /* we're going to open the mail anyway */ + struct istream *input; + + (void)mail_get_stream(mail, NULL, NULL, &input); + } + + if (imail->data.stream != NULL && + (fd = i_stream_get_fd(imail->data.stream)) != -1) { + mail->transaction->stats.fstat_lookup_count++; + if (fstat(fd, st_r) < 0) { + mail_set_critical(mail, "fstat(%s) failed: %m", + i_stream_get_name(imail->data.stream)); + return -1; + } + } else if (!mail->saving) { + mail->transaction->stats.stat_lookup_count++; + ret = maildir_file_do(mbox, mail->uid, do_stat, st_r); + if (ret <= 0) { + if (ret == 0) + mail_set_expunged(mail); + return -1; + } + } else { + mail->transaction->stats.stat_lookup_count++; + path = maildir_save_file_get_path(mail->transaction, mail->seq); + if (stat(path, st_r) < 0) { + mail_set_critical(mail, "stat(%s) failed: %m", path); + return -1; + } + } + return 0; +} + +static int maildir_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct stat st; + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + + *date_r = data->received_date = st.st_mtime; + return 0; +} + +static int maildir_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct stat st; + + if (index_mail_get_save_date(_mail, date_r) > 0) + return 1; + + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + + *date_r = data->save_date = st.st_ctime; + return 1; +} + +static int +maildir_mail_get_fname(struct maildir_mailbox *mbox, struct mail *mail, + const char **fname_r) +{ + enum maildir_uidlist_rec_flag flags; + struct mail_index_view *view; + uint32_t seq; + bool exists; + int ret; + + ret = maildir_sync_lookup(mbox, mail->uid, &flags, fname_r); + if (ret != 0) + return ret; + + /* file exists in index file, but not in dovecot-uidlist anymore. */ + mail_set_expunged(mail); + + /* one reason this could happen is if we delayed opening + dovecot-uidlist and we're trying to open a mail that got recently + expunged. Let's test this theory first: */ + mail_index_refresh(mbox->box.index); + view = mail_index_view_open(mbox->box.index); + exists = mail_index_lookup_seq(view, mail->uid, &seq); + mail_index_view_close(&view); + + if (exists) { + /* the message still exists in index. this means there's some + kind of a desync, which doesn't get fixed if cur/ mtime is + the same as in index. fix this by forcing a resync. */ + (void)maildir_storage_sync_force(mbox, mail->uid); + } + return 0; +} + +static int maildir_get_pop3_state(struct index_mail *mail) +{ + struct mailbox *box = mail->mail.mail.box; + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const struct mail_cache_field *fields; + unsigned int i, count, psize_idx, vsize_idx; + enum mail_cache_decision_type dec, vsize_dec; + enum mail_fetch_field allowed_pop3_fields; + bool not_pop3_only = FALSE; + + if (mail->pop3_state_set) + return mail->pop3_state; + + /* if this mail itself has non-pop3 fields we know we're not + pop3-only */ + allowed_pop3_fields = MAIL_FETCH_FLAGS | MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY | MAIL_FETCH_STORAGE_ID | + MAIL_FETCH_VIRTUAL_SIZE; + + if (mail->data.wanted_headers != NULL || + (mail->data.wanted_fields & ENUM_NEGATE(allowed_pop3_fields)) != 0) + not_pop3_only = TRUE; + + /* get vsize decisions */ + psize_idx = ibox->cache_fields[MAIL_CACHE_PHYSICAL_FULL_SIZE].idx; + vsize_idx = ibox->cache_fields[MAIL_CACHE_VIRTUAL_FULL_SIZE].idx; + if (not_pop3_only) { + vsize_dec = mail_cache_field_get_decision(box->cache, + vsize_idx); + vsize_dec &= ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + } else { + /* also check if there are any non-[pv]size cached fields */ + vsize_dec = MAIL_CACHE_DECISION_NO; + fields = mail_cache_register_get_list(box->cache, + pool_datastack_create(), + &count); + for (i = 0; i < count; i++) { + dec = fields[i].decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + if (fields[i].idx == vsize_idx) + vsize_dec = dec; + else if (dec != MAIL_CACHE_DECISION_NO && + fields[i].idx != psize_idx) + not_pop3_only = TRUE; + } + } + + if (index_mail_get_vsize_extension(&mail->mail.mail) != NULL) { + /* having a vsize extension in index is the same as having + vsize's caching decision YES */ + vsize_dec = MAIL_CACHE_DECISION_YES; + } + + if (!not_pop3_only) { + /* either nothing is cached, or only vsize is cached. */ + mail->pop3_state = 1; + } else if (vsize_dec != MAIL_CACHE_DECISION_YES && + (box->flags & MAILBOX_FLAG_POP3_SESSION) == 0) { + /* if virtual size isn't cached permanently, + POP3 isn't being used */ + mail->pop3_state = -1; + } else { + /* possibly a mixed pop3/imap */ + mail->pop3_state = 0; + } + mail->pop3_state_set = TRUE; + return mail->pop3_state; +} + +static int maildir_quick_size_lookup(struct index_mail *mail, bool vsize, + uoff_t *size_r) +{ + struct mail *_mail = &mail->mail.mail; + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + enum maildir_uidlist_rec_ext_key key; + const char *path, *fname, *value; + + if (!_mail->saving) { + if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) + return -1; + } else { + if (maildir_save_file_get_size(_mail->transaction, _mail->seq, + vsize, size_r) == 0) + return 1; + + path = maildir_save_file_get_path(_mail->transaction, + _mail->seq); + fname = strrchr(path, '/'); + fname = fname != NULL ? fname + 1 : path; + } + + /* size can be included in filename */ + if (vsize || !mbox->storage->set->maildir_broken_filename_sizes) { + if (maildir_filename_get_size(fname, + vsize ? MAILDIR_EXTRA_VIRTUAL_SIZE : + MAILDIR_EXTRA_FILE_SIZE, size_r)) + return 1; + } + + /* size can be included in uidlist entry */ + if (!_mail->saving) { + key = vsize ? MAILDIR_UIDLIST_REC_EXT_VSIZE : + MAILDIR_UIDLIST_REC_EXT_PSIZE; + value = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + key); + if (value != NULL && str_to_uoff(value, size_r) == 0) + return 1; + } + return 0; +} + +static void +maildir_handle_size_caching(struct index_mail *mail, bool quick_check, + bool vsize) +{ + struct mailbox *box = mail->mail.mail.box; + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + enum mail_fetch_field field; + uoff_t size; + int pop3_state; + + field = vsize ? MAIL_FETCH_VIRTUAL_SIZE : MAIL_FETCH_PHYSICAL_SIZE; + if ((mail->data.dont_cache_fetch_fields & field) != 0) + return; + + if (quick_check && maildir_quick_size_lookup(mail, vsize, &size) > 0) { + /* already in filename / uidlist. don't add it anywhere, + including to the uidlist if it's already in filename. + do some extra checks here to catch potential cache bugs. */ + if (vsize && mail->data.virtual_size != size) { + mail_set_mail_cache_corrupted(&mail->mail.mail, + "Corrupted virtual size: " + "%"PRIuUOFF_T" != %"PRIuUOFF_T, + mail->data.virtual_size, size); + mail->data.virtual_size = size; + } else if (!vsize && mail->data.physical_size != size) { + mail_set_mail_cache_corrupted(&mail->mail.mail, + "Corrupted physical size: " + "%"PRIuUOFF_T" != %"PRIuUOFF_T, + mail->data.physical_size, size); + mail->data.physical_size = size; + } + mail->data.dont_cache_fetch_fields |= field; + return; + } + + /* 1 = pop3-only, 0 = mixed, -1 = no pop3 */ + pop3_state = maildir_get_pop3_state(mail); + if (pop3_state >= 0 && mail->mail.mail.uid != 0) { + /* if size is wanted permanently, store it to uidlist + so that in case cache file gets lost we can get it quickly */ + mail->data.dont_cache_fetch_fields |= field; + size = vsize ? mail->data.virtual_size : + mail->data.physical_size; + maildir_uidlist_set_ext(mbox->uidlist, mail->mail.mail.uid, + vsize ? MAILDIR_UIDLIST_REC_EXT_VSIZE : + MAILDIR_UIDLIST_REC_EXT_PSIZE, + dec2str(size)); + } +} + +static int maildir_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + struct index_mail *mail = INDEX_MAIL(_mail); + struct index_mail_data *data = &mail->data; + struct message_size hdr_size, body_size; + struct istream *input; + uoff_t old_offset; + + if (maildir_uidlist_is_read(mbox->uidlist) || + (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) { + /* try to get the size from uidlist. this is especially useful + with pop3 to avoid unnecessarily opening the cache file. */ + if (maildir_quick_size_lookup(mail, TRUE, + &data->virtual_size) < 0) + return -1; + } + + if (data->virtual_size == UOFF_T_MAX) { + if (index_mail_get_cached_virtual_size(mail, size_r)) { + i_assert(mail->data.virtual_size != UOFF_T_MAX); + maildir_handle_size_caching(mail, TRUE, TRUE); + return 0; + } + if (maildir_quick_size_lookup(mail, TRUE, + &data->virtual_size) < 0) + return -1; + } + if (data->virtual_size != UOFF_T_MAX) { + data->dont_cache_fetch_fields |= MAIL_FETCH_VIRTUAL_SIZE; + *size_r = data->virtual_size; + return 0; + } + + /* fallback to reading the file */ + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0) + return -1; + i_stream_seek(data->stream, old_offset); + + maildir_handle_size_caching(mail, FALSE, TRUE); + *size_r = data->virtual_size; + return 0; +} + +static int maildir_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + struct index_mail_data *data = &mail->data; + struct stat st; + struct message_size hdr_size, body_size; + struct istream *input; + const char *path; + int ret; + + if (maildir_uidlist_is_read(mbox->uidlist) || + (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) { + /* try to get the size from uidlist (see virtual size above) */ + if (maildir_quick_size_lookup(mail, FALSE, + &data->physical_size) < 0) + return -1; + } + + if (data->physical_size == UOFF_T_MAX) { + if (index_mail_get_physical_size(_mail, size_r) == 0) { + i_assert(mail->data.physical_size != UOFF_T_MAX); + maildir_handle_size_caching(mail, TRUE, FALSE); + return 0; + } + if (maildir_quick_size_lookup(mail, FALSE, + &data->physical_size) < 0) + return -1; + } + if (data->physical_size != UOFF_T_MAX) { + data->dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE; + *size_r = data->physical_size; + return 0; + } + + if (mail->mail.v.istream_opened != NULL) { + /* we can't use stat(), because this may be a mail that some + plugin has changed (e.g. zlib). need to do it the slow + way. */ + if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0) + return -1; + st.st_size = hdr_size.physical_size + body_size.physical_size; + } else if (!_mail->saving) { + ret = maildir_file_do(mbox, _mail->uid, do_stat, &st); + if (ret <= 0) { + if (ret == 0) + mail_set_expunged(_mail); + return -1; + } + } else { + /* saved mail which hasn't been committed yet */ + path = maildir_save_file_get_path(_mail->transaction, + _mail->seq); + if (stat(path, &st) < 0) { + mail_set_critical(_mail, "stat(%s) failed: %m", path); + return -1; + } + } + + data->physical_size = st.st_size; + maildir_handle_size_caching(mail, FALSE, FALSE); + *size_r = st.st_size; + return 0; +} + +static int +maildir_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + const char *path, *fname = NULL, *end, *guid, *uidl, *order; + struct stat st; + + switch (field) { + case MAIL_FETCH_GUID: + /* use GUID from uidlist if it exists */ + i_assert(!_mail->saving); + + if (mail->data.guid != NULL) { + *value_r = mail->data.guid; + return 0; + } + + /* first make sure that we have a refreshed uidlist */ + if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) + return -1; + + guid = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_GUID); + if (guid != NULL) { + if (*guid != '\0') { + *value_r = mail->data.guid = + p_strdup(mail->mail.data_pool, guid); + return 0; + } + + mail_set_critical(_mail, + "Maildir: Corrupted dovecot-uidlist: " + "UID had empty GUID, clearing it"); + maildir_uidlist_unset_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_GUID); + } + + /* default to base filename: */ + if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID, + value_r) < 0) + return -1; + mail->data.guid = mail->data.filename; + return 0; + case MAIL_FETCH_STORAGE_ID: + if (mail->data.filename != NULL) { + *value_r = mail->data.filename; + return 0; + } + if (fname != NULL) { + /* we came here from MAIL_FETCH_GUID, + avoid a second lookup */ + } else if (!_mail->saving) { + if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) + return -1; + } else { + path = maildir_save_file_get_path(_mail->transaction, + _mail->seq); + fname = strrchr(path, '/'); + fname = fname != NULL ? fname + 1 : path; + } + end = strchr(fname, MAILDIR_INFO_SEP); + mail->data.filename = end == NULL ? + p_strdup(mail->mail.data_pool, fname) : + p_strdup_until(mail->mail.data_pool, fname, end); + *value_r = mail->data.filename; + return 0; + case MAIL_FETCH_UIDL_BACKEND: + uidl = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); + if (uidl == NULL) { + /* use the default */ + *value_r = ""; + } else if (*uidl == '\0') { + /* special optimization case: use the base file name */ + return maildir_mail_get_special(_mail, + MAIL_FETCH_STORAGE_ID, value_r); + } else { + *value_r = p_strdup(mail->mail.data_pool, uidl); + } + return 0; + case MAIL_FETCH_POP3_ORDER: + order = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_POP3_ORDER); + if (order == NULL) { + *value_r = ""; + } else { + *value_r = p_strdup(mail->mail.data_pool, order); + } + return 0; + case MAIL_FETCH_REFCOUNT: + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + *value_r = p_strdup_printf(mail->mail.data_pool, "%lu", + (unsigned long)st.st_nlink); + return 0; + case MAIL_FETCH_REFCOUNT_ID: + if (maildir_mail_stat(_mail, &st) < 0) + return -1; + *value_r = p_strdup_printf(mail->mail.data_pool, "%llu", + (unsigned long long)st.st_ino); + return 0; + default: + return index_mail_get_special(_mail, field, value_r); + } +} + +static int +maildir_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED, + struct message_size *hdr_size, + struct message_size *body_size, + struct istream **stream_r) +{ + struct index_mail *mail = INDEX_MAIL(_mail); + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + struct index_mail_data *data = &mail->data; + bool deleted; + + if (data->stream == NULL) { + data->stream = maildir_open_mail(mbox, _mail, &deleted); + if (data->stream == NULL) { + if (deleted) + mail_set_expunged(_mail); + return -1; + } + if (mail->mail.v.istream_opened != NULL) { + if (mail->mail.v.istream_opened(_mail, + &data->stream) < 0) { + i_stream_unref(&data->stream); + return -1; + } + } + } + + return index_mail_init_stream(mail, hdr_size, body_size, stream_r); +} + +static void maildir_update_pop3_uidl(struct mail *_mail, const char *uidl) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box); + const char *fname; + + if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID, + &fname) == 0 && + strcmp(uidl, fname) == 0) { + /* special case optimization: empty UIDL means the same + as base filename */ + uidl = ""; + } + + maildir_uidlist_set_ext(mbox->uidlist, _mail->uid, + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL, uidl); +} + +static void maildir_mail_remove_sizes_from_uidlist(struct mail *mail) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box); + + if (maildir_uidlist_lookup_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_VSIZE) != NULL) { + maildir_uidlist_unset_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_VSIZE); + } + if (maildir_uidlist_lookup_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_PSIZE) != NULL) { + maildir_uidlist_unset_ext(mbox->uidlist, mail->uid, + MAILDIR_UIDLIST_REC_EXT_PSIZE); + } +} + +struct maildir_size_fix_ctx { + uoff_t physical_size; + char wrong_key; +}; + +static int +do_fix_size(struct maildir_mailbox *mbox, const char *path, + struct maildir_size_fix_ctx *ctx) +{ + const char *fname, *newpath, *extra, *info, *dir; + struct stat st; + + fname = strrchr(path, '/'); + i_assert(fname != NULL); + dir = t_strdup_until(path, fname++); + + extra = strchr(fname, MAILDIR_EXTRA_SEP); + i_assert(extra != NULL); + info = strchr(fname, MAILDIR_INFO_SEP); + if (info == NULL) info = ""; + + if (ctx->physical_size == UOFF_T_MAX) { + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + mailbox_set_critical(&mbox->box, + "stat(%s) failed: %m", path); + return -1; + } + ctx->physical_size = st.st_size; + } + + newpath = t_strdup_printf("%s/%s,S=%"PRIuUOFF_T"%s", dir, + t_strdup_until(fname, extra), + ctx->physical_size, info); + + if (rename(path, newpath) == 0) { + mailbox_set_critical(&mbox->box, + "Maildir filename has wrong %c value, " + "renamed the file from %s to %s", + ctx->wrong_key, path, newpath); + return 1; + } + if (errno == ENOENT) + return 0; + + mailbox_set_critical(&mbox->box, "rename(%s, %s) failed: %m", + path, newpath); + return -1; +} + +static void +maildir_mail_remove_sizes_from_filename(struct mail *mail, + enum mail_fetch_field field) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box); + struct mail_private *pmail = (struct mail_private *)mail; + enum maildir_uidlist_rec_flag flags; + const char *fname; + uoff_t size; + struct maildir_size_fix_ctx ctx; + + if (mbox->storage->set->maildir_broken_filename_sizes) { + /* never try to fix sizes in maildir filenames */ + return; + } + + if (maildir_sync_lookup(mbox, mail->uid, &flags, &fname) <= 0) + return; + if (strchr(fname, MAILDIR_EXTRA_SEP) == NULL) + return; + + i_zero(&ctx); + ctx.physical_size = UOFF_T_MAX; + if (field == MAIL_FETCH_VIRTUAL_SIZE && + maildir_filename_get_size(fname, MAILDIR_EXTRA_VIRTUAL_SIZE, + &size)) { + ctx.wrong_key = 'W'; + } else if (field == MAIL_FETCH_PHYSICAL_SIZE && + maildir_filename_get_size(fname, MAILDIR_EXTRA_FILE_SIZE, + &size)) { + ctx.wrong_key = 'S'; + } else { + /* the broken size isn't in filename */ + return; + } + + if (pmail->v.istream_opened != NULL) { + /* the mail could be e.g. compressed. get the physical size + the slow way by actually reading the mail. */ + struct istream *input; + const struct stat *stp; + + if (mail_get_stream(mail, NULL, NULL, &input) < 0) + return; + if (i_stream_stat(input, TRUE, &stp) < 0) + return; + ctx.physical_size = stp->st_size; + } + + (void)maildir_file_do(mbox, mail->uid, do_fix_size, &ctx); +} + +static void maildir_mail_set_cache_corrupted(struct mail *_mail, + enum mail_fetch_field field, + const char *reason) +{ + if (field == MAIL_FETCH_PHYSICAL_SIZE || + field == MAIL_FETCH_VIRTUAL_SIZE) { + maildir_mail_remove_sizes_from_uidlist(_mail); + maildir_mail_remove_sizes_from_filename(_mail, field); + } + index_mail_set_cache_corrupted(_mail, field, reason); +} + +struct mail_vfuncs maildir_mail_vfuncs = { + index_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + maildir_mail_get_received_date, + maildir_mail_get_save_date, + maildir_mail_get_virtual_size, + maildir_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + maildir_mail_get_stream, + index_mail_get_binary_stream, + maildir_mail_get_special, + index_mail_get_backend_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + maildir_update_pop3_uidl, + index_mail_expunge, + maildir_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c new file mode 100644 index 0000000..5cf7d6a --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-save.c @@ -0,0 +1,1084 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "fdatasync-path.h" +#include "eacces-error.h" +#include "str.h" +#include "index-mail.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename.h" +#include "maildir-filename-flags.h" +#include "maildir-sync.h" +#include "mailbox-recent-flags.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <utime.h> +#include <sys/stat.h> + +#define MAILDIR_FILENAME_FLAG_MOVED 0x10000000 + +struct maildir_filename { + struct maildir_filename *next; + const char *tmp_name, *dest_basename; + const char *pop3_uidl, *guid; + + uoff_t size, vsize; + enum mail_flags flags; + unsigned int pop3_order; + bool preserve_filename:1; + ARRAY_TYPE(keyword_indexes) keywords; +}; + +struct maildir_save_context { + struct mail_save_context ctx; + pool_t pool; + + struct maildir_mailbox *mbox; + struct mail_index_transaction *trans; + struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; + struct maildir_keywords_sync_ctx *keywords_sync_ctx; + struct maildir_index_sync_context *sync_ctx; + struct mail *cur_dest_mail; + + const char *tmpdir, *newdir, *curdir; + struct maildir_filename *files, **files_tail, *file_last; + unsigned int files_count; + + struct istream *input; + int fd; + uint32_t first_seq, seq, last_nonrecent_uid; + + bool have_keywords:1; + bool have_preserved_filenames:1; + bool locked:1; + bool failed:1; + bool last_save_finished:1; + bool locked_uidlist_refresh:1; +}; + +#define MAILDIR_SAVECTX(s) container_of(s, struct maildir_save_context, ctx) + +static int maildir_file_move(struct maildir_save_context *ctx, + struct maildir_filename *mf, const char *destname, + bool newdir) +{ + struct mail_storage *storage = &ctx->mbox->storage->storage; + const char *tmp_path, *new_path; + + i_assert(*destname != '\0'); + i_assert(*mf->tmp_name != '\0'); + + /* if we have flags, we'll move it to cur/ directly, because files in + new/ directory can't have flags. alternative would be to write it + in new/ and set the flags dirty in index file, but in that case + external MUAs would see wrong flags. */ + tmp_path = t_strconcat(ctx->tmpdir, "/", mf->tmp_name, NULL); + new_path = newdir ? + t_strconcat(ctx->newdir, "/", destname, NULL) : + t_strconcat(ctx->curdir, "/", destname, NULL); + + /* maildir spec says we should use link() + unlink() here. however + since our filename is guaranteed to be unique, rename() works just + as well, except faster. even if the filename wasn't unique, the + problem could still happen if the file was already moved from + new/ to cur/, so link() doesn't really provide any safety anyway. + + Besides the small temporary performance benefits, this rename() is + almost required with OSX's HFS+ filesystem, since it implements + hard links in a pretty ugly way, which makes the performance crawl + when a lot of hard links are used. */ + if (rename(tmp_path, new_path) == 0) { + mf->flags |= MAILDIR_FILENAME_FLAG_MOVED; + return 0; + } else if (ENOQUOTA(errno)) { + mail_storage_set_error(storage, MAIL_ERROR_NOQUOTA, + MAIL_ERRSTR_NO_QUOTA); + return -1; + } else { + mailbox_set_critical(&ctx->mbox->box, + "rename(%s, %s) failed: %m", + tmp_path, new_path); + return -1; + } +} + +static struct mail_save_context * +maildir_save_transaction_init(struct mailbox_transaction_context *t) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(t->box); + struct maildir_save_context *ctx; + const char *path; + pool_t pool; + + pool = pool_alloconly_create("maildir_save_context", 4096); + ctx = p_new(pool, struct maildir_save_context, 1); + ctx->ctx.transaction = t; + ctx->pool = pool; + ctx->mbox = mbox; + ctx->trans = t->itrans; + ctx->files_tail = &ctx->files; + ctx->fd = -1; + + path = mailbox_get_path(&mbox->box); + ctx->tmpdir = p_strconcat(pool, path, "/tmp", NULL); + ctx->newdir = p_strconcat(pool, path, "/new", NULL); + ctx->curdir = p_strconcat(pool, path, "/cur", NULL); + + ctx->last_save_finished = TRUE; + return &ctx->ctx; +} + +struct maildir_filename * +maildir_save_add(struct mail_save_context *_ctx, const char *tmp_fname, + struct mail *src_mail) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct mail_save_data *mdata = &_ctx->data; + struct maildir_filename *mf; + struct istream *input; + + i_assert(*tmp_fname != '\0'); + + /* allow caller to specify recent flag only when uid is specified + (we're replicating, converting, etc.). */ + if (mdata->uid == 0) + mdata->flags |= MAIL_RECENT; + else if ((mdata->flags & MAIL_RECENT) == 0 && + ctx->last_nonrecent_uid < mdata->uid) + ctx->last_nonrecent_uid = mdata->uid; + + /* now, we want to be able to rollback the whole append session, + so we'll just store the name of this temp file and move it later + into new/ or cur/. */ + mf = p_new(ctx->pool, struct maildir_filename, 1); + mf->tmp_name = mf->dest_basename = p_strdup(ctx->pool, tmp_fname); + mf->flags = mdata->flags; + mf->size = UOFF_T_MAX; + mf->vsize = UOFF_T_MAX; + + ctx->file_last = mf; + i_assert(*ctx->files_tail == NULL); + *ctx->files_tail = mf; + ctx->files_tail = &mf->next; + ctx->files_count++; + + if (mdata->pop3_uidl != NULL) + mf->pop3_uidl = p_strdup(ctx->pool, mdata->pop3_uidl); + mf->pop3_order = mdata->pop3_order; + + /* insert into index */ + mail_index_append(ctx->trans, mdata->uid, &ctx->seq); + mail_index_update_flags(ctx->trans, ctx->seq, + MODIFY_REPLACE, + mdata->flags & ENUM_NEGATE(MAIL_RECENT)); + if (mdata->keywords != NULL) { + mail_index_update_keywords(ctx->trans, ctx->seq, + MODIFY_REPLACE, mdata->keywords); + } + if (mdata->min_modseq != 0) { + mail_index_update_modseq(ctx->trans, ctx->seq, + mdata->min_modseq); + } + + if (ctx->first_seq == 0) { + ctx->first_seq = ctx->seq; + i_assert(ctx->files->next == NULL); + } + + mail_set_seq_saving(_ctx->dest_mail, ctx->seq); + + if (ctx->input == NULL) { + /* copying with hardlinking. */ + i_assert(src_mail != NULL); + index_copy_cache_fields(_ctx, src_mail, ctx->seq); + ctx->cur_dest_mail = NULL; + } else { + input = index_mail_cache_parse_init(_ctx->dest_mail, + ctx->input); + i_stream_unref(&ctx->input); + ctx->input = input; + ctx->cur_dest_mail = _ctx->dest_mail; + } + return mf; +} + +void maildir_save_set_dest_basename(struct mail_save_context *_ctx, + struct maildir_filename *mf, + const char *basename) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + mf->preserve_filename = TRUE; + mf->dest_basename = p_strdup(ctx->pool, basename); + ctx->have_preserved_filenames = TRUE; +} + +void maildir_save_set_sizes(struct maildir_filename *mf, + uoff_t size, uoff_t vsize) +{ + mf->size = size; + mf->vsize = vsize; +} + +static bool +maildir_get_dest_filename(struct maildir_save_context *ctx, + struct maildir_filename *mf, + const char **fname_r) +{ + const char *basename = mf->dest_basename; + + if (mf->size != UOFF_T_MAX && !mf->preserve_filename) { + basename = t_strdup_printf("%s,%c=%"PRIuUOFF_T, basename, + MAILDIR_EXTRA_FILE_SIZE, mf->size); + } + + if (mf->vsize != UOFF_T_MAX && !mf->preserve_filename) { + basename = t_strdup_printf("%s,%c=%"PRIuUOFF_T, basename, + MAILDIR_EXTRA_VIRTUAL_SIZE, + mf->vsize); + } + + if (!array_is_created(&mf->keywords) || array_count(&mf->keywords) == 0) { + if ((mf->flags & MAIL_FLAGS_MASK) == MAIL_RECENT) { + *fname_r = basename; + return TRUE; + } + + *fname_r = maildir_filename_flags_set(basename, + mf->flags & MAIL_FLAGS_MASK); + return FALSE; + } + + i_assert(ctx->keywords_sync_ctx != NULL || + !array_is_created(&mf->keywords) || array_count(&mf->keywords) == 0); + *fname_r = maildir_filename_flags_kw_set(ctx->keywords_sync_ctx, + basename, + mf->flags & MAIL_FLAGS_MASK, + &mf->keywords); + return FALSE; +} + +static const char *maildir_mf_get_path(struct maildir_save_context *ctx, + struct maildir_filename *mf) +{ + const char *fname, *dir; + + if ((mf->flags & MAILDIR_FILENAME_FLAG_MOVED) == 0) { + /* file is still in tmp/ */ + return t_strdup_printf("%s/%s", ctx->tmpdir, mf->tmp_name); + } + + /* already moved to new/ or cur/ */ + dir = maildir_get_dest_filename(ctx, mf, &fname) ? + ctx->newdir : ctx->curdir; + return t_strdup_printf("%s/%s", dir, fname); +} + + +static struct maildir_filename * +maildir_save_get_mf(struct mailbox_transaction_context *t, uint32_t seq) +{ + struct maildir_save_context *save_ctx = MAILDIR_SAVECTX(t->save_ctx); + struct maildir_filename *mf; + + i_assert(seq >= save_ctx->first_seq); + + seq -= save_ctx->first_seq; + mf = save_ctx->files; + while (seq > 0) { + mf = mf->next; + i_assert(mf != NULL); + seq--; + } + return mf; +} + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r) +{ + struct maildir_filename *mf = maildir_save_get_mf(t, seq); + + *size_r = vsize ? mf->vsize : mf->size; + return *size_r == UOFF_T_MAX ? -1 : 0; +} + +const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, + uint32_t seq) +{ + struct maildir_save_context *save_ctx = MAILDIR_SAVECTX(t->save_ctx); + struct maildir_filename *mf = maildir_save_get_mf(t, seq); + + return maildir_mf_get_path(save_ctx, mf); +} + +static int maildir_create_tmp(struct maildir_mailbox *mbox, const char *dir, + const char **fname_r) +{ + struct mailbox *box = &mbox->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + size_t prefix_len; + const char *tmp_fname; + string_t *path; + mode_t old_mask; + int fd; + + path = t_str_new(256); + str_append(path, dir); + str_append_c(path, '/'); + prefix_len = str_len(path); + + do { + tmp_fname = maildir_filename_generate(); + str_truncate(path, prefix_len); + str_append(path, tmp_fname); + + /* the generated filename is unique. the only reason why it + might return an existing filename is if the time moved + backwards. so we'll use O_EXCL anyway, although it's mostly + useless. */ + old_mask = umask(0777 & ~perm->file_create_mode); + fd = open(str_c(path), + O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0777); + umask(old_mask); + } while (fd == -1 && errno == EEXIST); + + *fname_r = tmp_fname; + if (fd == -1) { + if (ENOQUOTA(errno)) { + mail_storage_set_error(box->storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + } else { + mailbox_set_critical(box, + "open(%s) failed: %m", str_c(path)); + } + } else if (perm->file_create_gid != (gid_t)-1) { + if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", + str_c(path), + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, + "fchown(%s) failed: %m", str_c(path)); + } + } + } + + return fd; +} + +struct mail_save_context * +maildir_save_alloc(struct mailbox_transaction_context *t) +{ + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx == NULL) + t->save_ctx = maildir_save_transaction_init(t); + return t->save_ctx; +} + +int maildir_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct maildir_filename *mf; + + /* new mail, new failure state */ + ctx->failed = FALSE; + + T_BEGIN { + /* create a new file in tmp/ directory */ + const char *fname; + + ctx->fd = maildir_create_tmp(ctx->mbox, ctx->tmpdir, &fname); + if (ctx->fd == -1) + ctx->failed = TRUE; + else { + if (ctx->mbox->storage->storage.set->mail_save_crlf) + ctx->input = i_stream_create_crlf(input); + else + ctx->input = i_stream_create_lf(input); + mf = maildir_save_add(_ctx, fname, NULL); + if (_ctx->data.guid != NULL) { + maildir_save_set_dest_basename(_ctx, mf, + _ctx->data.guid); + } + } + } T_END; + + if (!ctx->failed) { + _ctx->data.output = o_stream_create_fd_file(ctx->fd, 0, FALSE); + o_stream_set_name(_ctx->data.output, t_strdup_printf( + "%s/%s", ctx->tmpdir, ctx->file_last->tmp_name)); + o_stream_cork(_ctx->data.output); + ctx->last_save_finished = FALSE; + } + return ctx->failed ? -1 : 0; +} + +int maildir_save_continue(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + if (ctx->failed) + return -1; + + if (index_storage_save_continue(_ctx, ctx->input, + ctx->cur_dest_mail) < 0) { + ctx->failed = TRUE; + return -1; + } + return 0; +} + +static int maildir_save_finish_received_date(struct maildir_save_context *ctx, + const char *path) +{ + struct utimbuf buf; + struct stat st; + + if (ctx->ctx.data.received_date != (time_t)-1) { + /* set the received_date by modifying mtime */ + buf.actime = ioloop_time; + buf.modtime = ctx->ctx.data.received_date; + + if (utime(path, &buf) < 0) { + mail_set_critical(ctx->ctx.dest_mail, + "utime(%s) failed: %m", path); + return -1; + } + } else if (ctx->fd != -1) { + if (fstat(ctx->fd, &st) == 0) + ctx->ctx.data.received_date = st.st_mtime; + else { + mail_set_critical(ctx->ctx.dest_mail, + "fstat(%s) failed: %m", path); + return -1; + } + } else { + /* hardlinked */ + if (stat(path, &st) == 0) + ctx->ctx.data.received_date = st.st_mtime; + else { + mail_set_critical(ctx->ctx.dest_mail, + "stat(%s) failed: %m", path); + return -1; + } + } + return 0; +} + +static void maildir_save_remove_last_filename(struct maildir_save_context *ctx) +{ + struct maildir_filename **fm; + + index_storage_save_abort_last(&ctx->ctx, ctx->seq); + ctx->seq--; + + for (fm = &ctx->files; (*fm)->next != NULL; fm = &(*fm)->next) ; + i_assert(*fm == ctx->file_last); + *fm = NULL; + + ctx->files_tail = fm; + ctx->file_last = NULL; + ctx->files_count--; +} + +void maildir_save_finish_keywords(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + ARRAY_TYPE(keyword_indexes) keyword_idx; + t_array_init(&keyword_idx, 8); + mail_index_lookup_keywords(ctx->ctx.transaction->view, ctx->seq, + &keyword_idx); + + if (array_count(&keyword_idx) > 0) { + /* copy keywords */ + p_array_init(&ctx->file_last->keywords, ctx->pool, + array_count(&keyword_idx)); + array_copy(&ctx->file_last->keywords.arr, 0, &keyword_idx.arr, 0, + array_count(&keyword_idx)); + ctx->have_keywords = TRUE; + } +} + +static int maildir_save_finish_real(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct mail_storage *storage = &ctx->mbox->storage->storage; + const char *path, *output_errstr; + off_t real_size; + uoff_t size; + int output_errno; + + ctx->last_save_finished = TRUE; + if (ctx->failed && ctx->fd == -1) { + /* tmp file creation failed */ + return -1; + } + + path = t_strconcat(ctx->tmpdir, "/", ctx->file_last->tmp_name, NULL); + if (o_stream_finish(_ctx->data.output) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(_ctx->dest_mail, + "write(%s) failed: %s", path, + o_stream_get_error(_ctx->data.output)); + } + ctx->failed = TRUE; + } + + if (_ctx->data.save_date != (time_t)-1) { + /* we can't change ctime, but we can add the date to cache */ + struct index_mail *mail = INDEX_MAIL(_ctx->dest_mail); + uint32_t t = _ctx->data.save_date; + + index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t)); + } + + if (maildir_save_finish_received_date(ctx, path) < 0) + ctx->failed = TRUE; + + if (ctx->cur_dest_mail != NULL) { + index_mail_cache_parse_deinit(ctx->cur_dest_mail, + ctx->ctx.data.received_date, + !ctx->failed); + } + i_stream_unref(&ctx->input); + + /* remember the size in case we want to add it to filename */ + ctx->file_last->size = _ctx->data.output->offset; + if (ctx->cur_dest_mail == NULL || + mail_get_virtual_size(ctx->cur_dest_mail, + &ctx->file_last->vsize) < 0) + ctx->file_last->vsize = UOFF_T_MAX; + + output_errno = _ctx->data.output->stream_errno; + output_errstr = t_strdup(o_stream_get_error(_ctx->data.output)); + o_stream_destroy(&_ctx->data.output); + + maildir_save_finish_keywords(_ctx); + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER && + !ctx->failed) { + if (fsync(ctx->fd) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(_ctx->dest_mail, + "fsync(%s) failed: %m", path); + } + ctx->failed = TRUE; + } + } + real_size = lseek(ctx->fd, 0, SEEK_END); + if (real_size == (off_t)-1) { + mail_set_critical(_ctx->dest_mail, "lseek(%s) failed: %m", path); + } else if (real_size != (off_t)ctx->file_last->size && + (!maildir_filename_get_size(ctx->file_last->dest_basename, + MAILDIR_EXTRA_FILE_SIZE, &size) || + size != ctx->file_last->size)) { + /* e.g. zlib plugin was used. the "physical size" must be in + the maildir filename, since stat() will return wrong size */ + ctx->file_last->preserve_filename = FALSE; + /* preserve the GUID if needed */ + if (ctx->file_last->guid == NULL) + ctx->file_last->guid = ctx->file_last->dest_basename; + /* reset the base name as well, just in case there's a + ,W=vsize */ + ctx->file_last->dest_basename = ctx->file_last->tmp_name; + } + if (close(ctx->fd) < 0) { + if (!mail_storage_set_error_from_errno(storage)) { + mail_set_critical(_ctx->dest_mail, + "close(%s) failed: %m", path); + } + ctx->failed = TRUE; + } + ctx->fd = -1; + + if (ctx->failed) { + /* delete the tmp file */ + i_unlink_if_exists(path); + + if (ENOQUOTA(output_errno)) { + mail_storage_set_error(storage, + MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); + } else if (output_errno != 0) { + mail_set_critical(_ctx->dest_mail, + "write(%s) failed: %s", path, output_errstr); + } + + maildir_save_remove_last_filename(ctx); + return -1; + } + + ctx->file_last = NULL; + return 0; +} + +int maildir_save_finish(struct mail_save_context *ctx) +{ + int ret; + + T_BEGIN { + ret = maildir_save_finish_real(ctx); + } T_END; + index_save_context_free(ctx); + return ret; +} + +void maildir_save_cancel(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)maildir_save_finish(_ctx); +} + +static void +maildir_save_unlink_files(struct maildir_save_context *ctx) +{ + struct maildir_filename *mf; + + for (mf = ctx->files; mf != NULL; mf = mf->next) T_BEGIN { + i_unlink(maildir_mf_get_path(ctx, mf)); + } T_END; + ctx->files = NULL; +} + +static int maildir_transaction_fsync_dirs(struct maildir_save_context *ctx, + bool new_changed, bool cur_changed) +{ + struct mail_storage *storage = &ctx->mbox->storage->storage; + + if (storage->set->parsed_fsync_mode == FSYNC_MODE_NEVER) + return 0; + + if (new_changed) { + if (fdatasync_path(ctx->newdir) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "fdatasync_path(%s) failed: %m", ctx->newdir); + return -1; + } + } + if (cur_changed) { + if (fdatasync_path(ctx->curdir) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "fdatasync_path(%s) failed: %m", ctx->curdir); + return -1; + } + } + return 0; +} + +static int seq_range_cmp(const struct seq_range *r1, const struct seq_range *r2) +{ + if (r1->seq1 < r2->seq2) + return -1; + else if (r1->seq1 > r2->seq2) + return 1; + else + return 0; +} + +static uint32_t +maildir_save_set_recent_flags(struct maildir_save_context *ctx) +{ + struct maildir_mailbox *mbox = ctx->mbox; + ARRAY_TYPE(seq_range) saved_sorted_uids; + const struct seq_range *uids; + unsigned int i, count; + uint32_t uid; + + count = array_count(&ctx->ctx.transaction->changes->saved_uids); + if (count == 0) + return 0; + + t_array_init(&saved_sorted_uids, count); + array_append_array(&saved_sorted_uids, + &ctx->ctx.transaction->changes->saved_uids); + array_sort(&saved_sorted_uids, seq_range_cmp); + + uids = array_get(&saved_sorted_uids, &count); + for (i = 0; i < count; i++) { + for (uid = uids[i].seq1; uid <= uids[i].seq2; uid++) + mailbox_recent_flags_set_uid(&mbox->box, uid); + } + return uids[count-1].seq2 + 1; +} + +static int +maildir_save_sync_index(struct maildir_save_context *ctx) +{ + struct mailbox_transaction_context *_t = ctx->ctx.transaction; + struct maildir_mailbox *mbox = ctx->mbox; + uint32_t first_uid, next_uid, first_recent_uid; + int ret; + + /* we'll need to keep the lock past the sync deinit */ + ret = maildir_uidlist_lock(mbox->uidlist); + i_assert(ret > 0); + + if (maildir_sync_header_refresh(mbox) < 0) + return -1; + if ((ret = maildir_uidlist_refresh_fast_init(mbox->uidlist)) < 0) + return -1; + + if (ret == 0) { + /* uidlist doesn't exist. make sure all existing message + are added to uidlist first. */ + (void)maildir_storage_sync_force(mbox, 0); + } + + if (maildir_sync_index_begin(mbox, NULL, &ctx->sync_ctx) < 0) + return -1; + ctx->keywords_sync_ctx = + maildir_sync_get_keywords_sync_ctx(ctx->sync_ctx); + + /* now that uidlist is locked, make sure all the existing mails + have been added to index. we don't really look into the + maildir, just add all the new mails listed in + dovecot-uidlist to index. */ + if (maildir_sync_index(ctx->sync_ctx, TRUE) < 0) + return -1; + + /* if messages were added to index, assign them UIDs */ + first_uid = maildir_uidlist_get_next_uid(mbox->uidlist); + i_assert(first_uid != 0); + mail_index_append_finish_uids(ctx->trans, first_uid, + &_t->changes->saved_uids); + i_assert(ctx->files_count == seq_range_count(&_t->changes->saved_uids)); + + /* these mails are all recent in our session */ + T_BEGIN { + next_uid = maildir_save_set_recent_flags(ctx); + } T_END; + + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) + first_recent_uid = next_uid; + else if (ctx->last_nonrecent_uid != 0) + first_recent_uid = ctx->last_nonrecent_uid + 1; + else + first_recent_uid = 0; + + if (first_recent_uid != 0) { + /* maildir_sync_index() dropped recent flags from + existing messages. we'll still need to drop recent + flags from these newly added messages. */ + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, + first_recent_uid), + &first_recent_uid, + sizeof(first_recent_uid), FALSE); + } + return 0; +} + +static void +maildir_save_rollback_index_changes(struct maildir_save_context *ctx) +{ + uint32_t seq; + + if (ctx->seq == 0) + return; + + for (seq = ctx->seq; seq >= ctx->first_seq; seq--) + mail_index_expunge(ctx->trans, seq); + + mail_cache_transaction_reset(ctx->ctx.transaction->cache_trans); +} + +static bool maildir_filename_has_conflict(struct maildir_filename *mf, + struct maildir_filename *prev_mf) +{ + if (strcmp(mf->dest_basename, prev_mf->dest_basename) == 0) { + /* already used this */ + return TRUE; + } + if (prev_mf->guid != NULL && + strcmp(mf->dest_basename, prev_mf->guid) == 0) { + /* previous filename also had a conflict */ + return TRUE; + } + return FALSE; +} + +static void +maildir_filename_check_conflicts(struct maildir_save_context *ctx, + struct maildir_filename *mf, + struct maildir_filename *prev_mf) +{ + uoff_t size, vsize; + + if (!ctx->locked_uidlist_refresh && ctx->locked) { + (void)maildir_uidlist_refresh(ctx->mbox->uidlist); + ctx->locked_uidlist_refresh = TRUE; + } + + if (!maildir_filename_get_size(mf->dest_basename, + MAILDIR_EXTRA_FILE_SIZE, &size)) + size = UOFF_T_MAX; + if (!maildir_filename_get_size(mf->dest_basename, + MAILDIR_EXTRA_VIRTUAL_SIZE, &vsize)) + vsize = UOFF_T_MAX; + + if (size != mf->size || vsize != mf->vsize || + !ctx->locked_uidlist_refresh || + (prev_mf != NULL && maildir_filename_has_conflict(mf, prev_mf)) || + maildir_uidlist_get_full_filename(ctx->mbox->uidlist, + mf->dest_basename) != NULL) { + /* a) dest_basename didn't contain the (correct) size/vsize. + they're required for good performance. + + b) file already exists. give it another name. + but preserve the size/vsize in the filename if possible */ + if (mf->size == UOFF_T_MAX) + mf->size = size; + if (mf->vsize == UOFF_T_MAX) + mf->vsize = size; + + mf->guid = mf->dest_basename; + mf->dest_basename = p_strdup(ctx->pool, + maildir_filename_generate()); + mf->preserve_filename = FALSE; + } +} + +static int +maildir_filename_dest_basename_cmp(struct maildir_filename *const *f1, + struct maildir_filename *const *f2) +{ + return strcmp((*f1)->dest_basename, (*f2)->dest_basename); +} + +static int +maildir_save_move_files_to_newcur(struct maildir_save_context *ctx) +{ + ARRAY(struct maildir_filename *) files; + struct maildir_filename *mf, *prev_mf; + bool newdir, new_changed, cur_changed; + int ret; + + /* put files into an array sorted by the destination filename. + this way we can easily check if there are duplicate destination + filenames within this transaction. */ + t_array_init(&files, ctx->files_count); + for (mf = ctx->files; mf != NULL; mf = mf->next) + array_push_back(&files, &mf); + array_sort(&files, maildir_filename_dest_basename_cmp); + + new_changed = cur_changed = FALSE; + prev_mf = NULL; + array_foreach_elem(&files, mf) { + T_BEGIN { + const char *dest; + + if (mf->preserve_filename) { + maildir_filename_check_conflicts(ctx, mf, + prev_mf); + } + + newdir = maildir_get_dest_filename(ctx, mf, &dest); + if (newdir) + new_changed = TRUE; + else + cur_changed = TRUE; + ret = maildir_file_move(ctx, mf, dest, newdir); + } T_END; + if (ret < 0) + return -1; + prev_mf = mf; + } + + if (ctx->locked) { + i_assert(ctx->sync_ctx != NULL); + maildir_sync_set_new_msgs_count(ctx->sync_ctx, + array_count(&files)); + } + return maildir_transaction_fsync_dirs(ctx, new_changed, cur_changed); +} + +static void maildir_save_sync_uidlist(struct maildir_save_context *ctx) +{ + struct mailbox_transaction_context *t = ctx->ctx.transaction; + struct maildir_filename *mf; + struct seq_range_iter iter; + enum maildir_uidlist_rec_flag flags; + struct maildir_uidlist_rec *rec; + unsigned int n = 0; + uint32_t uid; + bool newdir, bret; + int ret; + + seq_range_array_iter_init(&iter, &t->changes->saved_uids); + for (mf = ctx->files; mf != NULL; mf = mf->next) T_BEGIN { + const char *dest; + + bret = seq_range_array_iter_nth(&iter, n++, &uid); + i_assert(bret); + + newdir = maildir_get_dest_filename(ctx, mf, &dest); + flags = MAILDIR_UIDLIST_REC_FLAG_RECENT; + if (newdir) + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + ret = maildir_uidlist_sync_next_uid(ctx->uidlist_sync_ctx, + dest, uid, flags, &rec); + i_assert(ret > 0); + i_assert(rec != NULL); + if (mf->guid != NULL) { + maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec, + MAILDIR_UIDLIST_REC_EXT_GUID, mf->guid); + } + if (mf->pop3_uidl != NULL) { + maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec, + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL, + mf->pop3_uidl); + } + if (mf->pop3_order > 0) { + maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec, + MAILDIR_UIDLIST_REC_EXT_POP3_ORDER, + t_strdup_printf("%u", mf->pop3_order)); + } + } T_END; + i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); +} + +int maildir_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + enum maildir_uidlist_sync_flags sync_flags; + int ret; + + i_assert(_ctx->data.output == NULL); + i_assert(ctx->last_save_finished); + + if (ctx->files_count == 0) + return 0; + + sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL | + MAILDIR_UIDLIST_SYNC_NOREFRESH; + + if ((_t->flags & MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS) != 0) { + /* we want to assign UIDs, we must lock uidlist */ + } else if (ctx->have_keywords) { + /* keywords file updating relies on uidlist lock. */ + } else if (ctx->have_preserved_filenames) { + /* we're trying to use some potentially existing filenames. + we must lock to avoid race conditions where two sessions + try to save the same filename. */ + } else { + /* no requirement to lock uidlist. if we happen to get a lock, + assign uids. */ + sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK; + } + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags, + &ctx->uidlist_sync_ctx); + if (ret > 0) { + ctx->locked = TRUE; + if (maildir_save_sync_index(ctx) < 0) { + maildir_transaction_save_rollback(_ctx); + return -1; + } + } else if (ret == 0 && + (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0) { + ctx->locked = FALSE; + i_assert(ctx->uidlist_sync_ctx == NULL); + /* since we couldn't lock uidlist, we'll have to drop the + appends to index. */ + maildir_save_rollback_index_changes(ctx); + } else { + maildir_transaction_save_rollback(_ctx); + return -1; + } + + T_BEGIN { + ret = maildir_save_move_files_to_newcur(ctx); + } T_END; + if (ctx->locked) { + if (ret == 0) { + /* update dovecot-uidlist file. */ + maildir_save_sync_uidlist(ctx); + } + + if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, + ret == 0) < 0) + ret = -1; + } + + _t->changes->uid_validity = + maildir_uidlist_get_uid_validity(ctx->mbox->uidlist); + + if (ctx->locked) { + /* It doesn't matter if index syncing fails */ + ctx->keywords_sync_ctx = NULL; + if (ret < 0) + maildir_sync_index_rollback(&ctx->sync_ctx); + else + (void)maildir_sync_index_commit(&ctx->sync_ctx); + } + + if (ret < 0) { + ctx->keywords_sync_ctx = !ctx->have_keywords ? NULL : + maildir_keywords_sync_init(ctx->mbox->keywords, + ctx->mbox->box.index); + + /* unlink the files we just moved in an attempt to rollback + the transaction. uidlist is still locked, so at least other + Dovecot instances haven't yet seen the files. we need + to have the keywords sync context to be able to generate + the destination filenames if keywords were used. */ + maildir_save_unlink_files(ctx); + + if (ctx->keywords_sync_ctx != NULL) + maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx); + /* returning failure finishes the save_context */ + maildir_transaction_save_rollback(_ctx); + return -1; + } + return 0; +} + +void maildir_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result ATTR_UNUSED) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + _ctx->transaction = NULL; /* transaction is already freed */ + + if (ctx->locked) + maildir_uidlist_unlock(ctx->mbox->uidlist); + pool_unref(&ctx->pool); +} + +void maildir_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx); + + i_assert(_ctx->data.output == NULL); + + if (!ctx->last_save_finished) + maildir_save_cancel(&ctx->ctx); + + /* delete files in tmp/ */ + maildir_save_unlink_files(ctx); + + if (ctx->uidlist_sync_ctx != NULL) + (void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE); + if (ctx->sync_ctx != NULL) + maildir_sync_index_rollback(&ctx->sync_ctx); + if (ctx->locked) + maildir_uidlist_unlock(ctx->mbox->uidlist); + + pool_unref(&ctx->pool); +} diff --git a/src/lib-storage/index/maildir/maildir-settings.c b/src/lib-storage/index/maildir/maildir-settings.c new file mode 100644 index 0000000..d986d18 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-settings.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "maildir-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct maildir_settings) + +static const struct setting_define maildir_setting_defines[] = { + DEF(BOOL, maildir_copy_with_hardlinks), + DEF(BOOL, maildir_very_dirty_syncs), + DEF(BOOL, maildir_broken_filename_sizes), + DEF(BOOL, maildir_empty_new), + + SETTING_DEFINE_LIST_END +}; + +static const struct maildir_settings maildir_default_settings = { + .maildir_copy_with_hardlinks = TRUE, + .maildir_very_dirty_syncs = FALSE, + .maildir_broken_filename_sizes = FALSE, + .maildir_empty_new = FALSE +}; + +static const struct setting_parser_info maildir_setting_parser_info = { + .module_name = "maildir", + .defines = maildir_setting_defines, + .defaults = &maildir_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct maildir_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info +}; + +const struct setting_parser_info *maildir_get_setting_parser_info(void) +{ + return &maildir_setting_parser_info; +} + diff --git a/src/lib-storage/index/maildir/maildir-settings.h b/src/lib-storage/index/maildir/maildir-settings.h new file mode 100644 index 0000000..cfcb732 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-settings.h @@ -0,0 +1,13 @@ +#ifndef MAILDIR_SETTINGS_H +#define MAILDIR_SETTINGS_H + +struct maildir_settings { + bool maildir_copy_with_hardlinks; + bool maildir_very_dirty_syncs; + bool maildir_broken_filename_sizes; + bool maildir_empty_new; +}; + +const struct setting_parser_info *maildir_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-storage.c b/src/lib-storage/index/maildir/maildir-storage.c new file mode 100644 index 0000000..e65579f --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-storage.c @@ -0,0 +1,749 @@ +/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "eacces-error.h" +#include "unlink-old-files.h" +#include "mailbox-uidvalidity.h" +#include "mailbox-list-private.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-sync.h" +#include "index-mail.h" + +#include <sys/stat.h> + +#define MAILDIR_LIST_CONTEXT(obj) \ + MODULE_CONTEXT(obj, maildir_mailbox_list_module) +#define MAILDIR_SUBFOLDER_FILENAME "maildirfolder" + +struct maildir_mailbox_list_context { + union mailbox_list_module_context module_ctx; + const struct maildir_settings *set; +}; + +extern struct mail_storage maildir_storage; +extern struct mailbox maildir_mailbox; + +static struct event_category event_category_maildir = { + .name = "maildir", + .parent = &event_category_storage, +}; + +static MODULE_CONTEXT_DEFINE_INIT(maildir_mailbox_list_module, + &mailbox_list_module_register); +static const char *maildir_subdirs[] = { "cur", "new", "tmp" }; + +static void maildir_mailbox_close(struct mailbox *box); + +static struct mail_storage *maildir_storage_alloc(void) +{ + struct maildir_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("maildir storage", 512+256); + storage = p_new(pool, struct maildir_storage, 1); + storage->storage = maildir_storage; + storage->storage.pool = pool; + return &storage->storage; +} + +static int +maildir_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, + const char **error_r ATTR_UNUSED) +{ + struct maildir_storage *storage = MAILDIR_STORAGE(_storage); + struct mailbox_list *list = ns->list; + const char *dir; + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + + storage->temp_prefix = p_strdup(_storage->pool, + mailbox_list_get_temp_prefix(list)); + + if (list->set.control_dir == NULL && list->set.inbox_path == NULL && + (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) { + /* put the temp files into tmp/ directory preferably */ + storage->temp_prefix = p_strconcat(_storage->pool, "tmp/", + storage->temp_prefix, NULL); + dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_DIR); + } else { + /* control dir should also be writable */ + dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + } + _storage->temp_path_prefix = p_strconcat(_storage->pool, dir, "/", + storage->temp_prefix, NULL); + return 0; +} + +static void maildir_storage_get_list_settings(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + if (set->layout == NULL) + set->layout = MAILBOX_LIST_NAME_MAILDIRPLUSPLUS; + if (set->subscription_fname == NULL) + set->subscription_fname = MAILDIR_SUBSCRIPTION_FILE_NAME; + + if (set->inbox_path == NULL && *set->maildir_name == '\0' && + (strcmp(set->layout, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) == 0 || + strcmp(set->layout, MAILBOX_LIST_NAME_FS) == 0) && + (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) { + /* Maildir++ INBOX is the Maildir base itself */ + set->inbox_path = set->root_dir; + } +} + +static const char * +maildir_storage_find_root_dir(const struct mail_namespace *ns) +{ + bool debug = ns->mail_set->mail_debug; + const char *home, *path; + + /* we'll need to figure out the maildir location ourself. + It's ~/Maildir unless we are chrooted. */ + if (ns->owner != NULL && + mail_user_get_home(ns->owner, &home) > 0) { + path = t_strconcat(home, "/Maildir", NULL); + if (access(path, R_OK|W_OK|X_OK) == 0) { + if (debug) + i_debug("maildir: root exists (%s)", path); + return path; + } + if (debug) + i_debug("maildir: access(%s, rwx): failed: %m", path); + } else { + if (debug) + i_debug("maildir: Home directory not set"); + if (access("/cur", R_OK|W_OK|X_OK) == 0) { + if (debug) + i_debug("maildir: /cur exists, assuming chroot"); + return "/"; + } + } + return NULL; +} + +static bool maildir_storage_autodetect(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + bool debug = ns->mail_set->mail_debug; + struct stat st; + const char *path, *root_dir; + + if (set->root_dir != NULL) + root_dir = set->root_dir; + else { + root_dir = maildir_storage_find_root_dir(ns); + if (root_dir == NULL) { + if (debug) + i_debug("maildir: couldn't find root dir"); + return FALSE; + } + } + + path = t_strconcat(root_dir, "/cur", NULL); + if (stat(path, &st) < 0) { + if (debug) + i_debug("maildir autodetect: stat(%s) failed: %m", path); + return FALSE; + } + + if (!S_ISDIR(st.st_mode)) { + if (debug) + i_debug("maildir autodetect: %s not a directory", path); + return FALSE; + } + + set->root_dir = root_dir; + maildir_storage_get_list_settings(ns, set); + return TRUE; +} + +static int +mkdir_verify(struct mailbox *box, const char *dir, bool verify) +{ + const struct mailbox_permissions *perm; + struct stat st; + + if (verify) { + if (stat(dir, &st) == 0) + return 0; + + if (errno != ENOENT) { + mailbox_set_critical(box, "stat(%s) failed: %m", dir); + return -1; + } + } + + perm = mailbox_get_permissions(box); + if (mkdir_parents_chgrp(dir, perm->dir_create_mode, + perm->file_create_gid, + perm->file_create_gid_origin) == 0) + return 0; + + if (errno == EEXIST) { + if (verify) + return 0; + mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS, + "Mailbox already exists"); + } else if (errno == ENOENT) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + "Mailbox was deleted while it was being created"); + } else if (errno == EACCES) { + if (box->list->ns->type == MAIL_NAMESPACE_TYPE_SHARED) { + /* shared namespace, don't log permission errors */ + mail_storage_set_error(box->storage, MAIL_ERROR_PERM, + MAIL_ERRSTR_NO_PERMISSION); + return -1; + } + mailbox_set_critical(box, "%s", + mail_error_create_eacces_msg("mkdir", dir)); + } else { + mailbox_set_critical(box, "mkdir(%s) failed: %m", dir); + } + return -1; +} + +static int maildir_check_tmp(struct mail_storage *storage, const char *dir) +{ + unsigned int interval = storage->set->mail_temp_scan_interval; + const char *path; + struct stat st; + + /* if tmp/ directory exists, we need to clean it up once in a while */ + path = t_strconcat(dir, "/tmp", NULL); + if (stat(path, &st) < 0) { + if (errno == ENOENT || errno == ENAMETOOLONG) + return 0; + if (errno == EACCES) { + mail_storage_set_critical(storage, "%s", + mail_error_eacces_msg("stat", path)); + return -1; + } + mail_storage_set_critical(storage, "stat(%s) failed: %m", path); + return -1; + } + + if (interval == 0) { + /* disabled */ + } else if (st.st_atime > st.st_ctime + MAILDIR_TMP_DELETE_SECS) { + /* the directory should be empty. we won't do anything + until ctime changes. */ + } else if (st.st_atime < ioloop_time - (time_t)interval) { + /* time to scan */ + (void)unlink_old_files(path, "", + ioloop_time - MAILDIR_TMP_DELETE_SECS); + } + return 1; +} + +/* create or fix maildir, ignore if it already exists */ +static int create_maildir_subdirs(struct mailbox *box, bool verify) +{ + const char *path, *box_path; + unsigned int i; + enum mail_error error; + int ret = 0; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &box_path) < 0) + return -1; + + for (i = 0; i < N_ELEMENTS(maildir_subdirs); i++) { + path = t_strconcat(box_path, "/", maildir_subdirs[i], NULL); + if (mkdir_verify(box, path, verify) < 0) { + error = mailbox_get_last_mail_error(box); + if (error != MAIL_ERROR_EXISTS) + return -1; + /* try to create all of the directories in case one + of them doesn't exist */ + ret = -1; + } + } + return ret; +} + +static void maildir_lock_touch_timeout(struct maildir_mailbox *mbox) +{ + (void)maildir_uidlist_lock_touch(mbox->uidlist); +} + +static struct mailbox * +maildir_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct maildir_mailbox *mbox; + pool_t pool; + + pool = pool_alloconly_create("maildir mailbox", 1024*3); + mbox = p_new(pool, struct maildir_mailbox, 1); + mbox->box = maildir_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &maildir_mail_vfuncs; + mbox->maildir_list_index_ext_id = (uint32_t)-1; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = MAILDIR_STORAGE(storage); + return &mbox->box; +} + +static int maildir_mailbox_open_existing(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + mbox->uidlist = maildir_uidlist_init(mbox); + mbox->keywords = maildir_keywords_init(mbox); + + if ((box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0) { + if (maildir_uidlist_lock(mbox->uidlist) <= 0) { + maildir_mailbox_close(box); + return -1; + } + mbox->keep_lock_to = timeout_add(MAILDIR_LOCK_TOUCH_SECS * 1000, + maildir_lock_touch_timeout, + mbox); + } + + if (index_storage_mailbox_open(box, FALSE) < 0) { + maildir_mailbox_close(box); + return -1; + } + + mbox->maildir_ext_id = + mail_index_ext_register(mbox->box.index, "maildir", + sizeof(mbox->maildir_hdr), 0, 0); + return 0; +} + +static bool maildir_storage_is_readonly(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + if (index_storage_is_readonly(box)) + return TRUE; + + if (maildir_is_backend_readonly(mbox)) { + /* return read-only only if there are no private flags + (that are stored in index files) */ + if (mailbox_get_private_flags_mask(box) == 0) + return TRUE; + } + return FALSE; +} + +static int +maildir_mailbox_exists(struct mailbox *box, bool auto_boxes, + enum mailbox_existence *existence_r) +{ + if (auto_boxes && mailbox_is_autocreated(box)) { + *existence_r = MAILBOX_EXISTENCE_SELECT; + return 0; + } + + return index_storage_mailbox_exists_full(box, "cur", existence_r); +} + +static int maildir_mailbox_open(struct mailbox *box) +{ + const char *box_path = mailbox_get_path(box); + const char *root_dir; + struct stat st; + int ret; + + /* begin by checking if tmp/ directory exists and if it should be + cleaned up. */ + ret = maildir_check_tmp(box->storage, box_path); + if (ret > 0) { + /* exists */ + return maildir_mailbox_open_existing(box); + } + if (ret < 0) + return -1; + + /* tmp/ directory doesn't exist. does the maildir? autocreate missing + dirs only with Maildir++ and imapdir layouts. */ + if (strcmp(box->list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) != 0 && + strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPDIR) != 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } + root_dir = mailbox_list_get_root_forced(box->list, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + if (strcmp(box_path, root_dir) == 0 && !box->inbox_any) { + /* root directory for some namespace. */ + errno = ENOENT; + } else if (stat(box_path, &st) == 0) { + /* yes, we'll need to create the missing dirs */ + if (create_maildir_subdirs(box, TRUE) < 0) + return -1; + + return maildir_mailbox_open_existing(box); + } + + if (errno == ENOENT || errno == ENAMETOOLONG) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } else { + mailbox_set_critical(box, "stat(%s) failed: %m", box_path); + return -1; + } +} + +static int maildir_create_shared(struct mailbox *box) +{ + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *path; + mode_t old_mask; + int fd, ret; + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &path); + if (ret < 0) + return -1; + i_assert(ret > 0); + + old_mask = umask(0); + path = t_strconcat(path, "/dovecot-shared", NULL); + fd = open(path, O_WRONLY | O_CREAT, perm->file_create_mode); + umask(old_mask); + + if (fd == -1) { + mailbox_set_critical(box, "open(%s) failed: %m", path); + return -1; + } + + if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, + "fchown(%s) failed: %m", path); + } + } + i_close_fd(&fd); + return 0; +} + +static int +maildir_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + struct maildir_uidlist *uidlist; + bool locked = FALSE; + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + uidlist = mbox->uidlist; + + if (update->uid_validity != 0 || update->min_next_uid != 0 || + !guid_128_is_empty(update->mailbox_guid)) { + if (maildir_uidlist_lock(uidlist) <= 0) + return -1; + + locked = TRUE; + if (!guid_128_is_empty(update->mailbox_guid)) + maildir_uidlist_set_mailbox_guid(uidlist, update->mailbox_guid); + if (update->uid_validity != 0) + maildir_uidlist_set_uid_validity(uidlist, update->uid_validity); + if (update->min_next_uid != 0) { + maildir_uidlist_set_next_uid(uidlist, update->min_next_uid, + FALSE); + } + ret = maildir_uidlist_update(uidlist); + } + if (ret == 0) + ret = index_storage_mailbox_update(box, update); + if (locked) + maildir_uidlist_unlock(uidlist); + return ret; +} + +static int maildir_create_maildirfolder_file(struct mailbox *box) +{ + const struct mailbox_permissions *perm; + const char *path; + mode_t old_mask; + int fd; + + /* Maildir++ spec wants that maildirfolder named file is created for + all subfolders. Do this only with Maildir++ layout. */ + if (strcmp(box->list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) != 0) + return 0; + perm = mailbox_get_permissions(box); + + path = t_strconcat(mailbox_get_path(box), + "/"MAILDIR_SUBFOLDER_FILENAME, NULL); + old_mask = umask(0); + fd = open(path, O_CREAT | O_WRONLY, perm->file_create_mode); + umask(old_mask); + if (fd != -1) { + /* ok */ + } else if (errno == ENOENT) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + "Mailbox was deleted while it was being created"); + return -1; + } else { + mailbox_set_critical(box, "open(%s, O_CREAT) failed: %m", path); + return -1; + } + + if (perm->file_create_gid != (gid_t)-1) { + if (fchown(fd, (uid_t)-1, perm->file_create_gid) == 0) { + /* ok */ + } else if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, "fchown(%s) failed: %m", path); + } + } + i_close_fd(&fd); + return 0; +} + +static int +maildir_mailbox_create(struct mailbox *box, const struct mailbox_update *update, + bool directory) +{ + const char *root_dir, *shared_path; + /* allow physical location to exist when we rebuild list index, this + happens with LAYOUT=INDEX only. */ + bool verify = box->storage->rebuilding_list_index; + struct stat st; + int ret; + + if ((ret = index_storage_mailbox_create(box, directory)) <= 0) + return ret; + ret = 0; + /* the maildir is created now. finish the creation as best as we can */ + if (create_maildir_subdirs(box, verify) < 0) + ret = -1; + if (maildir_create_maildirfolder_file(box) < 0) + ret = -1; + /* if dovecot-shared exists in the root dir, copy it to newly + created mailboxes */ + root_dir = mailbox_list_get_root_forced(box->list, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + shared_path = t_strconcat(root_dir, "/dovecot-shared", NULL); + if (stat(shared_path, &st) == 0) { + if (maildir_create_shared(box) < 0) + ret = -1; + } + if (update != NULL) { + if (maildir_mailbox_update(box, update) < 0) + ret = -1; + } + return ret; +} + +static int +maildir_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + + if ((items & MAILBOX_METADATA_GUID) != 0) { + if (maildir_uidlist_get_mailbox_guid(mbox->uidlist, + metadata_r->guid) < 0) + return -1; + } + return 0; +} + +static void maildir_mailbox_close(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + + if (mbox->keep_lock_to != NULL) { + maildir_uidlist_unlock(mbox->uidlist); + timeout_remove(&mbox->keep_lock_to); + } + + if (mbox->flags_view != NULL) + mail_index_view_close(&mbox->flags_view); + if (mbox->keywords != NULL) + maildir_keywords_deinit(&mbox->keywords); + if (mbox->uidlist != NULL) + maildir_uidlist_deinit(&mbox->uidlist); + index_storage_mailbox_close(box); +} + +static void maildir_notify_changes(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + const char *box_path = mailbox_get_path(box); + + if (box->notify_callback == NULL) + mailbox_watch_remove_all(&mbox->box); + else { + mailbox_watch_add(&mbox->box, + t_strconcat(box_path, "/new", NULL)); + mailbox_watch_add(&mbox->box, + t_strconcat(box_path, "/cur", NULL)); + } +} + +static bool +maildir_is_internal_name(struct mailbox_list *list ATTR_UNUSED, + const char *name) +{ + return strcmp(name, "cur") == 0 || + strcmp(name, "new") == 0 || + strcmp(name, "tmp") == 0; +} + +static void maildir_storage_add_list(struct mail_storage *storage, + struct mailbox_list *list) +{ + struct maildir_mailbox_list_context *mlist; + + mlist = p_new(list->pool, struct maildir_mailbox_list_context, 1); + mlist->module_ctx.super = list->v; + mlist->set = mail_namespace_get_driver_settings(list->ns, storage); + + list->v.is_internal_name = maildir_is_internal_name; + MODULE_CONTEXT_SET(list, maildir_mailbox_list_module, mlist); +} + +uint32_t maildir_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"MAILDIR_UIDVALIDITY_FNAME, NULL); + return mailbox_uidvalidity_next(list, path); +} + +static enum mail_flags maildir_get_private_flags_mask(struct mailbox *box) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + const char *path, *path2; + struct stat st; + + if (mbox->private_flags_mask_set) + return mbox->_private_flags_mask; + mbox->private_flags_mask_set = TRUE; + + path = mailbox_list_get_root_forced(box->list, MAILBOX_LIST_PATH_TYPE_MAILBOX); + if (box->list->set.index_pvt_dir != NULL) { + /* private index directory is set. we'll definitely have + private flags. */ + mbox->_private_flags_mask = MAIL_SEEN; + } else if (!mailbox_list_get_root_path(box->list, + MAILBOX_LIST_PATH_TYPE_INDEX, + &path2) || + strcmp(path, path2) == 0) { + /* no separate index directory. we can't have private flags, + so don't even bother checking if dovecot-shared exists */ + } else { + path = t_strconcat(mailbox_get_path(box), + "/dovecot-shared", NULL); + if (stat(path, &st) == 0) + mbox->_private_flags_mask = MAIL_SEEN; + } + return mbox->_private_flags_mask; +} + +bool maildir_is_backend_readonly(struct maildir_mailbox *mbox) +{ + if (!mbox->backend_readonly_set) { + const char *box_path = mailbox_get_path(&mbox->box); + + mbox->backend_readonly_set = TRUE; + if (access(t_strconcat(box_path, "/cur", NULL), W_OK) < 0 && + errno == EACCES) + mbox->backend_readonly = TRUE; + } + return mbox->backend_readonly; +} + +struct mail_storage maildir_storage = { + .name = MAILDIR_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + .event_category = &event_category_maildir, + + .v = { + maildir_get_setting_parser_info, + maildir_storage_alloc, + maildir_storage_create, + index_storage_destroy, + maildir_storage_add_list, + maildir_storage_get_list_settings, + maildir_storage_autodetect, + maildir_mailbox_alloc, + NULL, + mail_storage_list_index_rebuild, + } +}; + +struct mailbox maildir_mailbox = { + .v = { + maildir_storage_is_readonly, + index_storage_mailbox_enable, + maildir_mailbox_exists, + maildir_mailbox_open, + maildir_mailbox_close, + index_storage_mailbox_free, + maildir_mailbox_create, + maildir_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + maildir_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + maildir_list_index_has_changed, + maildir_list_index_update_sync, + maildir_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + maildir_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + maildir_get_private_flags_mask, + index_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + maildir_save_alloc, + maildir_save_begin, + maildir_save_continue, + maildir_save_finish, + maildir_save_cancel, + maildir_copy, + maildir_transaction_save_commit_pre, + maildir_transaction_save_commit_post, + maildir_transaction_save_rollback, + index_storage_is_inconsistent + } +}; diff --git a/src/lib-storage/index/maildir/maildir-storage.h b/src/lib-storage/index/maildir/maildir-storage.h new file mode 100644 index 0000000..da92866 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-storage.h @@ -0,0 +1,148 @@ +#ifndef MAILDIR_STORAGE_H +#define MAILDIR_STORAGE_H + +#include "maildir-settings.h" + +#define MAILDIR_STORAGE_NAME "maildir" +#define MAILDIR_SUBSCRIPTION_FILE_NAME "subscriptions" +#define MAILDIR_UIDVALIDITY_FNAME "dovecot-uidvalidity" + +/* "base,S=123:2," means: + <base> [<extra sep> <extra data> [..]] <info sep> 2 <flags sep> */ +#define MAILDIR_INFO_SEP ':' +#define MAILDIR_EXTRA_SEP ',' +#define MAILDIR_FLAGS_SEP ',' + +#define MAILDIR_INFO_SEP_S ":" +#define MAILDIR_EXTRA_SEP_S "," +#define MAILDIR_FLAGS_SEP_S "," + +/* ":2," is the standard flags separator */ +#define MAILDIR_FLAGS_FULL_SEP MAILDIR_INFO_SEP_S "2" MAILDIR_FLAGS_SEP_S + +#define MAILDIR_KEYWORD_FIRST 'a' +#define MAILDIR_KEYWORD_LAST 'z' +#define MAILDIR_MAX_KEYWORDS (MAILDIR_KEYWORD_LAST - MAILDIR_KEYWORD_FIRST + 1) + +/* Maildir++ extension: include file size in the filename to avoid stat() */ +#define MAILDIR_EXTRA_FILE_SIZE 'S' +/* Something (can't remember what anymore) could use 'W' in filename to avoid + calculating file's virtual size (added missing CRs). */ +#define MAILDIR_EXTRA_VIRTUAL_SIZE 'W' + +/* Delete files having ctime older than this from tmp/. 36h is standard. */ +#define MAILDIR_TMP_DELETE_SECS (36*60*60) + +/* How often to touch the uidlist lock file when it's locked. + This is done both when using KEEP_LOCKED flag and when syncing a large + maildir. */ +#define MAILDIR_LOCK_TOUCH_SECS 10 + +/* If an operation fails with ENOENT, we'll check if the mailbox is deleted + or if some directory is just missing. If it's missing, we'll create the + directories and try again this many times before failing. */ +#define MAILDIR_DELETE_RETRY_COUNT 3 + +#include "index-storage.h" + +struct timeval; +struct maildir_save_context; +struct maildir_copy_context; + +struct maildir_index_header { + uint32_t new_check_time, new_mtime, new_mtime_nsecs; + uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs; + uint32_t uidlist_mtime, uidlist_mtime_nsecs, uidlist_size; +}; + +struct maildir_list_index_record { + uint32_t new_mtime, cur_mtime; +}; + +struct maildir_storage { + struct mail_storage storage; + + const struct maildir_settings *set; + const char *temp_prefix; +}; + +struct maildir_mailbox { + struct mailbox box; + struct maildir_storage *storage; + struct mail_index_view *flags_view; + + struct timeout *keep_lock_to; + + /* Filled lazily by mailbox_get_private_flags_mask() */ + enum mail_flags _private_flags_mask; + + /* maildir sync: */ + struct maildir_uidlist *uidlist; + struct maildir_keywords *keywords; + + struct maildir_index_header maildir_hdr; + uint32_t maildir_ext_id; + uint32_t maildir_list_index_ext_id; + + bool synced:1; + bool syncing_commit:1; + bool private_flags_mask_set:1; + bool backend_readonly:1; + bool backend_readonly_set:1; + bool sync_uidlist_refreshed:1; +}; + +#define MAILDIR_STORAGE(s) container_of(s, struct maildir_storage, storage) +#define MAILDIR_MAILBOX(s) container_of(s, struct maildir_mailbox, box) + +extern struct mail_vfuncs maildir_mail_vfuncs; + +/* Return -1 = error, 0 = file not found, 1 = ok */ +typedef int maildir_file_do_func(struct maildir_mailbox *mbox, + const char *path, void *context); + +int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid, + maildir_file_do_func *callback, void *context); +#define maildir_file_do(mbox, seq, callback, context) \ + maildir_file_do(mbox, seq - \ + CALLBACK_TYPECHECK(callback, int (*)( \ + struct maildir_mailbox *, const char *, typeof(context))), \ + (maildir_file_do_func *)callback, context) + +bool maildir_set_deleted(struct mailbox *box); +uint32_t maildir_get_uidvalidity_next(struct mailbox_list *list); +int maildir_lose_unexpected_dir(struct mail_storage *storage, const char *path); +bool maildir_is_backend_readonly(struct maildir_mailbox *mbox); + +struct mail_save_context * +maildir_save_alloc(struct mailbox_transaction_context *_t); +int maildir_save_begin(struct mail_save_context *ctx, struct istream *input); +int maildir_save_continue(struct mail_save_context *ctx); +void maildir_save_finish_keywords(struct mail_save_context *ctx); +int maildir_save_finish(struct mail_save_context *ctx); +void maildir_save_cancel(struct mail_save_context *ctx); + +struct maildir_filename * +maildir_save_add(struct mail_save_context *_ctx, const char *tmp_fname, + struct mail *src_mail) ATTR_NULL(3); +void maildir_save_set_dest_basename(struct mail_save_context *ctx, + struct maildir_filename *mf, + const char *basename); +void maildir_save_set_sizes(struct maildir_filename *mf, + uoff_t size, uoff_t vsize); + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r); +const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, + uint32_t seq); + +int maildir_transaction_save_commit_pre(struct mail_save_context *ctx); +void maildir_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void maildir_transaction_save_rollback(struct mail_save_context *ctx); + +int maildir_copy(struct mail_save_context *ctx, struct mail *mail); +int maildir_transaction_copy_commit(struct maildir_copy_context *ctx); +void maildir_transaction_copy_rollback(struct maildir_copy_context *ctx); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-sync-index.c b/src/lib-storage/index/maildir/maildir-sync-index.c new file mode 100644 index 0000000..da0e40c --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-sync-index.c @@ -0,0 +1,810 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "maildir-storage.h" +#include "index-sync-changes.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename-flags.h" +#include "maildir-sync.h" +#include "mailbox-recent-flags.h" + +#include <stdio.h> +#include <unistd.h> + +struct maildir_index_sync_context { + struct maildir_mailbox *mbox; + struct maildir_sync_context *maildir_sync_ctx; + + struct mail_index_view *view; + struct mail_index_sync_ctx *sync_ctx; + struct maildir_keywords_sync_ctx *keywords_sync_ctx; + struct mail_index_transaction *trans; + + struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; + struct index_sync_changes_context *sync_changes; + enum mail_flags flags; + ARRAY_TYPE(keyword_indexes) keywords, idx_keywords; + + uint32_t uid; + bool update_maildir_hdr_cur; + + time_t start_time; + unsigned int flag_change_count, expunge_count, new_msgs_count; +}; + +struct maildir_keywords_sync_ctx * +maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx) +{ + return ctx->keywords_sync_ctx; +} + +void maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx, + unsigned int count) +{ + ctx->new_msgs_count = count; +} + +static bool +maildir_expunge_is_valid_guid(struct maildir_index_sync_context *ctx, + uint32_t uid, const char *filename, + guid_128_t expunged_guid_128) +{ + guid_128_t guid_128; + const char *guid; + + if (guid_128_is_empty(expunged_guid_128)) { + /* no GUID associated with expunge */ + return TRUE; + } + + T_BEGIN { + guid = maildir_uidlist_lookup_ext(ctx->mbox->uidlist, uid, + MAILDIR_UIDLIST_REC_EXT_GUID); + if (guid == NULL) + guid = t_strcut(filename, *MAILDIR_INFO_SEP_S); + mail_generate_guid_128_hash(guid, guid_128); + } T_END; + + if (memcmp(guid_128, expunged_guid_128, sizeof(guid_128)) == 0) + return TRUE; + + mailbox_set_critical(&ctx->mbox->box, + "Expunged GUID mismatch for UID %u: %s vs %s", + ctx->uid, guid_128_to_string(guid_128), + guid_128_to_string(expunged_guid_128)); + return FALSE; +} + +static int maildir_expunge(struct maildir_mailbox *mbox, const char *path, + struct maildir_index_sync_context *ctx) +{ + struct mailbox *box = &mbox->box; + + ctx->expunge_count++; + + if (unlink(path) == 0) { + mailbox_sync_notify(box, ctx->uid, MAILBOX_SYNC_TYPE_EXPUNGE); + return 1; + } + if (errno == ENOENT) + return 0; + if (UNLINK_EISDIR(errno)) + return maildir_lose_unexpected_dir(box->storage, path); + + mailbox_set_critical(&mbox->box, "unlink(%s) failed: %m", path); + return -1; +} + +static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path, + struct maildir_index_sync_context *ctx) +{ + struct mailbox *box = &mbox->box; + struct stat st; + const char *dir, *fname, *newfname, *newpath; + enum mail_index_sync_type sync_type; + uint8_t flags8; + + ctx->flag_change_count++; + + fname = strrchr(path, '/'); + i_assert(fname != NULL); + fname++; + dir = t_strdup_until(path, fname); + + i_assert(*fname != '\0'); + + /* get the current flags and keywords */ + maildir_filename_flags_get(ctx->keywords_sync_ctx, + fname, &ctx->flags, &ctx->keywords); + + /* apply changes */ + flags8 = ctx->flags; + index_sync_changes_apply(ctx->sync_changes, NULL, + &flags8, &ctx->keywords, &sync_type); + ctx->flags = flags8; + + /* and try renaming with the new name */ + newfname = maildir_filename_flags_kw_set(ctx->keywords_sync_ctx, fname, + ctx->flags, &ctx->keywords); + newpath = t_strconcat(dir, newfname, NULL); + if (strcmp(path, newpath) == 0) { + /* just make sure that the file still exists. avoid rename() + here because it's slow on HFS. */ + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return -1; + } + } else { + if (rename(path, newpath) < 0) { + if (errno == ENOENT) + return 0; + if (!ENOSPACE(errno) && errno != EACCES) { + mailbox_set_critical(box, + "rename(%s, %s) failed: %m", + path, newpath); + } + return -1; + } + } + mailbox_sync_notify(box, ctx->uid, index_sync_type_convert(sync_type)); + return 1; +} + +static int maildir_handle_uid_insertion(struct maildir_index_sync_context *ctx, + enum maildir_uidlist_rec_flag uflags, + const char *filename, uint32_t uid) +{ + int ret; + + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + return 0; + } + + /* most likely a race condition: we read the maildir, then someone else + expunged messages and committed changes to index. so, this message + shouldn't actually exist. */ + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) == 0) { + /* mark it racy and check in next sync */ + ctx->mbox->maildir_hdr.cur_check_time = 0; + maildir_sync_set_racing(ctx->maildir_sync_ctx); + maildir_uidlist_add_flags(ctx->mbox->uidlist, filename, + MAILDIR_UIDLIST_REC_FLAG_RACING); + return 0; + } + + if (ctx->uidlist_sync_ctx == NULL) { + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, + MAILDIR_UIDLIST_SYNC_PARTIAL | + MAILDIR_UIDLIST_SYNC_KEEP_STATE, + &ctx->uidlist_sync_ctx); + if (ret <= 0) + return -1; + } + + uflags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + maildir_uidlist_sync_remove(ctx->uidlist_sync_ctx, filename); + ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx, + filename, uflags); + i_assert(ret > 0); + + /* give the new UID to it immediately */ + maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx); + + i_warning("Maildir %s: Expunged message reappeared, giving a new UID " + "(old uid=%u, file=%s)%s", mailbox_get_path(&ctx->mbox->box), + uid, filename, !str_begins(filename, "msg.") ? "" : + " (Your MDA is saving MH files into Maildir?)"); + return 0; +} + +int maildir_sync_index_begin(struct maildir_mailbox *mbox, + struct maildir_sync_context *maildir_sync_ctx, + struct maildir_index_sync_context **ctx_r) +{ + struct mailbox *_box = &mbox->box; + struct maildir_index_sync_context *ctx; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + enum mail_index_sync_flags sync_flags; + + sync_flags = index_storage_get_sync_flags(&mbox->box); + /* don't drop recent messages if we're saving messages */ + if (maildir_sync_ctx == NULL) + sync_flags &= ENUM_NEGATE(MAIL_INDEX_SYNC_FLAG_DROP_RECENT); + + if (index_storage_expunged_sync_begin(_box, &sync_ctx, &view, + &trans, sync_flags) < 0) + return -1; + + ctx = i_new(struct maildir_index_sync_context, 1); + ctx->mbox = mbox; + ctx->maildir_sync_ctx = maildir_sync_ctx; + ctx->sync_ctx = sync_ctx; + ctx->view = view; + ctx->trans = trans; + ctx->keywords_sync_ctx = + maildir_keywords_sync_init(mbox->keywords, _box->index); + ctx->sync_changes = + index_sync_changes_init(ctx->sync_ctx, ctx->view, ctx->trans, + maildir_is_backend_readonly(mbox)); + ctx->start_time = time(NULL); + + *ctx_r = ctx; + return 0; +} + +static bool +maildir_index_header_has_changed(const struct maildir_index_header *old_hdr, + const struct maildir_index_header *new_hdr) +{ +#define DIR_DELAYED_REFRESH(hdr, name) \ + ((hdr)->name ## _check_time <= \ + (hdr)->name ## _mtime + MAILDIR_SYNC_SECS) + + if (old_hdr->new_mtime != new_hdr->new_mtime || + old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs || + old_hdr->cur_mtime != new_hdr->cur_mtime || + old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs || + old_hdr->uidlist_mtime != new_hdr->uidlist_mtime || + old_hdr->uidlist_mtime_nsecs != new_hdr->uidlist_mtime_nsecs || + old_hdr->uidlist_size != new_hdr->uidlist_size) + return TRUE; + + return DIR_DELAYED_REFRESH(old_hdr, new) != + DIR_DELAYED_REFRESH(new_hdr, new) || + DIR_DELAYED_REFRESH(old_hdr, cur) != + DIR_DELAYED_REFRESH(new_hdr, cur); +} + +static void +maildir_sync_index_update_ext_header(struct maildir_index_sync_context *ctx) +{ + struct maildir_mailbox *mbox = ctx->mbox; + const char *cur_path; + const void *data; + size_t data_size; + struct stat st; + + cur_path = t_strconcat(mailbox_get_path(&mbox->box), "/cur", NULL); + if (ctx->update_maildir_hdr_cur && stat(cur_path, &st) == 0) { + if ((time_t)mbox->maildir_hdr.cur_check_time < st.st_mtime) + mbox->maildir_hdr.cur_check_time = st.st_mtime; + mbox->maildir_hdr.cur_mtime = st.st_mtime; + mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st); + } + + mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id, + &data, &data_size); + if (data_size != sizeof(mbox->maildir_hdr) || + maildir_index_header_has_changed(data, &mbox->maildir_hdr)) { + mail_index_update_header_ext(ctx->trans, mbox->maildir_ext_id, + 0, &mbox->maildir_hdr, + sizeof(mbox->maildir_hdr)); + } +} + +static int maildir_sync_index_finish(struct maildir_index_sync_context *ctx, + bool success) +{ + struct maildir_mailbox *mbox = ctx->mbox; + unsigned int time_diff; + int ret = success ? 0 : -1; + + time_diff = time(NULL) - ctx->start_time; + if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) { + i_warning("Maildir %s: Synchronization took %u seconds " + "(%u new msgs, %u flag change attempts, " + "%u expunge attempts)", + mailbox_get_path(&ctx->mbox->box), time_diff, + ctx->new_msgs_count, ctx->flag_change_count, + ctx->expunge_count); + mail_index_sync_no_warning(ctx->sync_ctx); + } + + if (ret < 0) + mail_index_sync_rollback(&ctx->sync_ctx); + else { + maildir_sync_index_update_ext_header(ctx); + + /* Set syncing_commit=TRUE so that if any sync callbacks try + to access mails which got lost (eg. expunge callback trying + to open the file which was just unlinked) we don't try to + start a second index sync and crash. */ + mbox->syncing_commit = TRUE; + if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } + mbox->syncing_commit = FALSE; + } + + index_storage_expunging_deinit(&mbox->box); + maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx); + index_sync_changes_deinit(&ctx->sync_changes); + i_free(ctx); + return ret; +} + +int maildir_sync_index_commit(struct maildir_index_sync_context **_ctx) +{ + struct maildir_index_sync_context *ctx = *_ctx; + + *_ctx = NULL; + return maildir_sync_index_finish(ctx, TRUE); +} + +void maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx) +{ + struct maildir_index_sync_context *ctx = *_ctx; + + *_ctx = NULL; + (void)maildir_sync_index_finish(ctx, FALSE); +} + +static int uint_cmp(const unsigned int *i1, const unsigned int *i2) +{ + if (*i1 < *i2) + return -1; + else if (*i1 > *i2) + return 1; + else + return 0; +} + +static void +maildir_sync_mail_keywords(struct maildir_index_sync_context *ctx, uint32_t seq) +{ + struct mailbox *box = &ctx->mbox->box; + struct mail_keywords *kw; + unsigned int i, j, old_count, new_count; + const unsigned int *old_indexes, *new_indexes; + bool have_indexonly_keywords; + int diff; + + mail_index_lookup_keywords(ctx->view, seq, &ctx->idx_keywords); + if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords)) { + /* no changes - we should get here usually */ + return; + } + + /* sort the keywords */ + array_sort(&ctx->idx_keywords, uint_cmp); + array_sort(&ctx->keywords, uint_cmp); + + /* drop keywords that are in index-only. we don't want to touch them. */ + old_indexes = array_get(&ctx->idx_keywords, &old_count); + have_indexonly_keywords = FALSE; + for (i = old_count; i > 0; i--) { + if (maildir_keywords_idx_char(ctx->keywords_sync_ctx, + old_indexes[i-1]) == '\0') { + have_indexonly_keywords = TRUE; + array_delete(&ctx->idx_keywords, i-1, 1); + } + } + + if (!have_indexonly_keywords) { + /* no index-only keywords found, so something changed. + just replace them all. */ + kw = mail_index_keywords_create_from_indexes(box->index, + &ctx->keywords); + mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, kw); + mail_index_keywords_unref(&kw); + return; + } + + /* check again if non-index-only keywords changed */ + if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords)) + return; + + /* we can't reset all the keywords or we'd drop indexonly keywords too. + so first remove the unwanted keywords and then add back the wanted + ones. we can get these lists easily by removing common elements + from old and new keywords. */ + new_indexes = array_get(&ctx->keywords, &new_count); + for (i = j = 0; i < old_count && j < new_count; ) { + diff = (int)old_indexes[i] - (int)new_indexes[j]; + if (diff == 0) { + array_delete(&ctx->keywords, j, 1); + array_delete(&ctx->idx_keywords, i, 1); + old_indexes = array_get(&ctx->idx_keywords, &old_count); + new_indexes = array_get(&ctx->keywords, &new_count); + } else if (diff < 0) { + i++; + } else { + j++; + } + } + + if (array_count(&ctx->idx_keywords) > 0) { + kw = mail_index_keywords_create_from_indexes(box->index, + &ctx->idx_keywords); + mail_index_update_keywords(ctx->trans, seq, MODIFY_REMOVE, kw); + mail_index_keywords_unref(&kw); + } + + if (array_count(&ctx->keywords) > 0) { + kw = mail_index_keywords_create_from_indexes(box->index, + &ctx->keywords); + mail_index_update_keywords(ctx->trans, seq, MODIFY_ADD, kw); + mail_index_keywords_unref(&kw); + } +} + +int maildir_sync_index(struct maildir_index_sync_context *ctx, + bool partial) +{ + struct maildir_mailbox *mbox = ctx->mbox; + struct mail_index_view *view = ctx->view; + struct mail_index_view *view2; + struct maildir_uidlist_iter_ctx *iter; + struct mail_index_transaction *trans = ctx->trans; + const struct mail_index_header *hdr; + struct mail_index_header empty_hdr; + const struct mail_index_record *rec; + uint32_t seq, seq2, uid, prev_uid; + enum maildir_uidlist_rec_flag uflags; + const char *filename; + uint32_t uid_validity, next_uid, hdr_next_uid, first_recent_uid; + uint32_t first_uid; + unsigned int changes = 0; + int ret = 0; + time_t time_before_sync; + guid_128_t expunged_guid_128; + enum mail_flags private_flags_mask; + bool expunged, full_rescan = FALSE; + + i_assert(!mbox->syncing_commit); + + first_uid = 1; + hdr = mail_index_get_header(view); + uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); + if (uid_validity != hdr->uid_validity && + uid_validity != 0 && hdr->uid_validity != 0) { + /* uidvalidity changed and index isn't being synced for the + first time, reset the index so we can add all messages as + new */ + i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)", + mailbox_get_path(&ctx->mbox->box), + hdr->uid_validity, uid_validity); + mail_index_reset(trans); + mailbox_recent_flags_reset(&mbox->box); + + first_uid = hdr->messages_count + 1; + i_zero(&empty_hdr); + empty_hdr.next_uid = 1; + hdr = &empty_hdr; + } + hdr_next_uid = hdr->next_uid; + + ctx->mbox->box.tmp_sync_view = view; + private_flags_mask = mailbox_get_private_flags_mask(&mbox->box); + time_before_sync = time(NULL); + mbox->syncing_commit = TRUE; + seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1); + i_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS); + i_array_init(&ctx->idx_keywords, MAILDIR_MAX_KEYWORDS); + iter = maildir_uidlist_iter_init(mbox->uidlist); + while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) { + maildir_filename_flags_get(ctx->keywords_sync_ctx, filename, + &ctx->flags, &ctx->keywords); + + i_assert(uid > prev_uid); + prev_uid = uid; + + /* the private flags are kept only in indexes. don't use them + at all even for newly seen mails */ + ctx->flags &= ENUM_NEGATE(private_flags_mask); + + again: + seq++; + ctx->uid = uid; + + if (seq > hdr->messages_count) { + if (uid < hdr_next_uid) { + if (maildir_handle_uid_insertion(ctx, uflags, + filename, + uid) < 0) + ret = -1; + seq--; + continue; + } + + /* Trust uidlist recent flags only for newly added + messages. When saving/copying messages with flags + they're stored to cur/ and uidlist treats them + as non-recent. */ + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) == 0) { + if (uid >= first_recent_uid) + first_recent_uid = uid + 1; + } + + hdr_next_uid = uid + 1; + mail_index_append(trans, uid, &seq); + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + ctx->flags); + if (array_count(&ctx->keywords) > 0) { + struct mail_keywords *kw; + + kw = mail_index_keywords_create_from_indexes( + mbox->box.index, &ctx->keywords); + mail_index_update_keywords(trans, seq, + MODIFY_REPLACE, kw); + mail_index_keywords_unref(&kw); + } + continue; + } + + rec = mail_index_lookup(view, seq); + if (uid > rec->uid) { + /* already expunged (no point in showing guid in the + expunge record anymore) */ + mail_index_expunge(ctx->trans, seq); + goto again; + } + + if (uid < rec->uid) { + if (maildir_handle_uid_insertion(ctx, uflags, + filename, uid) < 0) + ret = -1; + seq--; + continue; + } + + index_sync_changes_read(ctx->sync_changes, ctx->uid, &expunged, + expunged_guid_128); + if (expunged) { + if (!maildir_expunge_is_valid_guid(ctx, ctx->uid, + filename, + expunged_guid_128)) + continue; + if (maildir_file_do(mbox, ctx->uid, + maildir_expunge, ctx) >= 0) { + /* successful expunge */ + mail_index_expunge(ctx->trans, seq); + } + if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx->maildir_sync_ctx); + continue; + } + + /* the private flags are stored only in indexes, keep them */ + ctx->flags |= rec->flags & private_flags_mask; + + if (index_sync_changes_have(ctx->sync_changes)) { + /* apply flag changes to maildir */ + if (maildir_file_do(mbox, ctx->uid, + maildir_sync_flags, ctx) < 0) + ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx->maildir_sync_ctx); + } + + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) { + /* we last saw this mail in new/, but it's + not there anymore. possibly expunged, + make sure. */ + full_rescan = TRUE; + } + continue; + } + + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* we haven't been able to update maildir with this + record's flag changes. don't sync them. */ + continue; + } + + if (ctx->flags != (rec->flags & MAIL_FLAGS_NONRECENT)) { + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + ctx->flags); + } + + maildir_sync_mail_keywords(ctx, seq); + } + maildir_uidlist_iter_deinit(&iter); + + if (!partial) { + /* expunge the rest */ + for (seq++; seq <= hdr->messages_count; seq++) + mail_index_expunge(ctx->trans, seq); + } + + /* add \Recent flags. use updated view so it contains newly + appended messages. */ + view2 = mail_index_transaction_open_updated_view(trans); + if (mail_index_lookup_seq_range(view2, first_recent_uid, (uint32_t)-1, + &seq, &seq2) && seq2 >= first_uid) { + if (seq < first_uid) { + /* UIDVALIDITY changed, skip over the old messages */ + seq = first_uid; + } + mailbox_recent_flags_set_seqs(&mbox->box, view2, seq, seq2); + } + mail_index_view_close(&view2); + + if (ctx->uidlist_sync_ctx != NULL) { + if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, + TRUE) < 0) + ret = -1; + } + + mailbox_sync_notify(&mbox->box, 0, 0); + ctx->mbox->box.tmp_sync_view = NULL; + + /* check cur/ mtime later. if we came here from saving messages they + could still be moved to cur/ directory. */ + ctx->update_maildir_hdr_cur = TRUE; + mbox->maildir_hdr.cur_check_time = time_before_sync; + + if (uid_validity == 0) { + uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity : + maildir_get_uidvalidity_next(mbox->box.list); + maildir_uidlist_set_uid_validity(mbox->uidlist, uid_validity); + } + maildir_uidlist_set_next_uid(mbox->uidlist, hdr_next_uid, FALSE); + + if (uid_validity != hdr->uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + next_uid = maildir_uidlist_get_next_uid(mbox->uidlist); + if (hdr_next_uid < next_uid) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &next_uid, sizeof(next_uid), FALSE); + } + + i_assert(hdr->first_recent_uid <= first_recent_uid); + if (hdr->first_recent_uid < first_recent_uid) { + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + array_free(&ctx->keywords); + array_free(&ctx->idx_keywords); + mbox->syncing_commit = FALSE; + return ret < 0 ? -1 : (full_rescan ? 0 : 1); +} + +static unsigned int +maildir_list_get_ext_id(struct maildir_mailbox *mbox, + struct mail_index_view *view) +{ + if (mbox->maildir_list_index_ext_id == (uint32_t)-1) { + mbox->maildir_list_index_ext_id = + mail_index_ext_register(mail_index_view_get_index(view), + "maildir", 0, + sizeof(struct maildir_list_index_record), + sizeof(uint32_t)); + } + return mbox->maildir_list_index_ext_id; +} + +int maildir_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, + const char **reason_r) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + const struct maildir_list_index_record *rec; + const void *data; + const char *root_dir, *new_dir, *cur_dir; + struct stat st; + uint32_t ext_id; + bool expunged; + int ret; + + ret = index_storage_list_index_has_changed(box, list_view, seq, + quick, reason_r); + if (ret != 0 || box->storage->set->mailbox_list_index_very_dirty_syncs) + return ret; + if (mbox->storage->set->maildir_very_dirty_syncs) { + /* we don't track cur/new directories with dirty syncs */ + return 0; + } + + ext_id = maildir_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + rec = data; + + if (rec == NULL) { + *reason_r = "Maildir record is missing"; + return 1; + } else if (expunged) { + *reason_r = "Maildir record is expunged"; + return 1; + } else if (rec->new_mtime == 0) { + /* not synced */ + *reason_r = "Maildir record new_mtime=0"; + return 1; + } else if (rec->cur_mtime == 0) { + /* dirty-synced */ + *reason_r = "Maildir record cur_mtime=0"; + return 1; + } + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &root_dir); + if (ret < 0) + return ret; + i_assert(ret > 0); + + /* check if new/ changed */ + new_dir = t_strconcat(root_dir, "/new", NULL); + if (stat(new_dir, &st) < 0) { + mailbox_set_critical(box, "stat(%s) failed: %m", new_dir); + return -1; + } + if ((time_t)rec->new_mtime != st.st_mtime) { + *reason_r = t_strdup_printf( + "Maildir new_mtime changed %u != %"PRIdTIME_T, + rec->new_mtime, st.st_mtime); + return 1; + } + + /* check if cur/ changed */ + cur_dir = t_strconcat(root_dir, "/cur", NULL); + if (stat(cur_dir, &st) < 0) { + mailbox_set_critical(box, "stat(%s) failed: %m", cur_dir); + return -1; + } + if ((time_t)rec->cur_mtime != st.st_mtime) { + *reason_r = t_strdup_printf( + "Maildir cur_mtime changed %u != %"PRIdTIME_T, + rec->cur_mtime, st.st_mtime); + return 1; + } + return 0; +} + +void maildir_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + struct mail_index_view *list_view; + const struct maildir_index_header *mhdr = &mbox->maildir_hdr; + const struct maildir_list_index_record *old_rec; + struct maildir_list_index_record new_rec; + const void *data; + uint32_t ext_id; + bool expunged; + + index_storage_list_index_update_sync(box, trans, seq); + if (mbox->storage->set->maildir_very_dirty_syncs) { + /* we don't track cur/new directories with dirty syncs */ + return; + } + + /* get the current record */ + list_view = mail_index_transaction_get_view(trans); + ext_id = maildir_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + if (expunged) + return; + old_rec = data; + + i_zero(&new_rec); + if (mhdr->new_check_time <= mhdr->new_mtime + MAILDIR_SYNC_SECS || + mhdr->cur_check_time <= mhdr->cur_mtime + MAILDIR_SYNC_SECS) { + /* dirty, we need a refresh next time */ + } else { + new_rec.new_mtime = mhdr->new_mtime; + new_rec.cur_mtime = mhdr->cur_mtime; + } + + if (old_rec == NULL || + memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0) + mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL); +} diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c new file mode 100644 index 0000000..0814415 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-sync.c @@ -0,0 +1,1132 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* + Here's a description of how we handle Maildir synchronization and + it's problems: + + We want to be as efficient as we can. The most efficient way to + check if changes have occurred is to stat() the new/ and cur/ + directories and uidlist file - if their mtimes haven't changed, + there's no changes and we don't need to do anything. + + Problem 1: Multiple changes can happen within a single second - + nothing guarantees that once we synced it, someone else didn't just + then make a modification. Such modifications wouldn't get noticed + until a new modification occurred later. + + Problem 2: Syncing cur/ directory is much more costly than syncing + new/. Moving mails from new/ to cur/ will always change mtime of + cur/ causing us to sync it as well. + + Problem 3: We may not be able to move mail from new/ to cur/ + because we're out of quota, or simply because we're accessing a + read-only mailbox. + + + MAILDIR_SYNC_SECS + ----------------- + + Several checks below use MAILDIR_SYNC_SECS, which should be maximum + clock drift between all computers accessing the maildir (eg. via + NFS), rounded up to next second. Our default is 1 second, since + everyone should be using NTP. + + Note that setting it to 0 works only if there's only one computer + accessing the maildir. It's practically impossible to make two + clocks _exactly_ synchronized. + + It might be possible to only use file server's clock by looking at + the atime field, but I don't know how well that would actually work. + + cur directory + ------------- + + We have dirty_cur_time variable which is set to cur/ directory's + mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have + synchronized the directory. + + When dirty_cur_time is non-zero, we don't synchronize the cur/ + directory until + + a) cur/'s mtime changes + b) opening a mail fails with ENOENT + c) time() > dirty_cur_time + MAILDIR_SYNC_SECS + + This allows us to modify the maildir multiple times without having + to sync it at every change. The sync will eventually be done to + make sure we didn't miss any external changes. + + The dirty_cur_time is set when: + + - we change message flags + - we expunge messages + - we move mail from new/ to cur/ + - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS + + It's unset when we do the final syncing, ie. when mtime is + older than time() - MAILDIR_SYNC_SECS. + + new directory + ------------- + + If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize + it. dirty_cur_time-like feature might save us a few syncs, but + that might break a client which saves a mail in one connection and + tries to fetch it in another one. new/ directory is almost always + empty, so syncing it should be very fast anyway. Actually this can + still happen if we sync only new/ dir while another client is also + moving mails from it to cur/ - it takes us a while to see them. + That's pretty unlikely to happen however, and only way to fix it + would be to always synchronize cur/ after new/. + + Normally we move all mails from new/ to cur/ whenever we sync it. If + it's not possible for some reason, we mark the mail with "probably + exists in new/ directory" flag. + + If rename() still fails because of ENOSPC or EDQUOT, we still save + the flag changes in index with dirty-flag on. When moving the mail + to cur/ directory, or when we notice it's already moved there, we + apply the flag changes to the filename, rename it and remove the + dirty flag. If there's dirty flags, this should be tried every time + after expunge or when closing the mailbox. + + uidlist + ------- + + This file contains UID <-> filename mappings. It's updated only when + new mail arrives, so it may contain filenames that have already been + deleted. Updating is done by getting uidlist.lock file, writing the + whole uidlist into it and rename()ing it over the old uidlist. This + means there's no need to lock the file for reading. + + Whenever uidlist is rewritten, it's mtime must be larger than the old + one's. Use utime() before rename() if needed. Note that inode checking + wouldn't have been sufficient as inode numbers can be reused. + + This file is usually read the first time you need to know filename for + given UID. After that it's not re-read unless new mails come that we + don't know about. + + broken clients + -------------- + + Originally the middle identifier in Maildir filename was specified + only as <process id>_<delivery counter>. That however created a + problem with randomized PIDs which made it possible that the same + PID was reused within one second. + + So if within one second a mail was delivered, MUA moved it to cur/ + and another mail was delivered by a new process using same PID as + the first one, we likely ended up overwriting the first mail when + the second mail was moved over it. + + Nowadays everyone should be giving a bit more specific identifier, + for example include microseconds in it which Dovecot does. + + There's a simple way to prevent this from happening in some cases: + Don't move the mail from new/ to cur/ if it's mtime is >= time() - + MAILDIR_SYNC_SECS. The second delivery's link() call then fails + because the file is already in new/, and it will then use a + different filename. There's a few problems with this however: + + - it requires extra stat() call which is unneeded extra I/O + - another MUA might still move the mail to cur/ + - if first file's flags are modified by either Dovecot or another + MUA, it's moved to cur/ (you _could_ just do the dirty-flagging + but that'd be ugly) + + Because this is useful only for very few people and it requires + extra I/O, I decided not to implement this. It should be however + quite easy to do since we need to be able to deal with files in new/ + in any case. + + It's also possible to never accidentally overwrite a mail by using + link() + unlink() rather than rename(). This however isn't very + good idea as it introduces potential race conditions when multiple + clients are accessing the mailbox: + + Trying to move the same mail from new/ to cur/ at the same time: + + a) Client 1 uses slightly different filename than client 2, + for example one sets read-flag on but the other doesn't. + You have the same mail duplicated now. + + b) Client 3 sees the mail between Client 1's and 2's link() calls + and changes it's flag. You have the same mail duplicated now. + + And it gets worse when they're unlink()ing in cur/ directory: + + c) Client 1 changes mails's flag and client 2 changes it back + between 1's link() and unlink(). The mail is now expunged. + + d) If you try to deal with the duplicates by unlink()ing another + one of them, you might end up unlinking both of them. + + So, what should we do then if we notice a duplicate? First of all, + it might not be a duplicate at all, readdir() might have just + returned it twice because it was just renamed. What we should do is + create a completely new base name for it and rename() it to that. + If the call fails with ENOENT, it only means that it wasn't a + duplicate after all. +*/ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "hash.h" +#include "str.h" +#include "eacces-error.h" +#include "nfs-workarounds.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-filename.h" +#include "maildir-sync.h" + +#include <stdio.h> +#include <stddef.h> +#include <unistd.h> +#include <dirent.h> +#include <sys/stat.h> + +#define MAILDIR_FILENAME_FLAG_FOUND 128 + +/* When rename()ing many files from new/ to cur/, it's possible that next + readdir() skips some files. we don't of course wish to lose them, so we + go and rescan the new/ directory again from beginning until no files are + left. This value is just an optimization to avoid checking the directory + twice needlessly. usually only NFS is the problem case. 1 is the safest + bet here, but I guess 5 will do just fine too. */ +#define MAILDIR_RENAME_RESCAN_COUNT 5 + +/* This is mostly to avoid infinite looping when rename() destination already + exists as the hard link of the file itself. */ +#define MAILDIR_SCAN_DIR_MAX_COUNT 5 + +#define DUPE_LINKS_DELETE_SECS 30 + +enum maildir_scan_why { + WHY_FORCED = 0x01, + WHY_FIRSTSYNC = 0x02, + WHY_NEWCHANGED = 0x04, + WHY_CURCHANGED = 0x08, + WHY_DROPRECENT = 0x10, + WHY_FINDRECENT = 0x20, + WHY_DELAYEDNEW = 0x40, + WHY_DELAYEDCUR = 0x80 +}; + +struct maildir_sync_context { + struct maildir_mailbox *mbox; + const char *new_dir, *cur_dir; + + enum mailbox_sync_flags flags; + time_t last_touch, last_notify; + + struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; + struct maildir_index_sync_context *index_sync_ctx; + + bool partial:1; + bool locked:1; + bool racing:1; +}; + +void maildir_sync_set_racing(struct maildir_sync_context *ctx) +{ + ctx->racing = TRUE; +} + +void maildir_sync_notify(struct maildir_sync_context *ctx) +{ + time_t now; + + if (ctx == NULL) { + /* we got here from maildir-save.c. it has no + maildir_sync_context, */ + return; + } + + now = time(NULL); + if (now - ctx->last_touch > MAILDIR_LOCK_TOUCH_SECS && ctx->locked) { + (void)maildir_uidlist_lock_touch(ctx->mbox->uidlist); + ctx->last_touch = now; + } + if (now - ctx->last_notify > MAIL_STORAGE_STAYALIVE_SECS) { + struct mailbox *box = &ctx->mbox->box; + + if (box->storage->callbacks.notify_ok != NULL) { + box->storage->callbacks. + notify_ok(box, "Hang in there..", + box->storage->callback_context); + } + ctx->last_notify = now; + } +} + +static struct maildir_sync_context * +maildir_sync_context_new(struct maildir_mailbox *mbox, + enum mailbox_sync_flags flags) +{ + struct maildir_sync_context *ctx; + + ctx = t_new(struct maildir_sync_context, 1); + ctx->mbox = mbox; + ctx->new_dir = t_strconcat(mailbox_get_path(&mbox->box), "/new", NULL); + ctx->cur_dir = t_strconcat(mailbox_get_path(&mbox->box), "/cur", NULL); + ctx->last_touch = ioloop_time; + ctx->last_notify = ioloop_time; + ctx->flags = flags; + return ctx; +} + +static void maildir_sync_deinit(struct maildir_sync_context *ctx) +{ + if (ctx->uidlist_sync_ctx != NULL) + (void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE); + if (ctx->index_sync_ctx != NULL) + maildir_sync_index_rollback(&ctx->index_sync_ctx); + if (ctx->mbox->storage->storage.rebuild_list_index) + (void)mail_storage_list_index_rebuild_and_set_uncorrupted(&ctx->mbox->storage->storage); +} + +static int maildir_fix_duplicate(struct maildir_sync_context *ctx, + const char *dir, const char *fname2) +{ + const char *fname1, *path1, *path2; + const char *new_fname, *new_path; + struct stat st1, st2; + uoff_t size; + + fname1 = maildir_uidlist_sync_get_full_filename(ctx->uidlist_sync_ctx, + fname2); + i_assert(fname1 != NULL); + + path1 = t_strconcat(dir, "/", fname1, NULL); + path2 = t_strconcat(dir, "/", fname2, NULL); + + if (stat(path1, &st1) < 0 || stat(path2, &st2) < 0) { + /* most likely the files just don't exist anymore. + don't really care about other errors much. */ + return 0; + } + if (st1.st_ino == st2.st_ino && + CMP_DEV_T(st1.st_dev, st2.st_dev)) { + /* Files are the same. this means either a race condition + between stat() calls, or that the files were link()ed. */ + if (st1.st_nlink > 1 && st2.st_nlink == st1.st_nlink && + st1.st_ctime == st2.st_ctime && + st1.st_ctime < ioloop_time - DUPE_LINKS_DELETE_SECS) { + /* The file has hard links and it hasn't had any + changes (such as renames) for a while, so this + isn't a race condition. + + rename()ing one file on top of the other would fix + this safely, except POSIX decided that rename() + doesn't work that way. So we'll have unlink() one + and hope that another process didn't just decide to + unlink() the other (uidlist lock prevents this from + happening) */ + if (i_unlink(path2) == 0) + i_warning("Unlinked a duplicate: %s", path2); + } + return 0; + } + + new_fname = maildir_filename_generate(); + /* preserve S= and W= sizes if they're available. + (S=size is required for zlib plugin to work) */ + if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_FILE_SIZE, &size)) { + new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T, + new_fname, MAILDIR_EXTRA_FILE_SIZE, size); + } + if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) { + new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T, + new_fname, MAILDIR_EXTRA_VIRTUAL_SIZE, size); + } + new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box), + "/new/", new_fname, NULL); + + if (rename(path2, new_path) == 0) + i_warning("Fixed a duplicate: %s -> %s", path2, new_fname); + else if (errno != ENOENT) { + mailbox_set_critical(&ctx->mbox->box, + "Couldn't fix a duplicate: rename(%s, %s) failed: %m", + path2, new_path); + return -1; + } + return 0; +} + +static int +maildir_rename_empty_basename(struct maildir_sync_context *ctx, + const char *dir, const char *fname) +{ + const char *old_path, *new_fname, *new_path; + + old_path = t_strconcat(dir, "/", fname, NULL); + new_fname = maildir_filename_generate(); + new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box), + "/new/", new_fname, NULL); + if (rename(old_path, new_path) == 0) + i_warning("Fixed broken filename: %s -> %s", old_path, new_fname); + else if (errno != ENOENT) { + mailbox_set_critical(&ctx->mbox->box, + "Couldn't fix a broken filename: rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + return 0; +} + +static int +maildir_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st_r) +{ + struct mailbox *box = &mbox->box; + int i; + + for (i = 0;; i++) { + if (nfs_safe_stat(path, st_r) == 0) + return 0; + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) + break; + + if (!maildir_set_deleted(box)) + return -1; + /* try again */ + } + + mailbox_set_critical(box, "stat(%s) failed: %m", path); + return -1; +} + +static int +maildir_scan_dir(struct maildir_sync_context *ctx, bool new_dir, bool final, + enum maildir_scan_why why) +{ + const char *path; + DIR *dirp; + string_t *src, *dest; + struct dirent *dp; + struct stat st; + enum maildir_uidlist_rec_flag flags; + unsigned int time_diff, i, readdir_count = 0, move_count = 0; + time_t start_time; + int ret = 1; + bool move_new, dir_changed = FALSE; + + path = new_dir ? ctx->new_dir : ctx->cur_dir; + for (i = 0;; i++) { + dirp = opendir(path); + if (dirp != NULL) + break; + + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + if (errno == EACCES) { + mailbox_set_critical(&ctx->mbox->box, "%s", + eacces_error_get("opendir", path)); + } else { + mailbox_set_critical(&ctx->mbox->box, + "opendir(%s) failed: %m", path); + } + return -1; + } + + if (!maildir_set_deleted(&ctx->mbox->box)) + return -1; + /* try again */ + } + +#ifdef HAVE_DIRFD + if (fstat(dirfd(dirp), &st) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "fstat(%s) failed: %m", path); + (void)closedir(dirp); + return -1; + } +#else + if (maildir_stat(ctx->mbox, path, &st) < 0) { + (void)closedir(dirp); + return -1; + } +#endif + + start_time = time(NULL); + if (new_dir) { + ctx->mbox->maildir_hdr.new_check_time = start_time; + ctx->mbox->maildir_hdr.new_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.new_mtime_nsecs = ST_MTIME_NSEC(st); + } else { + ctx->mbox->maildir_hdr.cur_check_time = start_time; + ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st); + } + + src = t_str_new(1024); + dest = t_str_new(1024); + + move_new = new_dir && ctx->locked && + ((ctx->mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0 || + ctx->mbox->storage->set->maildir_empty_new); + + errno = 0; + for (; (dp = readdir(dirp)) != NULL; errno = 0) { + if (dp->d_name[0] == '.') + continue; + + if (dp->d_name[0] == MAILDIR_INFO_SEP) { + /* don't even try to use file with empty base name */ + if (maildir_rename_empty_basename(ctx, path, + dp->d_name) < 0) + break; + continue; + } + + flags = 0; + if (move_new) { + i_assert(dp->d_name[0] != '\0'); + + str_truncate(src, 0); + str_truncate(dest, 0); + str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name); + str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name); + if (strchr(dp->d_name, MAILDIR_INFO_SEP) == NULL) { + str_append(dest, MAILDIR_FLAGS_FULL_SEP); + } + if (rename(str_c(src), str_c(dest)) == 0) { + /* we moved it - it's \Recent for us */ + dir_changed = TRUE; + move_count++; + flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED | + MAILDIR_UIDLIST_REC_FLAG_RECENT; + } else if (ENOTFOUND(errno)) { + /* someone else moved it already */ + dir_changed = TRUE; + move_count++; + flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED | + MAILDIR_UIDLIST_REC_FLAG_RECENT; + } else if (ENOSPACE(errno) || errno == EACCES) { + /* not enough disk space / read-only maildir, + leave here */ + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + move_new = FALSE; + } else { + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + mailbox_set_critical(&ctx->mbox->box, + "rename(%s, %s) failed: %m", + str_c(src), str_c(dest)); + } + if ((move_count % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx); + } else if (new_dir) { + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | + MAILDIR_UIDLIST_REC_FLAG_RECENT; + } + + readdir_count++; + if ((readdir_count % MAILDIR_SLOW_CHECK_COUNT) == 0) + maildir_sync_notify(ctx); + + ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx, + dp->d_name, flags); + if (ret <= 0) { + if (ret < 0) + break; + + /* possibly duplicate - try fixing it */ + T_BEGIN { + ret = maildir_fix_duplicate(ctx, path, + dp->d_name); + } T_END; + if (ret < 0) + break; + } + } + +#ifdef __APPLE__ + if (errno == EINVAL && move_count > 0 && !final) { + /* OS X HFS+: readdir() fails sometimes when rename() + have been done. */ + move_count = MAILDIR_RENAME_RESCAN_COUNT + 1; + } else +#endif + + if (errno != 0) { + mailbox_set_critical(&ctx->mbox->box, + "readdir(%s) failed: %m", path); + ret = -1; + } + + if (closedir(dirp) < 0) { + mailbox_set_critical(&ctx->mbox->box, + "closedir(%s) failed: %m", path); + ret = -1; + } + + if (dir_changed) { + /* save the exact new times. the new mtimes should be >= + "start_time", but just in case something weird happens and + mtime doesn't update, use "start_time". */ + if (stat(ctx->new_dir, &st) == 0) { + ctx->mbox->maildir_hdr.new_check_time = + I_MAX(st.st_mtime, start_time); + ctx->mbox->maildir_hdr.new_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.new_mtime_nsecs = + ST_MTIME_NSEC(st); + } + if (stat(ctx->cur_dir, &st) == 0) { + ctx->mbox->maildir_hdr.new_check_time = + I_MAX(st.st_mtime, start_time); + ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.cur_mtime_nsecs = + ST_MTIME_NSEC(st); + } + } + time_diff = time(NULL) - start_time; + if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) { + i_warning("Maildir: Scanning %s took %u seconds " + "(%u readdir()s, %u rename()s to cur/, why=0x%x)", + path, time_diff, readdir_count, move_count, why); + } + + return ret < 0 ? -1 : + (move_count <= MAILDIR_RENAME_RESCAN_COUNT || final ? 0 : 1); +} + +static void maildir_sync_get_header(struct maildir_mailbox *mbox) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id, + &data, &data_size); + if (data_size == 0) { + /* header doesn't exist */ + } else { + memcpy(&mbox->maildir_hdr, data, + I_MIN(sizeof(mbox->maildir_hdr), data_size)); + } +} + +int maildir_sync_header_refresh(struct maildir_mailbox *mbox) +{ + if (mail_index_refresh(mbox->box.index) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + maildir_sync_get_header(mbox); + return 0; +} + +static int maildir_sync_quick_check(struct maildir_mailbox *mbox, bool undirty, + const char *new_dir, const char *cur_dir, + bool *new_changed_r, bool *cur_changed_r, + enum maildir_scan_why *why_r) +{ +#define DIR_DELAYED_REFRESH(hdr, name) \ + ((hdr)->name ## _check_time <= \ + (hdr)->name ## _mtime + MAILDIR_SYNC_SECS && \ + (undirty || \ + (time_t)(hdr)->name ## _check_time < ioloop_time - MAILDIR_SYNC_SECS)) + +#define DIR_MTIME_CHANGED(st, hdr, name) \ + ((st).st_mtime != (time_t)(hdr)->name ## _mtime || \ + !ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), (hdr)->name ## _mtime_nsecs)) + + struct maildir_index_header *hdr = &mbox->maildir_hdr; + struct stat new_st, cur_st; + bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE; + + *why_r = 0; + + if (mbox->maildir_hdr.new_mtime == 0) { + maildir_sync_get_header(mbox); + if (mbox->maildir_hdr.new_mtime == 0) { + /* first sync */ + *why_r |= WHY_FIRSTSYNC; + *new_changed_r = *cur_changed_r = TRUE; + return 0; + } + } + + *new_changed_r = *cur_changed_r = FALSE; + + /* try to avoid stat()ing by first checking delayed changes */ + if (DIR_DELAYED_REFRESH(hdr, new) || + (DIR_DELAYED_REFRESH(hdr, cur) && + !mbox->storage->set->maildir_very_dirty_syncs)) { + /* refresh index and try again */ + if (maildir_sync_header_refresh(mbox) < 0) + return -1; + refreshed = TRUE; + + if (DIR_DELAYED_REFRESH(hdr, new)) { + *why_r |= WHY_DELAYEDNEW; + *new_changed_r = TRUE; + } + if (DIR_DELAYED_REFRESH(hdr, cur) && + !mbox->storage->set->maildir_very_dirty_syncs) { + *why_r |= WHY_DELAYEDCUR; + *cur_changed_r = TRUE; + } + if (*new_changed_r && *cur_changed_r) + return 0; + } + + if (!*new_changed_r) { + if (maildir_stat(mbox, new_dir, &new_st) < 0) + return -1; + check_new = TRUE; + } + if (!*cur_changed_r) { + if (maildir_stat(mbox, cur_dir, &cur_st) < 0) + return -1; + check_cur = TRUE; + } + + for (;;) { + if (check_new) { + *new_changed_r = DIR_MTIME_CHANGED(new_st, hdr, new); + if (*new_changed_r) + *why_r |= WHY_NEWCHANGED; + } + if (check_cur) { + *cur_changed_r = DIR_MTIME_CHANGED(cur_st, hdr, cur); + if (*cur_changed_r) + *why_r |= WHY_CURCHANGED; + } + + if ((!*new_changed_r && !*cur_changed_r) || refreshed) + break; + + /* refresh index and try again */ + if (maildir_sync_header_refresh(mbox) < 0) + return -1; + refreshed = TRUE; + } + + return 0; +} + +static void maildir_sync_update_next_uid(struct maildir_mailbox *mbox) +{ + const struct mail_index_header *hdr; + uint32_t uid_validity; + + hdr = mail_index_get_header(mbox->box.view); + if (hdr->uid_validity == 0) + return; + + uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); + if (uid_validity == hdr->uid_validity || uid_validity == 0) { + /* make sure uidlist's next_uid is at least as large as + index file's. typically this happens only if uidlist gets + deleted. */ + maildir_uidlist_set_uid_validity(mbox->uidlist, + hdr->uid_validity); + maildir_uidlist_set_next_uid(mbox->uidlist, + hdr->next_uid, FALSE); + } +} + +static bool +have_recent_messages(struct maildir_sync_context *ctx, bool seen_changes) +{ + const struct mail_index_header *hdr; + uint32_t next_uid; + + hdr = mail_index_get_header(ctx->mbox->box.view); + if (!seen_changes) { + /* index is up to date. get the next-uid from it */ + next_uid = hdr->next_uid; + } else { + (void)maildir_uidlist_refresh(ctx->mbox->uidlist); + next_uid = maildir_uidlist_get_next_uid(ctx->mbox->uidlist); + } + return hdr->first_recent_uid < next_uid; +} + +static int maildir_sync_get_changes(struct maildir_sync_context *ctx, + bool *new_changed_r, bool *cur_changed_r, + enum maildir_scan_why *why_r) +{ + struct maildir_mailbox *mbox = ctx->mbox; + enum mail_index_sync_flags flags = 0; + bool undirty = (ctx->flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0; + + *why_r = 0; + + if (maildir_sync_quick_check(mbox, undirty, ctx->new_dir, ctx->cur_dir, + new_changed_r, cur_changed_r, why_r) < 0) + return -1; + + /* if there are files in new/, we'll need to move them. we'll check + this by seeing if we have any recent messages */ + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) { + if (!*new_changed_r && have_recent_messages(ctx, FALSE)) { + *new_changed_r = TRUE; + *why_r |= WHY_DROPRECENT; + } + } else if (*new_changed_r) { + /* if recent messages have been externally deleted from new/, + we need to get them out of index. this requires that + we make sure they weren't just moved to cur/. */ + if (!*cur_changed_r && have_recent_messages(ctx, TRUE)) { + *cur_changed_r = TRUE; + *why_r |= WHY_FINDRECENT; + } + } + + if (*new_changed_r || *cur_changed_r) + return 1; + + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) + flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; + + if (mbox->synced) { + /* refresh index only after the first sync, i.e. avoid wasting + time on refreshing it immediately after it was just opened */ + mail_index_refresh(mbox->box.index); + } + return mail_index_sync_have_any(mbox->box.index, flags) ? 1 : 0; +} + +static int ATTR_NULL(3) +maildir_sync_context(struct maildir_sync_context *ctx, bool forced, + uint32_t *find_uid, bool *lost_files_r) +{ + enum maildir_uidlist_sync_flags sync_flags; + enum maildir_uidlist_rec_flag flags; + bool new_changed, cur_changed, lock_failure; + const char *fname; + enum maildir_scan_why why; + int ret; + + *lost_files_r = FALSE; + + if (forced) { + new_changed = cur_changed = TRUE; + why = WHY_FORCED; + } else { + ret = maildir_sync_get_changes(ctx, &new_changed, &cur_changed, + &why); + if (ret <= 0) + return ret; + } + + /* + Locking, locking, locking.. Wasn't maildir supposed to be lockless? + + We can get here either as beginning a real maildir sync, or when + committing changes to maildir but a file was lost (maybe renamed). + + So, we're going to need two locks. One for index and one for + uidlist. To avoid deadlocking do the uidlist lock always first. + + uidlist is needed only for figuring out UIDs for newly seen files, + so theoretically we wouldn't need to lock it unless there are new + files. It has a few problems though, assuming the index lock didn't + already protect it (eg. in-memory indexes): + + 1. Just because you see a new file which doesn't exist in uidlist + file, doesn't mean that the file really exists anymore, or that + your readdir() lists all new files. Meaning that this is possible: + + A: opendir(), readdir() -> new file ... + -- new files are written to the maildir -- + B: opendir(), readdir() -> new file, lock uidlist, + readdir() -> another new file, rewrite uidlist, unlock + A: ... lock uidlist, readdir() -> nothing left, rewrite uidlist, + unlock + + The second time running A didn't see the two new files. To handle + this correctly, it must not remove the new unseen files from + uidlist. This is possible to do, but adds extra complexity. + + 2. If another process is rename()ing files while we are + readdir()ing, it's possible that readdir() never lists some files, + causing Dovecot to assume they were expunged. In next sync they + would show up again, but client could have already been notified of + that and they would show up under new UIDs, so the damage is + already done. + + Both of the problems can be avoided if we simply lock the uidlist + before syncing and keep it until sync is finished. Typically this + would happen in any case, as there is the index lock.. + + The second case is still a problem with external changes though, + because maildir doesn't require any kind of locking. Luckily this + problem rarely happens except under high amount of modifications. + */ + + if (!cur_changed) { + ctx->partial = TRUE; + sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL; + } else { + ctx->partial = FALSE; + sync_flags = 0; + if (forced) + sync_flags |= MAILDIR_UIDLIST_SYNC_FORCE; + if ((ctx->flags & MAILBOX_SYNC_FLAG_FAST) != 0) + sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK; + } + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags, + &ctx->uidlist_sync_ctx); + lock_failure = ret <= 0; + if (ret <= 0) { + struct mail_storage *storage = ctx->mbox->box.storage; + + if (ret == 0) { + /* timeout */ + return 0; + } + /* locking failed. sync anyway without locking so that it's + possible to expunge messages when out of quota. */ + if (forced) { + /* we're already forcing a sync, we're trying to find + a message that was probably already expunged, don't + loop for a long time trying to find it. */ + return -1; + } + ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags | + MAILDIR_UIDLIST_SYNC_NOLOCK, + &ctx->uidlist_sync_ctx); + if (ret <= 0) { + i_assert(ret != 0); + return -1; + } + + if (storage->callbacks.notify_no != NULL) { + storage->callbacks.notify_no(&ctx->mbox->box, + "Internal mailbox synchronization failure, " + "showing only old mails.", + storage->callback_context); + } + } + ctx->locked = maildir_uidlist_is_locked(ctx->mbox->uidlist); + if (!ctx->locked) + ctx->partial = TRUE; + + if (!ctx->mbox->syncing_commit && (ctx->locked || lock_failure)) { + if (maildir_sync_index_begin(ctx->mbox, ctx, + &ctx->index_sync_ctx) < 0) + return -1; + } + + if (new_changed || cur_changed) { + /* if we're going to check cur/ dir our current logic requires + that new/ dir is checked as well. it's a good idea anyway. */ + unsigned int count = 0; + bool final = FALSE; + + while ((ret = maildir_scan_dir(ctx, TRUE, final, why)) > 0) { + /* rename()d at least some files, which might have + caused some other files to be missed. check again + (see MAILDIR_RENAME_RESCAN_COUNT). */ + if (++count >= MAILDIR_SCAN_DIR_MAX_COUNT) + final = TRUE; + } + if (ret < 0) + return -1; + + if (cur_changed) { + if (maildir_scan_dir(ctx, FALSE, TRUE, why) < 0) + return -1; + } + + maildir_sync_update_next_uid(ctx->mbox); + + /* finish uidlist syncing, but keep it still locked */ + maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx); + } + + if (!ctx->locked) { + /* make sure we sync the maildir later */ + ctx->mbox->maildir_hdr.new_mtime = 0; + ctx->mbox->maildir_hdr.cur_mtime = 0; + } + + if (ctx->index_sync_ctx != NULL) { + /* NOTE: index syncing here might cause a re-sync due to + files getting lost, so this function might be called + reentrantly. */ + ret = maildir_sync_index(ctx->index_sync_ctx, ctx->partial); + if (ret < 0) + maildir_sync_index_rollback(&ctx->index_sync_ctx); + else if (maildir_sync_index_commit(&ctx->index_sync_ctx) < 0) + return -1; + + if (ret < 0) + return -1; + if (ret == 0) + *lost_files_r = TRUE; + + i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist) || + lock_failure); + } + + if (find_uid != NULL && *find_uid != 0) { + ret = maildir_uidlist_lookup(ctx->mbox->uidlist, + *find_uid, &flags, &fname); + if (ret < 0) + return -1; + if (ret == 0) { + /* UID is expunged */ + *find_uid = 0; + } else if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) { + *find_uid = 0; + } else { + /* we didn't find it, possibly expunged? */ + } + } + + return maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, TRUE); +} + +int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r) +{ + int ret; + + ret = maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r); + if (ret != 0) + return ret; + + if (maildir_uidlist_is_open(mbox->uidlist)) { + /* refresh uidlist and check again in case it was added + after the last mailbox sync */ + if (mbox->sync_uidlist_refreshed) { + /* we've already refreshed it, don't bother again */ + return ret; + } + mbox->sync_uidlist_refreshed = TRUE; + if (maildir_uidlist_refresh(mbox->uidlist) < 0) + return -1; + } else { + /* the uidlist doesn't exist. */ + if (maildir_storage_sync_force(mbox, uid) < 0) + return -1; + } + + /* try again */ + return maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r); +} + +static int maildir_sync_run(struct maildir_mailbox *mbox, + enum mailbox_sync_flags flags, bool force_resync, + uint32_t *uid, bool *lost_files_r) +{ + struct maildir_sync_context *ctx; + bool retry, lost_files; + int ret; + + T_BEGIN { + ctx = maildir_sync_context_new(mbox, flags); + ret = maildir_sync_context(ctx, force_resync, uid, lost_files_r); + retry = ctx->racing; + maildir_sync_deinit(ctx); + } T_END; + + if (retry) T_BEGIN { + /* we're racing some file. retry the sync again to see if the + file is really gone or not. if it is, this is a bit of + unnecessary work, but if it's not, this is necessary for + e.g. doveadm force-resync to work. */ + ctx = maildir_sync_context_new(mbox, 0); + ret = maildir_sync_context(ctx, TRUE, NULL, &lost_files); + maildir_sync_deinit(ctx); + } T_END; + return ret; +} + +int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid) +{ + bool lost_files; + int ret; + + ret = maildir_sync_run(mbox, MAILBOX_SYNC_FLAG_FAST, + TRUE, &uid, &lost_files); + if (uid != 0) { + /* maybe it's expunged. check again. */ + ret = maildir_sync_run(mbox, 0, TRUE, NULL, &lost_files); + } + return ret; +} + +int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox) +{ + struct mail_index_view_sync_ctx *sync_ctx; + bool delayed_expunges; + + mail_index_refresh(mbox->box.index); + if (mbox->flags_view == NULL) + mbox->flags_view = mail_index_view_open(mbox->box.index); + + sync_ctx = mail_index_view_sync_begin(mbox->flags_view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + if (mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + /* make sure the map stays in private memory */ + if (mbox->flags_view->map->refcount > 1) { + struct mail_index_map *map; + + map = mail_index_map_clone(mbox->flags_view->map); + mail_index_unmap(&mbox->flags_view->map); + mbox->flags_view->map = map; + } + mail_index_record_map_move_to_private(mbox->flags_view->map); + mail_index_map_move_to_memory(mbox->flags_view->map); + return 0; +} + +struct mailbox_sync_context * +maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box); + bool lost_files, force_resync; + int ret = 0; + + force_resync = (flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0; + if (index_mailbox_want_full_sync(&mbox->box, flags)) { + ret = maildir_sync_run(mbox, flags, force_resync, + NULL, &lost_files); + i_assert(!maildir_uidlist_is_locked(mbox->uidlist) || + (box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0); + + if (lost_files) { + /* lost some files from new/, see if they're in cur/ */ + ret = maildir_storage_sync_force(mbox, 0); + } + } + + if (mbox->storage->set->maildir_very_dirty_syncs) { + if (maildir_sync_refresh_flags_view(mbox) < 0) + ret = -1; + maildir_uidlist_set_all_nonsynced(mbox->uidlist); + } + mbox->synced = TRUE; + mbox->sync_uidlist_refreshed = FALSE; + return index_mailbox_sync_init(box, flags, ret < 0); +} + +int maildir_sync_is_synced(struct maildir_mailbox *mbox) +{ + bool new_changed, cur_changed; + enum maildir_scan_why why; + int ret; + + T_BEGIN { + const char *box_path = mailbox_get_path(&mbox->box); + const char *new_dir, *cur_dir; + + new_dir = t_strconcat(box_path, "/new", NULL); + cur_dir = t_strconcat(box_path, "/cur", NULL); + + ret = maildir_sync_quick_check(mbox, FALSE, new_dir, cur_dir, + &new_changed, &cur_changed, + &why); + } T_END; + return ret < 0 ? -1 : (!new_changed && !cur_changed); +} diff --git a/src/lib-storage/index/maildir/maildir-sync.h b/src/lib-storage/index/maildir/maildir-sync.h new file mode 100644 index 0000000..9bc6db4 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-sync.h @@ -0,0 +1,59 @@ +#ifndef MAILDIR_SYNC_H +#define MAILDIR_SYNC_H + +/* All systems accessing the filesystem must have their clock less than this + many seconds apart from each others. 0 works only for local filesystems. */ +#define MAILDIR_SYNC_SECS 1 + +/* After moving this many mails from new/ to cur/, check if we need to touch + the uidlist lock. */ +#define MAILDIR_SLOW_MOVE_COUNT 100 +/* readdir() should be pretty fast to do, but check anyway every n files + to see if we need to touch the uidlist lock. */ +#define MAILDIR_SLOW_CHECK_COUNT 10000 +/* If syncing takes longer than this, log a warning. */ +#define MAILDIR_SYNC_TIME_WARN_SECS MAIL_TRANSACTION_LOG_LOCK_WARN_SECS + +struct maildir_mailbox; +struct maildir_sync_context; +struct maildir_keywords_sync_ctx; +struct maildir_index_sync_context; + +int maildir_sync_is_synced(struct maildir_mailbox *mbox); + +struct mailbox_sync_context * +maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); +int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid); + +int maildir_sync_header_refresh(struct maildir_mailbox *mbox); + +int maildir_sync_index_begin(struct maildir_mailbox *mbox, + struct maildir_sync_context *maildir_sync_ctx, + struct maildir_index_sync_context **ctx_r) + ATTR_NULL(2); +int maildir_sync_index(struct maildir_index_sync_context *sync_ctx, + bool partial); +int maildir_sync_index_commit(struct maildir_index_sync_context **_ctx); +void maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx); + +struct maildir_keywords_sync_ctx * +maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx); +void maildir_sync_set_racing(struct maildir_sync_context *ctx); +void maildir_sync_notify(struct maildir_sync_context *ctx); +void maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx, + unsigned int count); +int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox); + +int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r); + +int maildir_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq, bool quick, + const char **reason_r); +void maildir_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c new file mode 100644 index 0000000..bb17b9c --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-uidlist.c @@ -0,0 +1,2151 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +/* + Version 1 format has been used for most versions of Dovecot up to v1.0.x. + It's also compatible with Courier IMAP's courierimapuiddb file. + The format is: + + header: 1 <uid validity> <next uid> + entry: <uid> <filename> + + -- + + Version 2 format was written by a few development Dovecot versions, but + v1.0.x still parses the format. The format has <flags> field after <uid>. + + -- + + Version 3 format is an extensible format used by Dovecot v1.1 and later. + It's also parsed by v1.0.2 (and later). The format is: + + header: 3 [<key><value> ...] + entry: <uid> [<key><value> ...] :<filename> + + See enum maildir_uidlist_*_ext_key for used keys. +*/ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "file-dotlock.h" +#include "nfs-workarounds.h" +#include "eacces-error.h" +#include "maildir-storage.h" +#include "maildir-filename.h" +#include "maildir-uidlist.h" + +#include <stdio.h> +#include <sys/stat.h> + +/* NFS: How many times to retry reading dovecot-uidlist file if ESTALE + error occurs in the middle of reading it */ +#define UIDLIST_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT + +#define UIDLIST_VERSION 3 +#define UIDLIST_COMPRESS_PERCENTAGE 75 + +#define UIDLIST_IS_LOCKED(uidlist) \ + ((uidlist)->lock_count > 0) + +struct maildir_uidlist_rec { + uint32_t uid; + uint32_t flags; + char *filename; + unsigned char *extensions; /* <data>\0[<data>\0 ...]\0 */ +}; +ARRAY_DEFINE_TYPE(maildir_uidlist_rec_p, struct maildir_uidlist_rec *); + +HASH_TABLE_DEFINE_TYPE(path_to_maildir_uidlist_rec, + char *, struct maildir_uidlist_rec *); + +struct maildir_uidlist { + struct mailbox *box; + char *path; + struct maildir_index_header *mhdr; + + int fd; + dev_t fd_dev; + ino_t fd_ino; + off_t fd_size; + + unsigned int lock_count; + + struct dotlock_settings dotlock_settings; + struct dotlock *dotlock; + + pool_t record_pool; + ARRAY_TYPE(maildir_uidlist_rec_p) records; + HASH_TABLE_TYPE(path_to_maildir_uidlist_rec) files; + unsigned int change_counter; + + unsigned int version; + unsigned int uid_validity, next_uid, prev_read_uid, last_seen_uid; + unsigned int hdr_next_uid; + unsigned int read_records_count, read_line_count; + uoff_t last_read_offset; + string_t *hdr_extensions; + + guid_128_t mailbox_guid; + + bool recreate:1; + bool recreate_on_change:1; + bool initial_read:1; + bool initial_hdr_read:1; + bool retry_rewind:1; + bool locked_refresh:1; + bool unsorted:1; + bool have_mailbox_guid:1; + bool opened_readonly:1; +}; + +struct maildir_uidlist_sync_ctx { + struct maildir_uidlist *uidlist; + enum maildir_uidlist_sync_flags sync_flags; + + pool_t record_pool; + ARRAY_TYPE(maildir_uidlist_rec_p) records; + HASH_TABLE_TYPE(path_to_maildir_uidlist_rec) files; + + unsigned int first_unwritten_pos, first_new_pos; + unsigned int new_files_count; + unsigned int finish_change_counter; + + bool partial:1; + bool finished:1; + bool changed:1; + bool failed:1; + bool locked:1; +}; + +struct maildir_uidlist_iter_ctx { + struct maildir_uidlist *uidlist; + struct maildir_uidlist_rec *const *next, *const *end; + + unsigned int change_counter; + uint32_t prev_uid; +}; + +static int maildir_uidlist_open_latest(struct maildir_uidlist *uidlist); +static bool maildir_uidlist_iter_next_rec(struct maildir_uidlist_iter_ctx *ctx, + struct maildir_uidlist_rec **rec_r); + +static int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist, + bool nonblock, bool refresh, + bool refresh_when_locked) +{ + struct mailbox *box = uidlist->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *path = uidlist->path; + mode_t old_mask; + const enum dotlock_create_flags dotlock_flags = + nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0; + int i, ret; + + if (uidlist->lock_count > 0) { + if (!uidlist->locked_refresh && refresh_when_locked) { + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } + uidlist->lock_count++; + return 1; + } + + index_storage_lock_notify_reset(box); + + for (i = 0;; i++) { + old_mask = umask(0777 & ~perm->file_create_mode); + ret = file_dotlock_create(&uidlist->dotlock_settings, path, + dotlock_flags, &uidlist->dotlock); + umask(old_mask); + if (ret > 0) + break; + + /* failure */ + if (ret == 0) { + mail_storage_set_error(box->storage, + MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT); + return 0; + } + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + if (errno == EACCES) { + mailbox_set_critical(box, "%s", + eacces_error_get_creating("file_dotlock_create", path)); + } else { + mailbox_set_critical(box, + "file_dotlock_create(%s) failed: %m", + path); + } + return -1; + } + /* the control dir doesn't exist. create it unless the whole + mailbox was just deleted. */ + if (!maildir_set_deleted(uidlist->box)) + return -1; + } + + uidlist->lock_count++; + uidlist->locked_refresh = FALSE; + + if (refresh) { + /* make sure we have the latest changes before + changing anything */ + if (maildir_uidlist_refresh(uidlist) < 0) { + maildir_uidlist_unlock(uidlist); + return -1; + } + } + return 1; +} + +int maildir_uidlist_lock(struct maildir_uidlist *uidlist) +{ + return maildir_uidlist_lock_timeout(uidlist, FALSE, TRUE, FALSE); +} + +int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist) +{ + return maildir_uidlist_lock_timeout(uidlist, TRUE, TRUE, FALSE); +} + +int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist) +{ + i_assert(UIDLIST_IS_LOCKED(uidlist)); + + return file_dotlock_touch(uidlist->dotlock); +} + +bool maildir_uidlist_is_locked(struct maildir_uidlist *uidlist) +{ + return UIDLIST_IS_LOCKED(uidlist); +} + +bool maildir_uidlist_is_read(struct maildir_uidlist *uidlist) +{ + return uidlist->initial_read; +} + +bool maildir_uidlist_is_open(struct maildir_uidlist *uidlist) +{ + return uidlist->fd != -1; +} + +void maildir_uidlist_unlock(struct maildir_uidlist *uidlist) +{ + i_assert(uidlist->lock_count > 0); + + if (--uidlist->lock_count > 0) + return; + + uidlist->locked_refresh = FALSE; + file_dotlock_delete(&uidlist->dotlock); +} + +static bool dotlock_callback(unsigned int secs_left, bool stale, void *context) +{ + struct mailbox *box = context; + + index_storage_lock_notify(box, stale ? + MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE : + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + secs_left); + return TRUE; +} + +struct maildir_uidlist *maildir_uidlist_init(struct maildir_mailbox *mbox) +{ + struct mailbox *box = &mbox->box; + struct maildir_uidlist *uidlist; + const char *control_dir; + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, + &control_dir) <= 0) + i_unreached(); + + uidlist = i_new(struct maildir_uidlist, 1); + uidlist->box = box; + uidlist->mhdr = &mbox->maildir_hdr; + uidlist->fd = -1; + uidlist->path = i_strconcat(control_dir, "/"MAILDIR_UIDLIST_NAME, NULL); + i_array_init(&uidlist->records, 128); + hash_table_create(&uidlist->files, default_pool, 4096, + maildir_filename_base_hash, + maildir_filename_base_cmp); + uidlist->next_uid = 1; + uidlist->hdr_extensions = str_new(default_pool, 128); + + uidlist->dotlock_settings.use_io_notify = TRUE; + uidlist->dotlock_settings.use_excl_lock = + box->storage->set->dotlock_use_excl; + uidlist->dotlock_settings.nfs_flush = + box->storage->set->mail_nfs_storage; + uidlist->dotlock_settings.timeout = + mail_storage_get_lock_timeout(box->storage, + MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT + 2); + uidlist->dotlock_settings.stale_timeout = + MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT; + uidlist->dotlock_settings.callback = dotlock_callback; + uidlist->dotlock_settings.context = box; + uidlist->dotlock_settings.temp_prefix = mbox->storage->temp_prefix; + return uidlist; +} + +static void maildir_uidlist_close(struct maildir_uidlist *uidlist) +{ + if (uidlist->fd != -1) { + if (close(uidlist->fd) < 0) { + mailbox_set_critical(uidlist->box, + "close(%s) failed: %m", uidlist->path); + } + uidlist->fd = -1; + uidlist->fd_ino = 0; + } + uidlist->last_read_offset = 0; + uidlist->read_line_count = 0; +} + +static void maildir_uidlist_reset(struct maildir_uidlist *uidlist) +{ + maildir_uidlist_close(uidlist); + uidlist->last_seen_uid = 0; + uidlist->initial_hdr_read = FALSE; + uidlist->read_records_count = 0; + + hash_table_clear(uidlist->files, FALSE); + array_clear(&uidlist->records); +} + +void maildir_uidlist_deinit(struct maildir_uidlist **_uidlist) +{ + struct maildir_uidlist *uidlist = *_uidlist; + + i_assert(!UIDLIST_IS_LOCKED(uidlist)); + + *_uidlist = NULL; + (void)maildir_uidlist_update(uidlist); + maildir_uidlist_close(uidlist); + + hash_table_destroy(&uidlist->files); + pool_unref(&uidlist->record_pool); + + array_free(&uidlist->records); + str_free(&uidlist->hdr_extensions); + i_free(uidlist->path); + i_free(uidlist); +} + +static int maildir_uid_cmp(struct maildir_uidlist_rec *const *rec1, + struct maildir_uidlist_rec *const *rec2) +{ + return (*rec1)->uid < (*rec2)->uid ? -1 : + (*rec1)->uid > (*rec2)->uid ? 1 : 0; +} + +static void ATTR_FORMAT(2, 3) +maildir_uidlist_set_corrupted(struct maildir_uidlist *uidlist, + const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + if (uidlist->retry_rewind) { + mailbox_set_critical(uidlist->box, + "Broken or unexpectedly changed file %s " + "line %u: %s - re-reading from beginning", + uidlist->path, uidlist->read_line_count, + t_strdup_vprintf(fmt, args)); + } else { + mailbox_set_critical(uidlist->box, "Broken file %s line %u: %s", + uidlist->path, uidlist->read_line_count, + t_strdup_vprintf(fmt, args)); + } + va_end(args); +} + +static void maildir_uidlist_update_hdr(struct maildir_uidlist *uidlist, + const struct stat *st) +{ + struct maildir_index_header *mhdr = uidlist->mhdr; + + if (mhdr->uidlist_mtime == 0 && uidlist->version != UIDLIST_VERSION) { + /* upgrading from older version. don't update the + uidlist times until it uses the new format */ + uidlist->recreate = TRUE; + return; + } + mhdr->uidlist_mtime = st->st_mtime; + mhdr->uidlist_mtime_nsecs = ST_MTIME_NSEC(*st); + mhdr->uidlist_size = st->st_size; +} + +static unsigned int +maildir_uidlist_records_array_delete(struct maildir_uidlist *uidlist, + struct maildir_uidlist_rec *rec) +{ + struct maildir_uidlist_rec *const *recs, *const *pos; + unsigned int idx, count; + + recs = array_get(&uidlist->records, &count); + if (!uidlist->unsorted) { + pos = array_bsearch(&uidlist->records, &rec, maildir_uid_cmp); + i_assert(pos != NULL); + idx = pos - recs; + } else { + for (idx = 0; idx < count; idx++) { + if (recs[idx]->uid == rec->uid) + break; + } + i_assert(idx != count); + } + array_delete(&uidlist->records, idx, 1); + return idx; +} + +static bool +maildir_uidlist_read_extended(struct maildir_uidlist *uidlist, + const char **line_p, + struct maildir_uidlist_rec *rec) +{ + const char *start, *line = *line_p; + buffer_t *buf; + + buf = t_buffer_create(128); + while (*line != '\0' && *line != ':') { + /* skip over an extension field */ + start = line; + while (*line != ' ' && *line != '\0') line++; + if (MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*start)) { + buffer_append(buf, start, line - start); + buffer_append_c(buf, '\0'); + } else { + maildir_uidlist_set_corrupted(uidlist, + "Invalid extension record, removing: %s", + t_strdup_until(start, line)); + uidlist->recreate = TRUE; + } + while (*line == ' ') line++; + } + + if (buf->used > 0) { + /* save the extensions */ + buffer_append_c(buf, '\0'); + rec->extensions = p_malloc(uidlist->record_pool, buf->used); + memcpy(rec->extensions, buf->data, buf->used); + } + + if (*line == ':') + line++; + if (*line == '\0') + return FALSE; + + *line_p = line; + return TRUE; +} + +static bool maildir_uidlist_next(struct maildir_uidlist *uidlist, + const char *line) +{ + struct maildir_uidlist_rec *rec, *old_rec, *const *recs; + unsigned int count; + uint32_t uid; + + uid = 0; + while (*line >= '0' && *line <= '9') { + uid = uid*10 + (*line - '0'); + line++; + } + + if (uid == 0 || *line != ' ') { + /* invalid file */ + maildir_uidlist_set_corrupted(uidlist, "Invalid data: %s", + line); + return FALSE; + } + if (uid <= uidlist->prev_read_uid) { + maildir_uidlist_set_corrupted(uidlist, + "UIDs not ordered (%u >= %u)", + uid, uidlist->prev_read_uid); + return FALSE; + } + if (uid >= (uint32_t)-1) { + maildir_uidlist_set_corrupted(uidlist, + "UID too high (%u)", uid); + return FALSE; + } + uidlist->prev_read_uid = uid; + + if (uid <= uidlist->last_seen_uid) { + /* we already have this */ + return TRUE; + } + uidlist->last_seen_uid = uid; + + if (uid >= uidlist->next_uid && uidlist->version == 1) { + maildir_uidlist_set_corrupted(uidlist, + "UID larger than next_uid (%u >= %u)", + uid, uidlist->next_uid); + return FALSE; + } + + rec = p_new(uidlist->record_pool, struct maildir_uidlist_rec, 1); + rec->uid = uid; + rec->flags = MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; + + while (*line == ' ') line++; + + if (uidlist->version == UIDLIST_VERSION) { + /* read extended fields */ + bool ret; + + T_BEGIN { + ret = maildir_uidlist_read_extended(uidlist, &line, + rec); + } T_END; + if (!ret) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid extended fields: %s", line); + return FALSE; + } + } + + if (strchr(line, '/') != NULL) { + maildir_uidlist_set_corrupted(uidlist, + "%s: Broken filename at line %u: %s", + uidlist->path, uidlist->read_line_count, line); + return FALSE; + } + + old_rec = hash_table_lookup(uidlist->files, line); + if (old_rec == NULL) { + /* no conflicts */ + } else if (old_rec->uid == uid) { + /* most likely this is a record we saved ourself, but couldn't + update last_seen_uid because uidlist wasn't refreshed while + it was locked. + + another possibility is a duplicate file record. currently + it would be a bug, but not that big of a deal. also perhaps + in future such duplicate lines could be used to update + extended fields. so just let it through anyway. + + we'll waste a bit of memory here by allocating the record + twice, but that's not really a problem. */ + rec->filename = old_rec->filename; + hash_table_update(uidlist->files, rec->filename, rec); + uidlist->unsorted = TRUE; + return TRUE; + } else { + /* This can happen if expunged file is moved back and the file + was appended to uidlist. */ + i_warning("%s: Duplicate file entry at line %u: " + "%s (uid %u -> %u)%s", + uidlist->path, uidlist->read_line_count, line, + old_rec->uid, uid, uidlist->retry_rewind ? + " - retrying by re-reading from beginning" : ""); + if (uidlist->retry_rewind) + return FALSE; + /* Delete the old UID */ + (void)maildir_uidlist_records_array_delete(uidlist, old_rec); + /* Replace the old record with this new one */ + *old_rec = *rec; + rec = old_rec; + uidlist->recreate = TRUE; + } + + recs = array_get(&uidlist->records, &count); + if (count > 0 && recs[count-1]->uid > uid) { + /* we most likely have some records in the array that we saved + ourself without refreshing uidlist */ + uidlist->unsorted = TRUE; + } + + rec->filename = p_strdup(uidlist->record_pool, line); + hash_table_update(uidlist->files, rec->filename, rec); + array_push_back(&uidlist->records, &rec); + return TRUE; +} + +static int +maildir_uidlist_read_v3_header(struct maildir_uidlist *uidlist, + const char *line, + unsigned int *uid_validity_r, + unsigned int *next_uid_r) +{ + char key; + + str_truncate(uidlist->hdr_extensions, 0); + while (*line != '\0') { + const char *value; + + key = *line; + value = ++line; + while (*line != '\0' && *line != ' ') line++; + value = t_strdup_until(value, line); + + switch (key) { + case MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY: + if (str_to_uint(value, uid_validity_r) < 0) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid mailbox UID_VALIDITY: %s", value); + return -1; + } + break; + case MAILDIR_UIDLIST_HDR_EXT_NEXT_UID: + if (str_to_uint(value, next_uid_r) < 0) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid mailbox NEXT_UID: %s", value); + return -1; + } + break; + case MAILDIR_UIDLIST_HDR_EXT_GUID: + if (guid_128_from_string(value, + uidlist->mailbox_guid) < 0) { + maildir_uidlist_set_corrupted(uidlist, + "Invalid mailbox GUID: %s", value); + return -1; + } + uidlist->have_mailbox_guid = TRUE; + break; + default: + if (str_len(uidlist->hdr_extensions) > 0) + str_append_c(uidlist->hdr_extensions, ' '); + str_printfa(uidlist->hdr_extensions, + "%c%s", key, value); + break; + } + + while (*line == ' ') line++; + } + return 0; +} + +static int maildir_uidlist_read_header(struct maildir_uidlist *uidlist, + struct istream *input) +{ + unsigned int uid_validity = 0, next_uid = 0; + const char *line; + int ret; + + line = i_stream_read_next_line(input); + if (line == NULL) { + /* I/O error / empty file */ + return input->stream_errno == 0 ? 0 : -1; + } + uidlist->read_line_count = 1; + + if (*line < '0' || *line > '9' || line[1] != ' ') { + maildir_uidlist_set_corrupted(uidlist, + "Corrupted header (invalid version number)"); + return 0; + } + + uidlist->version = *line - '0'; + line += 2; + + switch (uidlist->version) { + case 1: + if (sscanf(line, "%u %u", &uid_validity, &next_uid) != 2) { + maildir_uidlist_set_corrupted(uidlist, + "Corrupted header (version 1)"); + return 0; + } + break; + case UIDLIST_VERSION: + T_BEGIN { + ret = maildir_uidlist_read_v3_header(uidlist, line, + &uid_validity, + &next_uid); + } T_END; + if (ret < 0) + return 0; + break; + default: + maildir_uidlist_set_corrupted(uidlist, "Unsupported version %u", + uidlist->version); + return 0; + } + + if (uid_validity == 0 || next_uid == 0) { + maildir_uidlist_set_corrupted(uidlist, + "Broken header (uidvalidity = %u, next_uid=%u)", + uid_validity, next_uid); + return 0; + } + + if (uid_validity == uidlist->uid_validity && + next_uid < uidlist->hdr_next_uid) { + maildir_uidlist_set_corrupted(uidlist, + "next_uid header was lowered (%u -> %u)", + uidlist->hdr_next_uid, next_uid); + return 0; + } + + uidlist->uid_validity = uid_validity; + uidlist->next_uid = next_uid; + uidlist->hdr_next_uid = next_uid; + return 1; +} + +static void maildir_uidlist_records_sort_by_uid(struct maildir_uidlist *uidlist) +{ + array_sort(&uidlist->records, maildir_uid_cmp); + uidlist->unsorted = FALSE; +} + +static int +maildir_uidlist_update_read(struct maildir_uidlist *uidlist, + bool *retry_r, bool try_retry) +{ + const char *line; + uint32_t orig_next_uid, orig_uid_validity; + struct istream *input; + struct stat st; + uoff_t last_read_offset; + int fd, ret; + bool readonly = FALSE; + + *retry_r = FALSE; + + if (uidlist->fd == -1) { + fd = nfs_safe_open(uidlist->path, O_RDWR); + if (fd == -1 && errno == EACCES) { + fd = nfs_safe_open(uidlist->path, O_RDONLY); + readonly = TRUE; + } + if (fd == -1) { + if (errno != ENOENT) { + mailbox_set_critical(uidlist->box, + "open(%s) failed: %m", uidlist->path); + return -1; + } + return 0; + } + last_read_offset = 0; + } else { + /* the file was updated */ + fd = uidlist->fd; + if (lseek(fd, 0, SEEK_SET) < 0) { + if (errno == ESTALE && try_retry) { + *retry_r = TRUE; + return -1; + } + mailbox_set_critical(uidlist->box, + "lseek(%s) failed: %m", uidlist->path); + return -1; + } + uidlist->fd = -1; + uidlist->fd_ino = 0; + last_read_offset = uidlist->last_read_offset; + uidlist->last_read_offset = 0; + } + + if (fstat(fd, &st) < 0) { + i_close_fd(&fd); + if (errno == ESTALE && try_retry) { + *retry_r = TRUE; + return -1; + } + mailbox_set_critical(uidlist->box, + "fstat(%s) failed: %m", uidlist->path); + return -1; + } + + if (uidlist->record_pool == NULL) { + uidlist->record_pool = + pool_alloconly_create(MEMPOOL_GROWING + "uidlist record_pool", + nearest_power(st.st_size - + st.st_size/8)); + } + + input = i_stream_create_fd(fd, SIZE_MAX); + i_stream_seek(input, last_read_offset); + + orig_uid_validity = uidlist->uid_validity; + orig_next_uid = uidlist->next_uid; + ret = input->v_offset != 0 ? 1 : + maildir_uidlist_read_header(uidlist, input); + if (ret > 0) { + uidlist->prev_read_uid = 0; + uidlist->change_counter++; + uidlist->retry_rewind = last_read_offset != 0 && try_retry; + + ret = 1; + while ((line = i_stream_read_next_line(input)) != NULL) { + uidlist->read_records_count++; + uidlist->read_line_count++; + if (!maildir_uidlist_next(uidlist, line)) { + if (!uidlist->retry_rewind) + ret = 0; + else { + ret = -1; + *retry_r = TRUE; + } + break; + } + } + uidlist->retry_rewind = FALSE; + if (input->stream_errno != 0) + ret = -1; + + if (uidlist->unsorted) { + uidlist->recreate_on_change = TRUE; + maildir_uidlist_records_sort_by_uid(uidlist); + } + if (uidlist->next_uid <= uidlist->prev_read_uid) + uidlist->next_uid = uidlist->prev_read_uid + 1; + if (ret > 0 && uidlist->uid_validity != orig_uid_validity && + orig_uid_validity != 0) { + uidlist->recreate = TRUE; + } else if (ret > 0 && uidlist->next_uid < orig_next_uid) { + mailbox_set_critical(uidlist->box, + "%s: next_uid was lowered (%u -> %u, hdr=%u)", + uidlist->path, orig_next_uid, + uidlist->next_uid, uidlist->hdr_next_uid); + uidlist->recreate = TRUE; + uidlist->next_uid = orig_next_uid; + } + } + + if (ret == 0) { + /* file is broken */ + i_unlink(uidlist->path); + } else if (ret > 0) { + /* success */ + if (readonly) + uidlist->recreate_on_change = TRUE; + uidlist->fd = fd; + uidlist->fd_dev = st.st_dev; + uidlist->fd_ino = st.st_ino; + uidlist->fd_size = st.st_size; + uidlist->last_read_offset = input->v_offset; + maildir_uidlist_update_hdr(uidlist, &st); + } else if (!*retry_r) { + /* I/O error */ + if (input->stream_errno == ESTALE && try_retry) + *retry_r = TRUE; + else { + mailbox_set_critical(uidlist->box, + "read(%s) failed: %s", uidlist->path, + i_stream_get_error(input)); + } + uidlist->last_read_offset = 0; + } + + i_stream_destroy(&input); + if (ret <= 0) { + if (close(fd) < 0) { + mailbox_set_critical(uidlist->box, + "close(%s) failed: %m", uidlist->path); + } + } + return ret; +} + +static int +maildir_uidlist_stat(struct maildir_uidlist *uidlist, struct stat *st_r) +{ + struct mail_storage *storage = uidlist->box->storage; + + if (storage->set->mail_nfs_storage) { + nfs_flush_file_handle_cache(uidlist->path); + nfs_flush_attr_cache_unlocked(uidlist->path); + } + if (nfs_safe_stat(uidlist->path, st_r) < 0) { + if (errno != ENOENT) { + mailbox_set_critical(uidlist->box, + "stat(%s) failed: %m", uidlist->path); + return -1; + } + return 0; + } + return 1; +} + +static int +maildir_uidlist_has_changed(struct maildir_uidlist *uidlist, bool *recreated_r) +{ + struct mail_storage *storage = uidlist->box->storage; + struct stat st; + int ret; + + *recreated_r = FALSE; + + if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0) + return -1; + if (ret == 0) { + *recreated_r = TRUE; + return 1; + } + + if (st.st_ino != uidlist->fd_ino || + !CMP_DEV_T(st.st_dev, uidlist->fd_dev)) { + /* file recreated */ + *recreated_r = TRUE; + return 1; + } + + if (storage->set->mail_nfs_storage) { + /* NFS: either the file hasn't been changed, or it has already + been deleted and the inodes just happen to be the same. + check if the fd is still valid. */ + if (fstat(uidlist->fd, &st) < 0) { + if (errno == ESTALE) { + *recreated_r = TRUE; + return 1; + } + mailbox_set_critical(uidlist->box, + "fstat(%s) failed: %m", uidlist->path); + return -1; + } + } + + if (st.st_size != uidlist->fd_size) { + /* file modified but not recreated */ + return 1; + } else { + /* unchanged */ + return 0; + } +} + +static int maildir_uidlist_open_latest(struct maildir_uidlist *uidlist) +{ + bool recreated; + int ret; + + if (uidlist->fd != -1) { + ret = maildir_uidlist_has_changed(uidlist, &recreated); + if (ret <= 0) { + if (UIDLIST_IS_LOCKED(uidlist)) + uidlist->locked_refresh = TRUE; + return ret < 0 ? -1 : 1; + } + + if (!recreated) + return 0; + maildir_uidlist_reset(uidlist); + } + + uidlist->fd = nfs_safe_open(uidlist->path, O_RDWR); + if (uidlist->fd == -1 && errno == EACCES) { + uidlist->fd = nfs_safe_open(uidlist->path, O_RDONLY); + uidlist->recreate_on_change = TRUE; + } + if (uidlist->fd == -1 && errno != ENOENT) { + mailbox_set_critical(uidlist->box, + "open(%s) failed: %m", uidlist->path); + return -1; + } + return 0; +} + +int maildir_uidlist_refresh(struct maildir_uidlist *uidlist) +{ + unsigned int i; + bool retry; + int ret; + + if (maildir_uidlist_open_latest(uidlist) < 0) + return -1; + + for (i = 0; ; i++) { + ret = maildir_uidlist_update_read(uidlist, &retry, + i < UIDLIST_ESTALE_RETRY_COUNT); + if (!retry) + break; + /* ESTALE - try reopening and rereading */ + maildir_uidlist_close(uidlist); + } + if (ret >= 0) { + uidlist->initial_read = TRUE; + uidlist->initial_hdr_read = TRUE; + if (UIDLIST_IS_LOCKED(uidlist)) + uidlist->locked_refresh = TRUE; + if (!uidlist->have_mailbox_guid) { + uidlist->recreate = TRUE; + (void)maildir_uidlist_update(uidlist); + } + } + return ret; +} + +int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist) +{ + const struct maildir_index_header *mhdr = uidlist->mhdr; + struct mail_index *index = uidlist->box->index; + struct mail_index_view *view; + const struct mail_index_header *hdr; + struct stat st; + int ret; + + i_assert(UIDLIST_IS_LOCKED(uidlist)); + + if (uidlist->fd != -1) + return maildir_uidlist_refresh(uidlist); + + if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0) + return ret; + + if (ret > 0 && st.st_size == mhdr->uidlist_size && + st.st_mtime == (time_t)mhdr->uidlist_mtime && + ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), mhdr->uidlist_mtime_nsecs) && + (!mail_index_is_in_memory(index) || st.st_mtime < ioloop_time-1)) { + /* index is up-to-date. look up the uidvalidity and next-uid + from it. we'll need to create a new view temporarily to + make sure we get the latest values. */ + view = mail_index_view_open(index); + hdr = mail_index_get_header(view); + uidlist->uid_validity = hdr->uid_validity; + uidlist->next_uid = hdr->next_uid; + uidlist->initial_hdr_read = TRUE; + mail_index_view_close(&view); + + if (UIDLIST_IS_LOCKED(uidlist)) + uidlist->locked_refresh = TRUE; + return 1; + } else { + return maildir_uidlist_refresh(uidlist); + } +} + +static int +maildir_uid_bsearch_cmp(const uint32_t *uidp, + struct maildir_uidlist_rec *const *recp) +{ + return *uidp < (*recp)->uid ? -1 : + *uidp > (*recp)->uid ? 1 : 0; +} + +static int +maildir_uidlist_lookup_rec(struct maildir_uidlist *uidlist, uint32_t uid, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist_rec *const *pos; + + if (!uidlist->initial_read) { + /* first time we need to read uidlist */ + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } + + pos = array_bsearch(&uidlist->records, &uid, + maildir_uid_bsearch_cmp); + if (pos == NULL) { + *rec_r = NULL; + return 0; + } + *rec_r = *pos; + return 1; +} + +int maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r) +{ + struct maildir_uidlist_rec *rec; + int ret; + + if ((ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec)) <= 0) + return ret; + + *flags_r = rec->flags; + *fname_r = rec->filename; + return 1; +} + +const char * +maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key) +{ + struct maildir_uidlist_rec *rec; + const unsigned char *p; + int ret; + + ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec); + if (ret <= 0 || rec->extensions == NULL) + return NULL; + + p = rec->extensions; + while (*p != '\0') { + /* <key><value>\0 */ + if (*p == (unsigned char)key) + return (const char *)p + 1; + + p += strlen((const char *)p) + 1; + } + return NULL; +} + +uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist) +{ + return uidlist->uid_validity; +} + +uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist) +{ + return !uidlist->initial_hdr_read ? 0 : uidlist->next_uid; +} + +int maildir_uidlist_get_mailbox_guid(struct maildir_uidlist *uidlist, + guid_128_t mailbox_guid) +{ + if (!uidlist->initial_hdr_read) { + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } + if (!uidlist->have_mailbox_guid) { + uidlist->recreate = TRUE; + if (maildir_uidlist_update(uidlist) < 0) + return -1; + } + memcpy(mailbox_guid, uidlist->mailbox_guid, GUID_128_SIZE); + return 0; +} + +void maildir_uidlist_set_mailbox_guid(struct maildir_uidlist *uidlist, + const guid_128_t mailbox_guid) +{ + if (memcmp(uidlist->mailbox_guid, mailbox_guid, + sizeof(uidlist->mailbox_guid)) != 0) { + memcpy(uidlist->mailbox_guid, mailbox_guid, + sizeof(uidlist->mailbox_guid)); + uidlist->recreate = TRUE; + } +} + +void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist, + uint32_t uid_validity) +{ + i_assert(uid_validity != 0); + + if (uid_validity != uidlist->uid_validity) { + uidlist->uid_validity = uid_validity; + uidlist->recreate = TRUE; + } +} + +void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist, + uint32_t next_uid, bool force) +{ + if (uidlist->next_uid < next_uid || force) { + i_assert(next_uid != 0); + uidlist->next_uid = next_uid; + uidlist->recreate = TRUE; + } +} + +static void +maildir_uidlist_rec_set_ext(struct maildir_uidlist_rec *rec, pool_t pool, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + const unsigned char *p; + buffer_t *buf; + size_t len; + + /* copy existing extensions, except for the one we're updating */ + buf = t_buffer_create(128); + if (rec->extensions != NULL) { + p = rec->extensions; + while (*p != '\0') { + /* <key><value>\0 */ + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*p)); + + len = strlen((const char *)p) + 1; + if (*p != (unsigned char)key) + buffer_append(buf, p, len); + p += len; + } + } + if (value != NULL) { + buffer_append_c(buf, key); + buffer_append(buf, value, strlen(value) + 1); + } + buffer_append_c(buf, '\0'); + + rec->extensions = p_malloc(pool, buf->used); + memcpy(rec->extensions, buf->data, buf->used); +} + +static void ATTR_NULL(4) +maildir_uidlist_set_ext_internal(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + struct maildir_uidlist_rec *rec; + int ret; + + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(key)); + + ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec); + if (ret <= 0) { + if (ret < 0) + return; + + /* maybe it's a new message */ + if (maildir_uidlist_refresh(uidlist) < 0) + return; + if (maildir_uidlist_lookup_rec(uidlist, uid, &rec) <= 0) { + /* message is already expunged, ignore */ + return; + } + } + + T_BEGIN { + maildir_uidlist_rec_set_ext(rec, uidlist->record_pool, + key, value); + } T_END; + + if (rec->uid != (uint32_t)-1) { + /* message already exists in uidlist, need to recreate it */ + uidlist->recreate = TRUE; + } +} + +void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + maildir_uidlist_set_ext_internal(uidlist, uid, key, value); +} + +void maildir_uidlist_unset_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key) +{ + maildir_uidlist_set_ext_internal(uidlist, uid, key, NULL); +} + +static void +maildir_uidlist_generate_uid_validity(struct maildir_uidlist *uidlist) +{ + const struct mail_index_header *hdr; + + if (uidlist->box->opened) { + hdr = mail_index_get_header(uidlist->box->view); + if (hdr->uid_validity != 0) { + uidlist->uid_validity = hdr->uid_validity; + return; + } + } + uidlist->uid_validity = + maildir_get_uidvalidity_next(uidlist->box->list); +} + +static int maildir_uidlist_write_fd(struct maildir_uidlist *uidlist, int fd, + const char *path, unsigned int first_idx, + uoff_t *file_size_r) +{ + struct mail_storage *storage = uidlist->box->storage; + struct maildir_uidlist_iter_ctx *iter; + struct ostream *output; + struct maildir_uidlist_rec *rec; + string_t *str; + const unsigned char *p; + const char *strp; + size_t len; + + i_assert(fd != -1); + + output = o_stream_create_fd_file(fd, UOFF_T_MAX, FALSE); + o_stream_cork(output); + str = t_str_new(512); + + if (output->offset == 0) { + i_assert(first_idx == 0); + uidlist->version = UIDLIST_VERSION; + + if (uidlist->uid_validity == 0) + maildir_uidlist_generate_uid_validity(uidlist); + if (!uidlist->have_mailbox_guid) + guid_128_generate(uidlist->mailbox_guid); + + i_assert(uidlist->next_uid > 0); + str_printfa(str, "%u %c%u %c%u %c%s", uidlist->version, + MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY, + uidlist->uid_validity, + MAILDIR_UIDLIST_HDR_EXT_NEXT_UID, + uidlist->next_uid, + MAILDIR_UIDLIST_HDR_EXT_GUID, + guid_128_to_string(uidlist->mailbox_guid)); + if (str_len(uidlist->hdr_extensions) > 0) { + str_append_c(str, ' '); + str_append_str(str, uidlist->hdr_extensions); + } + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); + } + + iter = maildir_uidlist_iter_init(uidlist); + i_assert(first_idx <= array_count(&uidlist->records)); + iter->next += first_idx; + + while (maildir_uidlist_iter_next_rec(iter, &rec)) { + uidlist->read_records_count++; + str_truncate(str, 0); + str_printfa(str, "%u", rec->uid); + if (rec->extensions != NULL) { + for (p = rec->extensions; *p != '\0'; ) { + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*p)); + len = strlen((const char *)p); + str_append_c(str, ' '); + str_append_data(str, p, len); + p += len + 1; + } + } + str_append(str, " :"); + strp = strchr(rec->filename, *MAILDIR_INFO_SEP_S); + if (strp == NULL) + str_append(str, rec->filename); + else + str_append_data(str, rec->filename, strp - rec->filename); + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); + } + maildir_uidlist_iter_deinit(&iter); + + if (o_stream_finish(output) < 0) { + mailbox_set_critical(uidlist->box, "write(%s) failed: %s", + path, o_stream_get_error(output)); + o_stream_unref(&output); + return -1; + } + + *file_size_r = output->offset; + o_stream_unref(&output); + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync(fd) < 0) { + mailbox_set_critical(uidlist->box, + "fdatasync(%s) failed: %m", path); + return -1; + } + } + return 0; +} + +static void +maildir_uidlist_records_drop_expunges(struct maildir_uidlist *uidlist) +{ + struct mail_index_view *view; + struct maildir_uidlist_rec *const *recs; + ARRAY_TYPE(maildir_uidlist_rec_p) new_records; + const struct mail_index_header *hdr; + const struct mail_index_record *rec; + unsigned int i, count; + uint32_t seq; + + /* we could get here when opening and locking mailbox, + before index files have been opened. */ + if (!uidlist->box->opened) + return; + + mail_index_refresh(uidlist->box->index); + view = mail_index_view_open(uidlist->box->index); + count = array_count(&uidlist->records); + hdr = mail_index_get_header(view); + if (count * UIDLIST_COMPRESS_PERCENTAGE / 100 <= hdr->messages_count) { + /* too much trouble to be worth it */ + mail_index_view_close(&view); + return; + } + + i_array_init(&new_records, hdr->messages_count + 64); + recs = array_get(&uidlist->records, &count); + for (i = 0, seq = 1; i < count && seq <= hdr->messages_count; ) { + rec = mail_index_lookup(view, seq); + if (recs[i]->uid < rec->uid) { + /* expunged entry */ + hash_table_remove(uidlist->files, recs[i]->filename); + i++; + } else if (recs[i]->uid > rec->uid) { + /* index isn't up to date. we're probably just + syncing it here. ignore this entry. */ + seq++; + } else { + array_push_back(&new_records, &recs[i]); + seq++; i++; + } + } + + /* drop messages expunged at the end of index */ + while (i < count && recs[i]->uid < hdr->next_uid) { + hash_table_remove(uidlist->files, recs[i]->filename); + i++; + } + /* view might not be completely up-to-date, so preserve any + messages left */ + for (; i < count; i++) + array_push_back(&new_records, &recs[i]); + + array_free(&uidlist->records); + uidlist->records = new_records; + + mail_index_view_close(&view); +} + +static int maildir_uidlist_recreate(struct maildir_uidlist *uidlist) +{ + struct mailbox *box = uidlist->box; + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *control_dir, *temp_path; + struct stat st; + mode_t old_mask; + uoff_t file_size; + int i, fd, ret; + + i_assert(uidlist->initial_read); + + maildir_uidlist_records_drop_expunges(uidlist); + + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, + &control_dir) <= 0) + i_unreached(); + temp_path = t_strconcat(control_dir, + "/" MAILDIR_UIDLIST_NAME ".tmp", NULL); + + for (i = 0;; i++) { + old_mask = umask(0777 & ~perm->file_create_mode); + fd = open(temp_path, O_RDWR | O_CREAT | O_TRUNC, 0777); + umask(old_mask); + if (fd != -1) + break; + + if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) { + mailbox_set_critical(box, + "open(%s, O_CREAT) failed: %m", temp_path); + return -1; + } + /* the control dir doesn't exist. create it unless the whole + mailbox was just deleted. */ + if (!maildir_set_deleted(uidlist->box)) + return -1; + } + + if (perm->file_create_gid != (gid_t)-1 && + fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) { + if (errno == EPERM) { + mailbox_set_critical(box, "%s", + eperm_error_get_chgrp("fchown", temp_path, + perm->file_create_gid, + perm->file_create_gid_origin)); + } else { + mailbox_set_critical(box, + "fchown(%s) failed: %m", temp_path); + } + } + + uidlist->read_records_count = 0; + ret = maildir_uidlist_write_fd(uidlist, fd, temp_path, 0, &file_size); + if (ret == 0) { + if (rename(temp_path, uidlist->path) < 0) { + mailbox_set_critical(box, + "rename(%s, %s) failed: %m", + temp_path, uidlist->path); + ret = -1; + } + } + + if (ret < 0) + i_unlink(temp_path); + else if (fstat(fd, &st) < 0) { + mailbox_set_critical(box, + "fstat(%s) failed: %m", temp_path); + ret = -1; + } else if (file_size != (uoff_t)st.st_size) { + i_assert(!file_dotlock_is_locked(uidlist->dotlock)); + mailbox_set_critical(box, + "Maildir uidlist dotlock overridden: %s", + uidlist->path); + ret = -1; + } else { + maildir_uidlist_close(uidlist); + uidlist->fd = fd; + uidlist->fd_dev = st.st_dev; + uidlist->fd_ino = st.st_ino; + uidlist->fd_size = st.st_size; + uidlist->last_read_offset = st.st_size; + uidlist->recreate = FALSE; + uidlist->recreate_on_change = FALSE; + uidlist->have_mailbox_guid = TRUE; + maildir_uidlist_update_hdr(uidlist, &st); + } + if (ret < 0) + i_close_fd(&fd); + return ret; +} + +int maildir_uidlist_update(struct maildir_uidlist *uidlist) +{ + int ret; + + if (!uidlist->recreate) + return 0; + + if (maildir_uidlist_lock(uidlist) <= 0) + return -1; + ret = maildir_uidlist_recreate(uidlist); + maildir_uidlist_unlock(uidlist); + return ret; +} + +static bool maildir_uidlist_want_compress(struct maildir_uidlist_sync_ctx *ctx) +{ + unsigned int min_rewrite_count; + + if (!ctx->uidlist->locked_refresh) + return FALSE; + if (ctx->uidlist->recreate) + return TRUE; + + min_rewrite_count = + (ctx->uidlist->read_records_count + ctx->new_files_count) * + UIDLIST_COMPRESS_PERCENTAGE / 100; + return min_rewrite_count >= array_count(&ctx->uidlist->records); +} + +static bool maildir_uidlist_want_recreate(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + + if (!uidlist->locked_refresh || !uidlist->initial_read) + return FALSE; + + if (ctx->finish_change_counter != uidlist->change_counter) + return TRUE; + if (uidlist->fd == -1 || uidlist->version != UIDLIST_VERSION || + !uidlist->have_mailbox_guid) + return TRUE; + return maildir_uidlist_want_compress(ctx); +} + +static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct stat st; + uoff_t file_size; + + if (maildir_uidlist_want_recreate(ctx) || uidlist->recreate_on_change) + return maildir_uidlist_recreate(uidlist); + + if (!uidlist->locked_refresh || uidlist->fd == -1) { + /* make sure we have the latest file (e.g. NOREFRESH used) */ + i_assert(uidlist->initial_hdr_read); + if (maildir_uidlist_open_latest(uidlist) < 0) + return -1; + if (uidlist->recreate_on_change) + return maildir_uidlist_recreate(uidlist); + } + i_assert(ctx->first_unwritten_pos != UINT_MAX); + + if (lseek(uidlist->fd, 0, SEEK_END) < 0) { + mailbox_set_critical(uidlist->box, + "lseek(%s) failed: %m", uidlist->path); + return -1; + } + + if (maildir_uidlist_write_fd(uidlist, uidlist->fd, uidlist->path, + ctx->first_unwritten_pos, &file_size) < 0) + return -1; + + if (fstat(uidlist->fd, &st) < 0) { + mailbox_set_critical(uidlist->box, + "fstat(%s) failed: %m", uidlist->path); + return -1; + } + if ((uoff_t)st.st_size != file_size) { + i_warning("%s: file size changed unexpectedly after write", + uidlist->path); + } else if (uidlist->locked_refresh) { + uidlist->fd_size = st.st_size; + uidlist->last_read_offset = st.st_size; + maildir_uidlist_update_hdr(uidlist, &st); + } + return 0; +} + +static void maildir_uidlist_mark_all(struct maildir_uidlist *uidlist, + bool nonsynced) +{ + struct maildir_uidlist_rec **recs; + unsigned int i, count; + + recs = array_get_modifiable(&uidlist->records, &count); + if (nonsynced) { + for (i = 0; i < count; i++) + recs[i]->flags |= MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; + } else { + for (i = 0; i < count; i++) + recs[i]->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + } +} + +static int maildir_uidlist_sync_lock(struct maildir_uidlist *uidlist, + enum maildir_uidlist_sync_flags sync_flags, + bool *locked_r) +{ + bool nonblock, refresh; + int ret; + + *locked_r = FALSE; + + if ((sync_flags & MAILDIR_UIDLIST_SYNC_NOLOCK) != 0) { + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + return 1; + } + + nonblock = (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0; + refresh = (sync_flags & MAILDIR_UIDLIST_SYNC_NOREFRESH) == 0; + + ret = maildir_uidlist_lock_timeout(uidlist, nonblock, refresh, refresh); + if (ret <= 0) { + if (ret < 0 || !nonblock) + return ret; + + /* couldn't lock it */ + if ((sync_flags & MAILDIR_UIDLIST_SYNC_FORCE) == 0) + return 0; + /* forcing the sync anyway */ + if (maildir_uidlist_refresh(uidlist) < 0) + return -1; + } else { + *locked_r = TRUE; + } + return 1; +} + +void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist) +{ + maildir_uidlist_mark_all(uidlist, TRUE); +} + +int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, + enum maildir_uidlist_sync_flags sync_flags, + struct maildir_uidlist_sync_ctx **sync_ctx_r) +{ + struct maildir_uidlist_sync_ctx *ctx; + bool locked; + int ret; + + ret = maildir_uidlist_sync_lock(uidlist, sync_flags, &locked); + if (ret <= 0) + return ret; + + *sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1); + ctx->uidlist = uidlist; + ctx->sync_flags = sync_flags; + ctx->partial = !locked || + (sync_flags & MAILDIR_UIDLIST_SYNC_PARTIAL) != 0; + ctx->locked = locked; + ctx->first_unwritten_pos = UINT_MAX; + ctx->first_new_pos = UINT_MAX; + + if (ctx->partial) { + if ((sync_flags & MAILDIR_UIDLIST_SYNC_KEEP_STATE) == 0) { + /* initially mark all nonsynced */ + maildir_uidlist_mark_all(uidlist, TRUE); + } + return 1; + } + i_assert(uidlist->locked_refresh); + + ctx->record_pool = pool_alloconly_create(MEMPOOL_GROWING + "maildir_uidlist_sync", 16384); + hash_table_create(&ctx->files, ctx->record_pool, 4096, + maildir_filename_base_hash, + maildir_filename_base_cmp); + + i_array_init(&ctx->records, array_count(&uidlist->records)); + return 1; +} + +static int +maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, uint32_t uid, + enum maildir_uidlist_rec_flag flags, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct maildir_uidlist_rec *rec, *const *recs; + unsigned int count; + + /* we'll update uidlist directly */ + rec = hash_table_lookup(uidlist->files, filename); + if (rec == NULL) { + /* doesn't exist in uidlist */ + if (!ctx->locked) { + /* we can't add it, so just ignore it */ + return 1; + } + if (ctx->first_new_pos == UINT_MAX) + ctx->first_new_pos = array_count(&uidlist->records); + ctx->new_files_count++; + ctx->changed = TRUE; + + if (uidlist->record_pool == NULL) { + uidlist->record_pool = + pool_alloconly_create(MEMPOOL_GROWING + "uidlist record_pool", + 1024); + } + + rec = p_new(uidlist->record_pool, + struct maildir_uidlist_rec, 1); + rec->uid = (uint32_t)-1; + rec->filename = p_strdup(uidlist->record_pool, filename); + array_push_back(&uidlist->records, &rec); + uidlist->change_counter++; + + hash_table_insert(uidlist->files, rec->filename, rec); + } else if (strcmp(rec->filename, filename) != 0) { + rec->filename = p_strdup(uidlist->record_pool, filename); + } + if (uid != 0) { + if (rec->uid != uid && rec->uid != (uint32_t)-1) { + mailbox_set_critical(uidlist->box, + "Maildir: %s changed UID %u -> %u", + filename, rec->uid, uid); + return -1; + } + rec->uid = uid; + if (uidlist->next_uid <= uid) + uidlist->next_uid = uid + 1; + else { + recs = array_get(&uidlist->records, &count); + if (count > 1 && uid < recs[count-1]->uid) + uidlist->unsorted = TRUE; + } + } + + rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR); + rec->flags = (rec->flags | flags) & + ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + + ctx->finished = FALSE; + *rec_r = rec; + return 1; +} + +static unsigned char *ext_dup(pool_t pool, const unsigned char *extensions) +{ + unsigned char *ret; + + if (extensions == NULL) + return NULL; + + T_BEGIN { + unsigned int len; + + for (len = 0; extensions[len] != '\0'; len++) { + while (extensions[len] != '\0') len++; + } + ret = p_malloc(pool, len + 1); + memcpy(ret, extensions, len); + } T_END; + return ret; +} + +int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, + enum maildir_uidlist_rec_flag flags) +{ + struct maildir_uidlist_rec *rec; + + return maildir_uidlist_sync_next_uid(ctx, filename, 0, flags, &rec); +} + +int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, uint32_t uid, + enum maildir_uidlist_rec_flag flags, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct maildir_uidlist_rec *rec, *old_rec; + const char *p; + + *rec_r = NULL; + + if (ctx->failed) + return -1; + for (p = filename; *p != '\0'; p++) { + if (*p == 13 || *p == 10) { + i_warning("Maildir %s: Ignoring a file with #0x%x: %s", + mailbox_get_path(uidlist->box), *p, filename); + return 1; + } + } + + if (ctx->partial) { + return maildir_uidlist_sync_next_partial(ctx, filename, + uid, flags, rec_r); + } + + rec = hash_table_lookup(ctx->files, filename); + if (rec != NULL) { + if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | + MAILDIR_UIDLIST_REC_FLAG_MOVED)) == 0) { + /* possibly duplicate */ + return 0; + } + + /* probably was in new/ and now we're seeing it in cur/. + remove new/moved flags so if this happens again we'll know + to check for duplicates. */ + rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | + MAILDIR_UIDLIST_REC_FLAG_MOVED); + if (strcmp(rec->filename, filename) != 0) + rec->filename = p_strdup(ctx->record_pool, filename); + } else { + old_rec = hash_table_lookup(uidlist->files, filename); + i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist)); + + rec = p_new(ctx->record_pool, struct maildir_uidlist_rec, 1); + + if (old_rec != NULL) { + *rec = *old_rec; + rec->extensions = + ext_dup(ctx->record_pool, rec->extensions); + } else { + rec->uid = (uint32_t)-1; + ctx->new_files_count++; + ctx->changed = TRUE; + /* didn't exist in uidlist, it's recent */ + flags |= MAILDIR_UIDLIST_REC_FLAG_RECENT; + } + rec->filename = p_strdup(ctx->record_pool, filename); + hash_table_insert(ctx->files, rec->filename, rec); + + array_push_back(&ctx->records, &rec); + } + if (uid != 0) { + rec->uid = uid; + if (uidlist->next_uid <= uid) + uidlist->next_uid = uid + 1; + } + + rec->flags = (rec->flags | flags) & ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + *rec_r = rec; + return 1; +} + +void maildir_uidlist_sync_remove(struct maildir_uidlist_sync_ctx *ctx, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + unsigned int idx; + + i_assert(ctx->partial); + i_assert(ctx->uidlist->locked_refresh); + + rec = hash_table_lookup(ctx->uidlist->files, filename); + i_assert(rec != NULL); + i_assert(rec->uid != (uint32_t)-1); + + hash_table_remove(ctx->uidlist->files, filename); + idx = maildir_uidlist_records_array_delete(ctx->uidlist, rec); + + if (ctx->first_unwritten_pos != UINT_MAX) { + i_assert(ctx->first_unwritten_pos > idx); + ctx->first_unwritten_pos--; + } + if (ctx->first_new_pos != UINT_MAX) { + i_assert(ctx->first_new_pos > idx); + ctx->first_new_pos--; + } + + ctx->changed = TRUE; + ctx->uidlist->recreate = TRUE; +} + +void maildir_uidlist_sync_set_ext(struct maildir_uidlist_sync_ctx *ctx, + struct maildir_uidlist_rec *rec, + enum maildir_uidlist_rec_ext_key key, + const char *value) +{ + pool_t pool = ctx->partial ? + ctx->uidlist->record_pool : ctx->record_pool; + + i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(key)); + + maildir_uidlist_rec_set_ext(rec, pool, key, value); +} + +const char * +maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(ctx->files, filename); + return rec == NULL ? NULL : rec->filename; +} + +bool maildir_uidlist_get_uid(struct maildir_uidlist *uidlist, + const char *filename, uint32_t *uid_r) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + if (rec == NULL) + return FALSE; + + *uid_r = rec->uid; + return TRUE; +} + +void maildir_uidlist_update_fname(struct maildir_uidlist *uidlist, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + if (rec == NULL) + return; + + rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + if (strcmp(rec->filename, filename) != 0) + rec->filename = p_strdup(uidlist->record_pool, filename); +} + +const char * +maildir_uidlist_get_full_filename(struct maildir_uidlist *uidlist, + const char *filename) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + return rec == NULL ? NULL : rec->filename; +} + +static int maildir_assign_uid_cmp(const void *p1, const void *p2) +{ + const struct maildir_uidlist_rec *const *rec1 = p1, *const *rec2 = p2; + + if ((*rec1)->uid != (*rec2)->uid) { + if ((*rec1)->uid < (*rec2)->uid) + return -1; + else + return 1; + } + return maildir_filename_sort_cmp((*rec1)->filename, (*rec2)->filename); +} + +static void maildir_uidlist_assign_uids(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist_rec **recs; + unsigned int dest, count; + + i_assert(UIDLIST_IS_LOCKED(ctx->uidlist)); + i_assert(ctx->first_new_pos != UINT_MAX); + + if (ctx->first_unwritten_pos == UINT_MAX) + ctx->first_unwritten_pos = ctx->first_new_pos; + + /* sort new files and assign UIDs for them */ + recs = array_get_modifiable(&ctx->uidlist->records, &count); + qsort(recs + ctx->first_new_pos, count - ctx->first_new_pos, + sizeof(*recs), maildir_assign_uid_cmp); + + for (dest = ctx->first_new_pos; dest < count; dest++) { + if (recs[dest]->uid == (uint32_t)-1) + break; + } + + for (; dest < count; dest++) { + i_assert(recs[dest]->uid == (uint32_t)-1); + i_assert(ctx->uidlist->next_uid < (uint32_t)-1); + recs[dest]->uid = ctx->uidlist->next_uid++; + recs[dest]->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_MOVED); + } + + if (ctx->uidlist->locked_refresh && ctx->uidlist->initial_read) + ctx->uidlist->last_seen_uid = ctx->uidlist->next_uid-1; + + ctx->new_files_count = 0; + ctx->first_new_pos = UINT_MAX; + ctx->uidlist->change_counter++; + ctx->finish_change_counter = ctx->uidlist->change_counter; +} + +static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + + /* buffer is unsorted, sort it by UID */ + array_sort(&ctx->records, maildir_uid_cmp); + + array_free(&uidlist->records); + uidlist->records = ctx->records; + ctx->records.arr.buffer = NULL; + i_assert(array_is_created(&uidlist->records)); + + hash_table_destroy(&uidlist->files); + uidlist->files = ctx->files; + i_zero(&ctx->files); + + pool_unref(&uidlist->record_pool); + uidlist->record_pool = ctx->record_pool; + ctx->record_pool = NULL; + + if (ctx->new_files_count != 0) { + ctx->first_new_pos = array_count(&uidlist->records) - + ctx->new_files_count; + maildir_uidlist_assign_uids(ctx); + } else { + ctx->uidlist->change_counter++; + } +} + +void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx) +{ + ctx->uidlist->recreate = TRUE; +} + +void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx) +{ + if (!ctx->partial) { + if (!ctx->failed) + maildir_uidlist_swap(ctx); + } else { + if (ctx->new_files_count != 0 && !ctx->failed) { + i_assert(ctx->changed); + i_assert(ctx->locked); + maildir_uidlist_assign_uids(ctx); + } + } + + ctx->finished = TRUE; + + /* mbox=NULL means we're coming from dbox rebuilding code. + the dbox is already locked, so allow uidlist recreation */ + i_assert(ctx->locked || !ctx->changed); + if ((ctx->changed || maildir_uidlist_want_compress(ctx)) && + !ctx->failed && ctx->locked) { + T_BEGIN { + if (maildir_uidlist_sync_update(ctx) < 0) { + /* we couldn't write everything we wanted. make + sure we don't continue using those UIDs */ + maildir_uidlist_reset(ctx->uidlist); + ctx->failed = TRUE; + } + } T_END; + } +} + +int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **_ctx, + bool success) +{ + struct maildir_uidlist_sync_ctx *ctx = *_ctx; + int ret; + + *_ctx = NULL; + + if (!success) + ctx->failed = TRUE; + ret = ctx->failed ? -1 : 0; + + if (!ctx->finished) + maildir_uidlist_sync_finish(ctx); + if (ctx->partial) + maildir_uidlist_mark_all(ctx->uidlist, FALSE); + if (ctx->locked) + maildir_uidlist_unlock(ctx->uidlist); + + hash_table_destroy(&ctx->files); + pool_unref(&ctx->record_pool); + if (array_is_created(&ctx->records)) + array_free(&ctx->records); + i_free(ctx); + return ret; +} + +void maildir_uidlist_add_flags(struct maildir_uidlist *uidlist, + const char *filename, + enum maildir_uidlist_rec_flag flags) +{ + struct maildir_uidlist_rec *rec; + + rec = hash_table_lookup(uidlist->files, filename); + i_assert(rec != NULL); + + rec->flags |= flags; +} + +struct maildir_uidlist_iter_ctx * +maildir_uidlist_iter_init(struct maildir_uidlist *uidlist) +{ + struct maildir_uidlist_iter_ctx *ctx; + unsigned int count; + + ctx = i_new(struct maildir_uidlist_iter_ctx, 1); + ctx->uidlist = uidlist; + ctx->next = array_get(&uidlist->records, &count); + ctx->end = ctx->next + count; + ctx->change_counter = ctx->uidlist->change_counter; + return ctx; +} + +static void +maildir_uidlist_iter_update_idx(struct maildir_uidlist_iter_ctx *ctx) +{ + unsigned int old_rev_idx, idx, count; + + old_rev_idx = ctx->end - ctx->next; + ctx->next = array_get(&ctx->uidlist->records, &count); + ctx->end = ctx->next + count; + + idx = old_rev_idx >= count ? 0 : + count - old_rev_idx; + while (idx < count && ctx->next[idx]->uid <= ctx->prev_uid) + idx++; + while (idx > 0 && ctx->next[idx-1]->uid > ctx->prev_uid) + idx--; + + ctx->next += idx; +} + +static bool maildir_uidlist_iter_next_rec(struct maildir_uidlist_iter_ctx *ctx, + struct maildir_uidlist_rec **rec_r) +{ + struct maildir_uidlist_rec *rec; + + if (ctx->change_counter != ctx->uidlist->change_counter) + maildir_uidlist_iter_update_idx(ctx); + + if (ctx->next == ctx->end) + return FALSE; + + rec = *ctx->next; + i_assert(rec->uid != (uint32_t)-1); + + ctx->prev_uid = rec->uid; + ctx->next++; + + *rec_r = rec; + return TRUE; +} + +bool maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, + uint32_t *uid_r, + enum maildir_uidlist_rec_flag *flags_r, + const char **filename_r) +{ + struct maildir_uidlist_rec *rec; + + if (!maildir_uidlist_iter_next_rec(ctx, &rec)) + return FALSE; + + *uid_r = rec->uid; + *flags_r = rec->flags; + *filename_r = rec->filename; + return TRUE; +} + +void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx **_ctx) +{ + i_free(*_ctx); + *_ctx = NULL; +} diff --git a/src/lib-storage/index/maildir/maildir-uidlist.h b/src/lib-storage/index/maildir/maildir-uidlist.h new file mode 100644 index 0000000..8b45723 --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-uidlist.h @@ -0,0 +1,161 @@ +#ifndef MAILDIR_UIDLIST_H +#define MAILDIR_UIDLIST_H + +#include "mail-storage.h" + +#define MAILDIR_UIDLIST_NAME "dovecot-uidlist" +/* how many seconds to wait before overriding uidlist.lock */ +#define MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT (60*2) + +struct maildir_mailbox; +struct maildir_uidlist; +struct maildir_uidlist_sync_ctx; +struct maildir_uidlist_rec; + +enum maildir_uidlist_sync_flags { + MAILDIR_UIDLIST_SYNC_PARTIAL = 0x01, + MAILDIR_UIDLIST_SYNC_KEEP_STATE = 0x02, + MAILDIR_UIDLIST_SYNC_FORCE = 0x04, + MAILDIR_UIDLIST_SYNC_TRYLOCK = 0x08, + MAILDIR_UIDLIST_SYNC_NOREFRESH = 0x10, + MAILDIR_UIDLIST_SYNC_NOLOCK = 0x20 +}; + +enum maildir_uidlist_rec_flag { + MAILDIR_UIDLIST_REC_FLAG_NEW_DIR = 0x01, + MAILDIR_UIDLIST_REC_FLAG_MOVED = 0x02, + MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04, + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED = 0x08, + MAILDIR_UIDLIST_REC_FLAG_RACING = 0x10 +}; + +enum maildir_uidlist_hdr_ext_key { + MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY = 'V', + MAILDIR_UIDLIST_HDR_EXT_NEXT_UID = 'N', + MAILDIR_UIDLIST_HDR_EXT_GUID = 'G', + /* POP3 UIDL format unless overridden by records */ + MAILDIR_UIDLIST_HDR_EXT_POP3_UIDL_FORMAT = 'P' +}; + +#define MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(c) \ + ((c) >= 'A' && (c) <= 'Z') +enum maildir_uidlist_rec_ext_key { + /* Physical message size. If filename also contains ,S=<vsize> this + isn't written to uidlist. */ + MAILDIR_UIDLIST_REC_EXT_PSIZE = 'S', + /* Virtual message size. If filename also contains ,W=<vsize> this + isn't written to uidlist. */ + MAILDIR_UIDLIST_REC_EXT_VSIZE = 'W', + /* POP3 UIDL overriding the default format */ + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL = 'P', + /* POP3 message ordering number. Lower numbered messages are listed + first. Messages without ordering number are listed after them. + The idea is to be able to preserve POP3 UIDL list and IMAP UIDs + perfectly when migrating from other servers. */ + MAILDIR_UIDLIST_REC_EXT_POP3_ORDER = 'O', + /* Message GUID (default is the base filename) */ + MAILDIR_UIDLIST_REC_EXT_GUID = 'G' +}; + +int maildir_uidlist_lock(struct maildir_uidlist *uidlist); +int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist); +int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist); +void maildir_uidlist_unlock(struct maildir_uidlist *uidlist); +bool maildir_uidlist_is_locked(struct maildir_uidlist *uidlist); +bool maildir_uidlist_is_read(struct maildir_uidlist *uidlist); +/* Returns TRUE if uidlist file is currently open */ +bool maildir_uidlist_is_open(struct maildir_uidlist *uidlist); + +struct maildir_uidlist *maildir_uidlist_init(struct maildir_mailbox *mbox); +void maildir_uidlist_deinit(struct maildir_uidlist **uidlist); + +/* Returns -1 if error, 0 if file is broken or lost, 1 if ok. If nfs_flush=TRUE + and storage has NFS_FLUSH flag set, the NFS attribute cache is flushed to + make sure that we see the latest uidlist file. */ +int maildir_uidlist_refresh(struct maildir_uidlist *uidlist); +/* Like maildir_uidlist_refresh(), but if uidlist isn't opened yet, try to + fill in the uidvalidity/nextuid from index file instead. */ +int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist); + +/* Look up uidlist record for given filename. Returns 1 if found, + 0 if not found, -1 if error */ +int maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_flag *flags_r, + const char **fname_r); +/* Returns extension's value or NULL if it doesn't exist. */ +const char * +maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key); + +uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist); +uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist); +int maildir_uidlist_get_mailbox_guid(struct maildir_uidlist *uidlist, + guid_128_t mailbox_guid); +void maildir_uidlist_set_mailbox_guid(struct maildir_uidlist *uidlist, + const guid_128_t mailbox_guid); + +void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist, + uint32_t uid_validity); +void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist, + uint32_t next_uid, bool force); + +/* Update extended record. */ +void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key, + const char *value); +void maildir_uidlist_unset_ext(struct maildir_uidlist *uidlist, uint32_t uid, + enum maildir_uidlist_rec_ext_key key); + +/* If uidlist has changed, update it. This is mostly meant to be used with + maildir_uidlist_set_ext() */ +int maildir_uidlist_update(struct maildir_uidlist *uidlist); + +void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist); +/* Sync uidlist with what's actually on maildir. Returns same as + maildir_uidlist_lock(). */ +int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, + enum maildir_uidlist_sync_flags sync_flags, + struct maildir_uidlist_sync_ctx **sync_ctx_r); +int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, + enum maildir_uidlist_rec_flag flags); +int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, uint32_t uid, + enum maildir_uidlist_rec_flag flags, + struct maildir_uidlist_rec **rec_r); +void maildir_uidlist_sync_remove(struct maildir_uidlist_sync_ctx *ctx, + const char *filename); +void maildir_uidlist_sync_set_ext(struct maildir_uidlist_sync_ctx *ctx, + struct maildir_uidlist_rec *rec, + enum maildir_uidlist_rec_ext_key key, + const char *value); +void maildir_uidlist_update_fname(struct maildir_uidlist *uidlist, + const char *filename); +const char * +maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx, + const char *filename); +void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx); +void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx); +int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **ctx, + bool success); + +bool maildir_uidlist_get_uid(struct maildir_uidlist *uidlist, + const char *filename, uint32_t *uid_r); +const char * +maildir_uidlist_get_full_filename(struct maildir_uidlist *uidlist, + const char *filename); + +void maildir_uidlist_add_flags(struct maildir_uidlist *uidlist, + const char *filename, + enum maildir_uidlist_rec_flag flags); + +/* List all maildir files. */ +struct maildir_uidlist_iter_ctx * +maildir_uidlist_iter_init(struct maildir_uidlist *uidlist); +bool maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, + uint32_t *uid_r, + enum maildir_uidlist_rec_flag *flags_r, + const char **filename_r); +void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx **ctx); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-util.c b/src/lib-storage/index/maildir/maildir-util.c new file mode 100644 index 0000000..c03546e --- /dev/null +++ b/src/lib-storage/index/maildir/maildir-util.c @@ -0,0 +1,323 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "str.h" +#include "mkdir-parents.h" +#include "mailbox-list-private.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename-flags.h" +#include "maildir-sync.h" +#include "mailbox-recent-flags.h" + +#include <stdio.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <utime.h> +#include <sys/stat.h> + +#define MAILDIR_RESYNC_RETRY_COUNT 10 + +static const char * +maildir_filename_guess(struct maildir_mailbox *mbox, uint32_t uid, + const char *fname, + enum maildir_uidlist_rec_flag *uidlist_flags, + bool *have_flags_r) + +{ + struct mail_index_view *view = mbox->flags_view; + struct maildir_keywords_sync_ctx *kw_ctx; + enum mail_flags flags; + ARRAY_TYPE(keyword_indexes) keywords; + const char *p; + uint32_t seq; + + if (view == NULL || !mail_index_lookup_seq(view, uid, &seq)) { + *have_flags_r = FALSE; + return fname; + } + + t_array_init(&keywords, 32); + mail_index_lookup_view_flags(view, seq, &flags, &keywords); + if (array_count(&keywords) == 0) { + *have_flags_r = (flags & MAIL_FLAGS_NONRECENT) != 0; + fname = maildir_filename_flags_set(fname, flags); + } else { + *have_flags_r = TRUE; + kw_ctx = maildir_keywords_sync_init_readonly(mbox->keywords, + mbox->box.index); + fname = maildir_filename_flags_kw_set(kw_ctx, fname, + flags, &keywords); + maildir_keywords_sync_deinit(&kw_ctx); + } + + if (*have_flags_r) { + /* don't even bother looking into new/ dir */ + *uidlist_flags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + } else if ((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0 && + ((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 || + mailbox_recent_flags_have_uid(&mbox->box, uid))) { + /* probably in new/ dir, drop ":2," from fname */ + *uidlist_flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + p = strrchr(fname, MAILDIR_INFO_SEP); + if (p != NULL) + fname = t_strdup_until(fname, p); + } + + return fname; +} + +static int maildir_file_do_try(struct maildir_mailbox *mbox, uint32_t uid, + maildir_file_do_func *callback, void *context) +{ + const char *path, *fname; + enum maildir_uidlist_rec_flag flags; + bool have_flags; + int ret; + + ret = maildir_sync_lookup(mbox, uid, &flags, &fname); + if (ret <= 0) + return ret == 0 ? -2 : -1; + + if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* let's see if we can guess the filename based on index */ + fname = maildir_filename_guess(mbox, uid, fname, + &flags, &have_flags); + } + /* make a copy, just in case callback refreshes uidlist and + the pointer becomes invalid. */ + fname = t_strdup(fname); + + ret = 0; + if ((flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) { + /* probably in new/ dir */ + path = t_strconcat(mailbox_get_path(&mbox->box), + "/new/", fname, NULL); + ret = callback(mbox, path, context); + } + if (ret == 0) { + path = t_strconcat(mailbox_get_path(&mbox->box), "/cur/", + fname, NULL); + ret = callback(mbox, path, context); + } + if (ret > 0 && (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* file was found. make sure we remember its latest name. */ + maildir_uidlist_update_fname(mbox->uidlist, fname); + } else if (ret == 0 && + (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) { + /* file wasn't found. mark this message nonsynced, so we can + retry the lookup by guessing the flags */ + maildir_uidlist_add_flags(mbox->uidlist, fname, + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); + } + return ret; +} + +static int do_racecheck(struct maildir_mailbox *mbox, const char *path, + void *context) +{ + const uint32_t *uidp = context; + struct stat st; + int ret; + + ret = lstat(path, &st); + if (ret == 0 && (st.st_mode & S_IFMT) == S_IFLNK) { + /* most likely a symlink pointing to a nonexistent file */ + mailbox_set_critical(&mbox->box, + "Maildir: Symlink destination doesn't exist for UID=%u: %s", *uidp, path); + return -2; + } else if (ret < 0 && errno != ENOENT) { + mailbox_set_critical(&mbox->box, "lstat(%s) failed: %m", path); + return -1; + } else { + /* success or ENOENT, either way we're done */ + mailbox_set_critical(&mbox->box, + "maildir_file_do(%s): Filename keeps changing for UID=%u", path, *uidp); + return -1; + } +} + +#undef maildir_file_do +int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid, + maildir_file_do_func *callback, void *context) +{ + int i, ret; + + T_BEGIN { + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; + if (ret == 0 && mbox->storage->set->maildir_very_dirty_syncs) T_BEGIN { + /* try guessing again with refreshed flags */ + if (maildir_sync_refresh_flags_view(mbox) == 0) + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; + for (i = 0; i < MAILDIR_RESYNC_RETRY_COUNT && ret == 0; i++) { + /* file is either renamed or deleted. sync the maildir and + see which one. if file appears to be renamed constantly, + don't try to open it more than 10 times. */ + if (maildir_storage_sync_force(mbox, uid) < 0) + return -1; + + T_BEGIN { + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; + } + + if (i == MAILDIR_RESYNC_RETRY_COUNT) T_BEGIN { + ret = maildir_file_do_try(mbox, uid, do_racecheck, &uid); + } T_END; + + return ret == -2 ? 0 : ret; +} + +static int maildir_create_path(struct mailbox *box, const char *path, + enum mailbox_list_path_type type, bool retry) +{ + const struct mailbox_permissions *perm = mailbox_get_permissions(box); + const char *p, *parent; + + if (mkdir_chgrp(path, perm->dir_create_mode, perm->file_create_gid, + perm->file_create_gid_origin) == 0) + return 0; + + switch (errno) { + case EEXIST: + return 0; + case ENOENT: + p = strrchr(path, '/'); + if (type == MAILBOX_LIST_PATH_TYPE_MAILBOX || + p == NULL || !retry) { + /* mailbox was being deleted just now */ + mailbox_set_deleted(box); + return -1; + } + /* create index/control root directory */ + parent = t_strdup_until(path, p); + if (mailbox_list_mkdir_root(box->list, parent, type) < 0) { + mail_storage_copy_list_error(box->storage, box->list); + return -1; + } + /* should work now, try again */ + return maildir_create_path(box, path, type, FALSE); + default: + mailbox_set_critical(box, "mkdir(%s) failed: %m", path); + return -1; + } +} + +static int maildir_create_subdirs(struct mailbox *box) +{ + static const char *subdirs[] = { "cur", "new", "tmp" }; + const char *dirs[N_ELEMENTS(subdirs) + 2]; + enum mailbox_list_path_type types[N_ELEMENTS(subdirs) + 2]; + struct stat st; + const char *path; + unsigned int i, count; + + /* @UNSAFE: get a list of directories we want to create */ + for (i = 0; i < N_ELEMENTS(subdirs); i++) { + types[i] = MAILBOX_LIST_PATH_TYPE_MAILBOX; + dirs[i] = t_strconcat(mailbox_get_path(box), + "/", subdirs[i], NULL); + } + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, &path) > 0) { + types[i] = MAILBOX_LIST_PATH_TYPE_CONTROL; + dirs[i++] = path; + } + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) > 0) { + types[i] = MAILBOX_LIST_PATH_TYPE_INDEX; + dirs[i++] = path; + } + count = i; + i_assert(count <= N_ELEMENTS(dirs)); + + for (i = 0; i < count; i++) { + path = dirs[i]; + if (stat(path, &st) == 0) + continue; + if (errno != ENOENT) { + mailbox_set_critical(box, "stat(%s) failed: %m", path); + break; + } + if (maildir_create_path(box, path, types[i], TRUE) < 0) + break; + } + return i == N_ELEMENTS(dirs) ? 0 : -1; +} + +bool maildir_set_deleted(struct mailbox *box) +{ + struct stat st; + int ret; + + if (stat(mailbox_get_path(box), &st) < 0) { + if (errno == ENOENT) + mailbox_set_deleted(box); + else { + mailbox_set_critical(box, + "stat(%s) failed: %m", mailbox_get_path(box)); + } + return FALSE; + } + /* maildir itself exists. create all of its subdirectories in case + they got lost. */ + T_BEGIN { + ret = maildir_create_subdirs(box); + } T_END; + return ret < 0 ? FALSE : TRUE; +} + +int maildir_lose_unexpected_dir(struct mail_storage *storage, const char *path) +{ + const char *dest, *fname, *p; + + /* There's a directory in maildir, get rid of it. + + In some installations this was caused by a messed up configuration + where e.g. mails was initially delivered to new/new/ directory. + Also Dovecot v2.0.0 - v2.0.4 sometimes may have renamed tmp/ + directory under new/ or cur/. */ + if (rmdir(path) == 0) { + mail_storage_set_critical(storage, + "Maildir: rmdir()ed unwanted empty directory: %s", + path); + return 1; + } else if (errno == ENOENT) { + /* someone else rmdired or renamed it */ + return 0; + } else if (errno != ENOTEMPTY) { + mail_storage_set_critical(storage, + "Maildir: Found unwanted directory %s, " + "but rmdir() failed: %m", path); + return -1; + } + + /* It's not safe to delete this directory since it has some files in it, + but it's also not helpful to log this message over and over again. + Get rid of this error by renaming the directory elsewhere */ + p = strrchr(path, '/'); + i_assert(p != NULL); + fname = p + 1; + while (p != path && p[-1] != '/') p--; + i_assert(p != NULL); + + dest = t_strconcat(t_strdup_until(path, p), "extra-", fname, NULL); + if (rename(path, dest) == 0) { + mail_storage_set_critical(storage, + "Maildir: renamed unwanted directory %s to %s", + path, dest); + return 1; + } else if (errno == ENOENT) { + /* someone else renamed it (could have been flag change) */ + return 0; + } else { + mail_storage_set_critical(storage, + "Maildir: Found unwanted directory, " + "but rename(%s, %s) failed: %m", path, dest); + return -1; + } +} |