diff options
Diffstat (limited to 'src/lib-storage/index/dbox-multi')
19 files changed, 6887 insertions, 0 deletions
diff --git a/src/lib-storage/index/dbox-multi/Makefile.am b/src/lib-storage/index/dbox-multi/Makefile.am new file mode 100644 index 0000000..810edd2 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/Makefile.am @@ -0,0 +1,36 @@ +noinst_LTLIBRARIES = libstorage_dbox_multi.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_multi_la_SOURCES = \ + mdbox-deleted-storage.c \ + mdbox-file.c \ + mdbox-mail.c \ + mdbox-map.c \ + mdbox-purge.c \ + mdbox-save.c \ + mdbox-settings.c \ + mdbox-sync.c \ + mdbox-storage.c \ + mdbox-storage-rebuild.c + +headers = \ + mdbox-file.h \ + mdbox-map.h \ + mdbox-map-private.h \ + mdbox-settings.h \ + mdbox-storage.h \ + mdbox-storage-rebuild.h \ + mdbox-sync.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff --git a/src/lib-storage/index/dbox-multi/Makefile.in b/src/lib-storage/index/dbox-multi/Makefile.in new file mode 100644 index 0000000..7e87b89 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/Makefile.in @@ -0,0 +1,866 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/lib-storage/index/dbox-multi +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libstorage_dbox_multi_la_LIBADD = +am_libstorage_dbox_multi_la_OBJECTS = mdbox-deleted-storage.lo \ + mdbox-file.lo mdbox-mail.lo mdbox-map.lo mdbox-purge.lo \ + mdbox-save.lo mdbox-settings.lo mdbox-sync.lo mdbox-storage.lo \ + mdbox-storage-rebuild.lo +libstorage_dbox_multi_la_OBJECTS = \ + $(am_libstorage_dbox_multi_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/mdbox-deleted-storage.Plo \ + ./$(DEPDIR)/mdbox-file.Plo ./$(DEPDIR)/mdbox-mail.Plo \ + ./$(DEPDIR)/mdbox-map.Plo ./$(DEPDIR)/mdbox-purge.Plo \ + ./$(DEPDIR)/mdbox-save.Plo ./$(DEPDIR)/mdbox-settings.Plo \ + ./$(DEPDIR)/mdbox-storage-rebuild.Plo \ + ./$(DEPDIR)/mdbox-storage.Plo ./$(DEPDIR)/mdbox-sync.Plo +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libstorage_dbox_multi_la_SOURCES) +DIST_SOURCES = $(libstorage_dbox_multi_la_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libstorage_dbox_multi.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-storage/index \ + -I$(top_srcdir)/src/lib-storage/index/dbox-common + +libstorage_dbox_multi_la_SOURCES = \ + mdbox-deleted-storage.c \ + mdbox-file.c \ + mdbox-mail.c \ + mdbox-map.c \ + mdbox-purge.c \ + mdbox-save.c \ + mdbox-settings.c \ + mdbox-sync.c \ + mdbox-storage.c \ + mdbox-storage-rebuild.c + +headers = \ + mdbox-file.h \ + mdbox-map.h \ + mdbox-map-private.h \ + mdbox-settings.h \ + mdbox-storage.h \ + mdbox-storage-rebuild.h \ + mdbox-sync.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/dbox-multi/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-storage/index/dbox-multi/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libstorage_dbox_multi.la: $(libstorage_dbox_multi_la_OBJECTS) $(libstorage_dbox_multi_la_DEPENDENCIES) $(EXTRA_libstorage_dbox_multi_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libstorage_dbox_multi_la_OBJECTS) $(libstorage_dbox_multi_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-deleted-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-map.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-purge.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-save.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-settings.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-storage-rebuild.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-storage.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-sync.Plo@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/mdbox-deleted-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-file.Plo + -rm -f ./$(DEPDIR)/mdbox-mail.Plo + -rm -f ./$(DEPDIR)/mdbox-map.Plo + -rm -f ./$(DEPDIR)/mdbox-purge.Plo + -rm -f ./$(DEPDIR)/mdbox-save.Plo + -rm -f ./$(DEPDIR)/mdbox-settings.Plo + -rm -f ./$(DEPDIR)/mdbox-storage-rebuild.Plo + -rm -f ./$(DEPDIR)/mdbox-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-sync.Plo + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/mdbox-deleted-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-file.Plo + -rm -f ./$(DEPDIR)/mdbox-mail.Plo + -rm -f ./$(DEPDIR)/mdbox-map.Plo + -rm -f ./$(DEPDIR)/mdbox-purge.Plo + -rm -f ./$(DEPDIR)/mdbox-save.Plo + -rm -f ./$(DEPDIR)/mdbox-settings.Plo + -rm -f ./$(DEPDIR)/mdbox-storage-rebuild.Plo + -rm -f ./$(DEPDIR)/mdbox-storage.Plo + -rm -f ./$(DEPDIR)/mdbox-sync.Plo + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-libtool clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-pkginc_libHEADERS install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c new file mode 100644 index 0000000..4fdf1ab --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c @@ -0,0 +1,319 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "master-service.h" +#include "mail-index-modseq.h" +#include "mail-index-alloc-cache.h" +#include "mailbox-log.h" +#include "mailbox-list-private.h" +#include "mail-copy.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-storage.h" + +extern struct mail_storage mdbox_deleted_storage; +extern struct mailbox mdbox_deleted_mailbox; +extern struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs; + +static struct mail_storage *mdbox_deleted_storage_alloc(void) +{ + struct mdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mdbox deleted storage", 2048); + storage = p_new(pool, struct mdbox_storage, 1); + storage->storage.v = mdbox_dbox_storage_vfuncs; + storage->storage.storage = mdbox_deleted_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +static struct mailbox * +mdbox_deleted_mailbox_alloc(struct mail_storage *storage, + struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mdbox_mailbox *mbox; + pool_t pool; + + flags |= MAILBOX_FLAG_READONLY | MAILBOX_FLAG_NO_INDEX_FILES; + + pool = pool_alloconly_create("mdbox deleted mailbox", 1024*3); + mbox = p_new(pool, struct mdbox_mailbox, 1); + mbox->box = mdbox_deleted_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mdbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = MDBOX_STORAGE(storage); + return &mbox->box; +} + +static int +mdbox_deleted_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + struct mail_index_transaction *new_trans = NULL; + uint32_t uid_validity = ioloop_time; + uint32_t uid_next = 1; + + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + + if (trans == NULL) { + new_trans = mail_index_transaction_begin(box->view, 0); + trans = new_trans; + } + + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + mbox->creating = TRUE; + mdbox_update_header(mbox, trans, update); + mbox->creating = FALSE; + + if (new_trans != NULL) { + if (mail_index_transaction_commit(&new_trans) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + return 0; +} + +static const char * +mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) +{ + return ""; +} + +static int +mdbox_deleted_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + + if ((items & MAILBOX_METADATA_GUID) != 0) + guid_128_generate(metadata_r->guid); + return 0; +} + +static struct mail_save_context * +mdbox_deleted_save_alloc(struct mailbox_transaction_context *t) +{ + struct mail_save_context *ctx; + + ctx = i_new(struct mail_save_context, 1); + ctx->transaction = t; + return ctx; +} + +static int +mdbox_deleted_save_begin(struct mail_save_context *ctx, + struct istream *input ATTR_UNUSED) +{ + mail_storage_set_error(ctx->transaction->box->storage, + MAIL_ERROR_NOTPOSSIBLE, "mdbox_deleted doesn't support saving mails"); + return -1; +} + +static int +mdbox_deleted_save_continue(struct mail_save_context *ctx ATTR_UNUSED) +{ + return -1; +} + +static int mdbox_deleted_save_finish(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); + return -1; +} + +static void +mdbox_deleted_save_cancel(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); +} + +static int mdbox_deleted_sync(struct mdbox_mailbox *mbox, + enum mdbox_sync_flags flags ATTR_UNUSED) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mdbox_mail_index_record rec; + struct mdbox_map_mail_index_record map_rec; + enum mail_index_sync_flags sync_flags; + uint16_t refcount; + uint32_t map_seq, map_count, seq, uid = 0; + int ret = 0; + + if (mbox->mdbox_deleted_synced) { + /* don't bother supporting incremental syncs */ + return 0; + } + if (!mbox->box.inbox_user && mbox->box.name[0] != '\0') { + /* since mailbox list currently shows all the existing + mailboxes, we don't want all of them to list the deleted + messages. only show messages in user's INBOX or the + namespace prefix. */ + return 0; + } + + if (mdbox_map_open(mbox->storage->map) < 0) + return -1; + + if (mdbox_deleted_mailbox_create_indexes(&mbox->box, NULL, NULL) < 0) + return -1; + + i_zero(&rec); + rec.save_date = ioloop_time; + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if (mail_index_sync_begin(mbox->box.index, &index_sync_ctx, + &sync_view, &trans, sync_flags) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + map_count = mdbox_map_get_messages_count(mbox->storage->map); + for (map_seq = 1; map_seq <= map_count; map_seq++) { + if (mdbox_map_lookup_seq_full(mbox->storage->map, map_seq, + &map_rec, &refcount) < 0) { + ret = -1; + break; + } + if (refcount == 0) { + rec.map_uid = mdbox_map_lookup_uid(mbox->storage->map, + map_seq); + mail_index_append(trans, ++uid, &seq); + mail_index_update_ext(trans, seq, + mbox->ext_id, &rec, NULL); + } + } + + if (ret < 0) + mail_index_sync_rollback(&index_sync_ctx); + else { + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } else { + mbox->mdbox_deleted_synced = TRUE; + } + } + return ret; +} + +static struct mailbox_sync_context * +mdbox_deleted_storage_sync_init(struct mailbox *box, + enum mailbox_sync_flags flags) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + enum mdbox_sync_flags mdbox_sync_flags = 0; + int ret = 0; + + if (index_mailbox_want_full_sync(&mbox->box, flags) || + mbox->storage->corrupted) + ret = mdbox_deleted_sync(mbox, mdbox_sync_flags); + + return index_mailbox_sync_init(box, flags, ret < 0); +} + +struct mail_storage mdbox_deleted_storage = { + .name = MDBOX_DELETED_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + + .v = { + mdbox_get_setting_parser_info, + mdbox_deleted_storage_alloc, + mdbox_storage_create, + mdbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + NULL, + mdbox_deleted_mailbox_alloc, + NULL, + NULL, + } +}; + +struct mailbox mdbox_deleted_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mdbox_mailbox_open, + index_storage_mailbox_close, + index_storage_mailbox_free, + dbox_mailbox_create, + index_storage_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mdbox_deleted_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + mdbox_deleted_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + dbox_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + mdbox_deleted_save_alloc, + mdbox_deleted_save_begin, + mdbox_deleted_save_continue, + mdbox_deleted_save_finish, + mdbox_deleted_save_cancel, + mail_storage_copy, + NULL, + NULL, + NULL, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs = { + mdbox_file_unrefed, + mdbox_file_create_fd, + mdbox_mail_open, + mdbox_deleted_mailbox_create_indexes, + mdbox_get_attachment_path_suffix, + mdbox_set_mailbox_corrupted, + mdbox_set_file_corrupted +}; diff --git a/src/lib-storage/index/dbox-multi/mdbox-file.c b/src/lib-storage/index/dbox-multi/mdbox-file.c new file mode 100644 index 0000000..65138bd --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-file.c @@ -0,0 +1,349 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "hex-dec.h" +#include "hex-binary.h" +#include "hostpid.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "file-set-size.h" +#include "mkdir-parents.h" +#include "fdatasync-path.h" +#include "eacces-error.h" +#include "str.h" +#include "mailbox-list-private.h" +#include "mdbox-storage.h" +#include "mdbox-map-private.h" +#include "mdbox-file.h" + +#include <stdio.h> +#include <unistd.h> +#include <ctype.h> +#include <fcntl.h> + +static struct mdbox_file * +mdbox_find_and_move_open_file(struct mdbox_storage *storage, uint32_t file_id) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) { + if (files[i]->file_id == file_id) + return files[i]; + } + return NULL; +} + +void mdbox_files_free(struct mdbox_storage *storage) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) + dbox_file_free(&files[i]->file); + array_clear(&storage->open_files); +} + +void mdbox_files_sync_input(struct mdbox_storage *storage) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count; i++) { + if (files[i]->file.input != NULL) + i_stream_sync(files[i]->file.input); + } +} + +static void +mdbox_close_open_files(struct mdbox_storage *storage, unsigned int close_count) +{ + struct mdbox_file *const *files; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + for (i = 0; i < count;) { + if (files[i]->file.refcount == 0) { + dbox_file_free(&files[i]->file); + array_delete(&storage->open_files, i, 1); + + if (--close_count == 0) + break; + + files = array_get(&storage->open_files, &count); + } else { + i++; + } + } +} + +static void +mdbox_file_init_paths(struct mdbox_file *file, const char *fname, bool alt) +{ + i_free(file->file.primary_path); + i_free(file->file.alt_path); + file->file.primary_path = + i_strdup_printf("%s/%s", file->storage->storage_dir, fname); + if (file->storage->alt_storage_dir != NULL) { + file->file.alt_path = + i_strdup_printf("%s/%s", file->storage->alt_storage_dir, + fname); + } + file->file.cur_path = !alt ? file->file.primary_path : + file->file.alt_path; +} + +static int mdbox_file_create(struct mdbox_file *file) +{ + struct dbox_file *_file = &file->file; + bool create_parents; + int ret; + + create_parents = dbox_file_is_in_alt(_file); + _file->fd = _file->storage->v. + file_create_fd(_file, _file->cur_path, create_parents); + if (_file->fd == -1) + return -1; + + if (file->storage->preallocate_space) { + ret = file_preallocate(_file->fd, + file->storage->set->mdbox_rotate_size); + if (ret < 0) { + switch (errno) { + case ENOSPC: + case EDQUOT: + /* ignore */ + break; + default: + i_error("file_preallocate(%s) failed: %m", + _file->cur_path); + break; + } + } else if (ret == 0) { + /* not supported by filesystem, disable. */ + file->storage->preallocate_space = FALSE; + } + } + return 0; +} + +static struct dbox_file * +mdbox_file_init_full(struct mdbox_storage *storage, + uint32_t file_id, bool alt_dir) +{ + struct mdbox_file *file; + const char *fname; + unsigned int count; + + file = file_id == 0 ? NULL : + mdbox_find_and_move_open_file(storage, file_id); + if (file != NULL) { + file->file.refcount++; + return &file->file; + } + + count = array_count(&storage->open_files); + if (count > MDBOX_MAX_OPEN_UNUSED_FILES) { + mdbox_close_open_files(storage, + count - MDBOX_MAX_OPEN_UNUSED_FILES); + } + + file = i_new(struct mdbox_file, 1); + file->storage = storage; + file->file.storage = &storage->storage; + file->file_id = file_id; + fname = file_id == 0 ? dbox_generate_tmp_filename() : + t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id); + mdbox_file_init_paths(file, fname, FALSE); + dbox_file_init(&file->file); + if (alt_dir) + file->file.cur_path = file->file.alt_path; + + if (file_id != 0) + array_push_back(&storage->open_files, &file); + else + (void)mdbox_file_create(file); + return &file->file; +} + +struct dbox_file * +mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id) +{ + return mdbox_file_init_full(storage, file_id, FALSE); +} + +struct dbox_file * +mdbox_file_init_new_alt(struct mdbox_storage *storage) +{ + return mdbox_file_init_full(storage, 0, TRUE); +} + +int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id) +{ + struct stat st; + const char *old_path; + const char *new_dir, *new_fname, *new_path; + + i_assert(file->file_id == 0); + i_assert(file_id != 0); + + old_path = file->file.cur_path; + new_fname = t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id); + new_dir = !dbox_file_is_in_alt(&file->file) ? + file->storage->storage_dir : file->storage->alt_storage_dir; + new_path = t_strdup_printf("%s/%s", new_dir, new_fname); + + if (stat(new_path, &st) == 0) { + mail_storage_set_critical(&file->file.storage->storage, + "mdbox: %s already exists, rebuilding index", new_path); + mdbox_storage_set_corrupted(file->storage); + return -1; + } + if (rename(old_path, new_path) < 0) { + mail_storage_set_critical(&file->storage->storage.storage, + "rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + mdbox_file_init_paths(file, new_fname, + dbox_file_is_in_alt(&file->file)); + file->file_id = file_id; + array_push_back(&file->storage->open_files, &file); + return 0; +} + +static struct mdbox_file * +mdbox_find_oldest_unused_file(struct mdbox_storage *storage, + unsigned int *idx_r) +{ + struct mdbox_file *const *files, *oldest_file = NULL; + unsigned int i, count; + + files = array_get(&storage->open_files, &count); + *idx_r = count; + for (i = 0; i < count; i++) { + if (files[i]->file.refcount == 0) { + if (oldest_file == NULL || + files[i]->close_time < oldest_file->close_time) { + oldest_file = files[i]; + *idx_r = i; + } + } + } + return oldest_file; +} + +static void mdbox_file_close_timeout(struct mdbox_storage *storage) +{ + struct mdbox_file *oldest; + unsigned int i; + time_t close_time = ioloop_time - MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS; + + while ((oldest = mdbox_find_oldest_unused_file(storage, &i)) != NULL) { + if (oldest->close_time > close_time) + break; + array_delete(&storage->open_files, i, 1); + dbox_file_free(&oldest->file); + } + + if (oldest == NULL) + timeout_remove(&storage->to_close_unused_files); +} + +static void mdbox_file_close_later(struct mdbox_file *mfile) +{ + if (mfile->storage->to_close_unused_files == NULL) { + mfile->storage->to_close_unused_files = + timeout_add(MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS*1000, + mdbox_file_close_timeout, mfile->storage); + } +} + +void mdbox_file_unrefed(struct dbox_file *file) +{ + struct mdbox_file *mfile = (struct mdbox_file *)file; + struct mdbox_file *oldest_file; + unsigned int i, count; + + /* don't cache metadata seeks while file isn't being referenced */ + file->metadata_read_offset = UOFF_T_MAX; + mfile->close_time = ioloop_time; + + if (mfile->file_id != 0) { + count = array_count(&mfile->storage->open_files); + if (count <= MDBOX_MAX_OPEN_UNUSED_FILES) { + /* we can leave this file open for now */ + mdbox_file_close_later(mfile); + return; + } + + /* close the oldest file with refcount=0 */ + oldest_file = mdbox_find_oldest_unused_file(mfile->storage, &i); + i_assert(oldest_file != NULL); + array_delete(&mfile->storage->open_files, i, 1); + if (oldest_file != mfile) { + dbox_file_free(&oldest_file->file); + mdbox_file_close_later(mfile); + return; + } + /* have to close ourself */ + } + dbox_file_free(file); +} + +int mdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents) +{ + struct mdbox_file *mfile = (struct mdbox_file *)file; + struct mdbox_map *map = mfile->storage->map; + struct mailbox_permissions perm; + mode_t old_mask; + const char *p, *dir; + int fd; + + mailbox_list_get_root_permissions(map->root_list, &perm); + + old_mask = umask(0666 & ~perm.file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + if (fd == -1 && errno == ENOENT && parents && + (p = strrchr(path, '/')) != NULL) { + dir = t_strdup_until(path, p); + if (mailbox_list_mkdir_root(map->root_list, dir, + path != file->alt_path ? + MAILBOX_LIST_PATH_TYPE_DIR : + MAILBOX_LIST_PATH_TYPE_ALT_DIR) < 0) { + mail_storage_copy_list_error(&file->storage->storage, + map->root_list); + return -1; + } + /* try again */ + old_mask = umask(0666 & ~perm.file_create_mode); + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666); + umask(old_mask); + } + if (fd == -1) { + mail_storage_set_critical(&file->storage->storage, + "open(%s, O_CREAT) failed: %m", path); + } else if (perm.file_create_gid == (gid_t)-1) { + /* no group change */ + } else if (fchown(fd, (uid_t)-1, perm.file_create_gid) < 0) { + if (errno == EPERM) { + mail_storage_set_critical(&file->storage->storage, "%s", + eperm_error_get_chgrp("fchown", path, + perm.file_create_gid, + perm.file_create_gid_origin)); + } else { + mail_storage_set_critical(&file->storage->storage, + "fchown(%s, -1, %ld) failed: %m", + path, (long)perm.file_create_gid); + } + /* continue anyway */ + } + return fd; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-file.h b/src/lib-storage/index/dbox-multi/mdbox-file.h new file mode 100644 index 0000000..9095cce --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-file.h @@ -0,0 +1,29 @@ +#ifndef MDBOX_FILE_H +#define MDBOX_FILE_H + +#include "dbox-file.h" + +struct mdbox_file { + struct dbox_file file; + struct mdbox_storage *storage; + + uint32_t file_id; + time_t close_time; +}; + +struct dbox_file * +mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id); +struct dbox_file * +mdbox_file_init_new_alt(struct mdbox_storage *storage); + +/* Assign file ID for a newly created file. */ +int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id); + +void mdbox_file_unrefed(struct dbox_file *file); +int mdbox_file_create_fd(struct dbox_file *file, const char *path, + bool parents); + +void mdbox_files_free(struct mdbox_storage *storage); +void mdbox_files_sync_input(struct mdbox_storage *storage); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-mail.c b/src/lib-storage/index/dbox-multi/mdbox-mail.c new file mode 100644 index 0000000..ed3fe06 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c @@ -0,0 +1,265 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "str.h" +#include "index-mail.h" +#include "dbox-mail.h" +#include "mdbox-storage.h" +#include "mdbox-sync.h" +#include "mdbox-map.h" +#include "mdbox-file.h" + +#include <sys/stat.h> + +int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, uint32_t *map_uid_r) +{ + const struct mdbox_mail_index_record *dbox_rec; + struct mdbox_index_header hdr; + const void *data; + uint32_t uid, cur_map_uid_validity; + bool need_resize; + + mail_index_lookup_ext(view, seq, mbox->ext_id, &data, NULL); + dbox_rec = data; + if (dbox_rec == NULL || dbox_rec->map_uid == 0) { + mail_index_lookup_uid(view, seq, &uid); + mailbox_set_critical(&mbox->box, + "mdbox: map uid lost for uid %u", uid); + mdbox_storage_set_corrupted(mbox->storage); + return -1; + } + + if (mbox->map_uid_validity == 0) { + if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) + return -1; + mbox->map_uid_validity = hdr.map_uid_validity; + } + if (mdbox_map_open_or_create(mbox->storage->map) < 0) + return -1; + + cur_map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map); + if (cur_map_uid_validity != mbox->map_uid_validity) { + mailbox_set_critical(&mbox->box, + "mdbox: map uidvalidity mismatch (%u vs %u)", + mbox->map_uid_validity, cur_map_uid_validity); + mdbox_storage_set_corrupted(mbox->storage); + return -1; + } + *map_uid_r = dbox_rec->map_uid; + return 0; +} + +static void dbox_mail_set_expunged(struct dbox_mail *mail, uint32_t map_uid) +{ + struct mail *_mail = &mail->imail.mail.mail; + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->box); + + mail_index_refresh(_mail->box->index); + if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) { + mail_set_expunged(_mail); + return; + } + + mdbox_map_set_corrupted(mbox->storage->map, + "Unexpectedly lost %s uid=%u map_uid=%u", + mailbox_get_vname(_mail->box), + _mail->uid, map_uid); +} + +static int dbox_mail_open_init(struct dbox_mail *mail, uint32_t map_uid) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->imail.mail.mail.box); + uint32_t file_id; + int ret; + + if ((ret = mdbox_map_lookup(mbox->storage->map, map_uid, + &file_id, &mail->offset)) <= 0) { + if (ret < 0) + return -1; + + /* map_uid doesn't exist anymore. either it + got just expunged or the map index is + corrupted. */ + dbox_mail_set_expunged(mail, map_uid); + return -1; + } else { + mail->open_file = mdbox_file_init(mbox->storage, file_id); + } + return 0; +} + +int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r) +{ + struct mail *_mail = &mail->imail.mail.mail; + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->box); + uint32_t prev_file_id = 0, map_uid = 0; + bool deleted; + + if (!mail_stream_access_start(_mail)) + return -1; + + do { + if (mail->open_file != NULL) { + /* already open */ + } else if (!_mail->saving) { + if (mdbox_mail_lookup(mbox, _mail->transaction->view, + _mail->seq, &map_uid) < 0) + return -1; + if (dbox_mail_open_init(mail, map_uid) < 0) + return -1; + } else { + /* mail is being saved in this transaction */ + mail->open_file = + mdbox_save_file_get_file(_mail->transaction, + _mail->seq, + &mail->offset); + } + + if (!dbox_file_is_open(mail->open_file)) + _mail->transaction->stats.open_lookup_count++; + if (dbox_file_open(mail->open_file, &deleted) <= 0) + return -1; + if (deleted) { + /* either it's expunged now or moved to another file. */ + struct mdbox_file *mfile = + (struct mdbox_file *)mail->open_file; + + if (mfile->file_id == prev_file_id) { + dbox_mail_set_expunged(mail, map_uid); + return -1; + } + prev_file_id = mfile->file_id; + if (mdbox_map_refresh(mbox->storage->map) < 0) + return -1; + dbox_file_unref(&mail->open_file); + } + } while (mail->open_file == NULL); + + *file_r = mail->open_file; + *offset_r = mail->offset; + return 0; +} + +static int mdbox_mail_get_save_date(struct mail *mail, time_t *date_r) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->transaction->box); + const struct mdbox_mail_index_record *dbox_rec; + const void *data; + + mail_index_lookup_ext(mail->transaction->view, mail->seq, + mbox->ext_id, &data, NULL); + dbox_rec = data; + if (dbox_rec == NULL || dbox_rec->map_uid == 0) { + /* lost for some reason, use fallback */ + return dbox_mail_get_save_date(mail, date_r); + } + + *date_r = dbox_rec->save_date; + return 1; +} + +static int +mdbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct dbox_mail *mail = DBOX_MAIL(_mail); + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->transaction->box); + struct mdbox_map_mail_index_record rec; + uint32_t map_uid; + uint16_t refcount; + + switch (field) { + case MAIL_FETCH_REFCOUNT: + if (mdbox_mail_lookup(mbox, _mail->transaction->view, + _mail->seq, &map_uid) < 0) + return -1; + if (mdbox_map_lookup_full(mbox->storage->map, map_uid, + &rec, &refcount) < 0) + return -1; + *value_r = p_strdup_printf(mail->imail.mail.data_pool, "%u", + refcount); + return 0; + case MAIL_FETCH_REFCOUNT_ID: + if (mdbox_mail_lookup(mbox, _mail->transaction->view, + _mail->seq, &map_uid) < 0) + return -1; + *value_r = p_strdup_printf(mail->imail.mail.data_pool, "%u", + map_uid); + return 0; + case MAIL_FETCH_UIDL_BACKEND: + if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id, + offsetof(struct mdbox_index_header, flags), + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) { + *value_r = ""; + return 0; + } + break; + case MAIL_FETCH_POP3_ORDER: + if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id, + offsetof(struct mdbox_index_header, flags), + DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) { + *value_r = ""; + return 0; + } + break; + default: + break; + } + return dbox_mail_get_special(_mail, field, value_r); +} + +static void +mdbox_mail_update_flags(struct mail *mail, enum modify_type modify_type, + enum mail_flags flags) +{ + if ((flags & DBOX_INDEX_FLAG_ALT) != 0) { + mdbox_purge_alt_flag_change(mail, modify_type != MODIFY_REMOVE); + flags &= ENUM_NEGATE(DBOX_INDEX_FLAG_ALT); + if (flags == 0 && modify_type != MODIFY_REPLACE) + return; + } + + index_mail_update_flags(mail, modify_type, flags); +} + +struct mail_vfuncs mdbox_mail_vfuncs = { + dbox_mail_close, + index_mail_free, + index_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + index_mail_prefetch, + index_mail_precache, + index_mail_add_temp_wanted_fields, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_pvt_modseq, + index_mail_get_parts, + index_mail_get_date, + dbox_mail_get_received_date, + mdbox_mail_get_save_date, + dbox_mail_get_virtual_size, + dbox_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + dbox_mail_get_stream, + index_mail_get_binary_stream, + mdbox_mail_get_special, + index_mail_get_backend_mail, + mdbox_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + index_mail_update_pvt_modseq, + NULL, + index_mail_expunge, + index_mail_set_cache_corrupted, + index_mail_opened, +}; diff --git a/src/lib-storage/index/dbox-multi/mdbox-map-private.h b/src/lib-storage/index/dbox-multi/mdbox-map-private.h new file mode 100644 index 0000000..259d854 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-map-private.h @@ -0,0 +1,64 @@ +#ifndef MDBOX_MAP_PRIVATE_H +#define MDBOX_MAP_PRIVATE_H + +#include "mdbox-map.h" + +struct dbox_mail_lookup_rec { + uint32_t map_uid; + uint16_t refcount; + struct mdbox_map_mail_index_record rec; +}; + +struct mdbox_map { + struct mdbox_storage *storage; + const struct mdbox_settings *set; + char *path, *index_path; + + struct mail_index *index; + struct mail_index_view *view; + + uint32_t map_ext_id, ref_ext_id; + + struct mailbox_list *root_list; + + bool verify_existing_file_ids:1; +}; + +struct mdbox_map_append { + struct dbox_file_append_context *file_append; + uoff_t offset, size; +}; + +struct mdbox_map_append_context { + struct mdbox_map *map; + struct mdbox_map_atomic_context *atomic; + struct mail_index_transaction *trans; + + ARRAY(struct dbox_file_append_context *) file_appends; + ARRAY(struct dbox_file *) files; + ARRAY(struct mdbox_map_append) appends; + + uint32_t first_new_file_id; + + unsigned int files_nonappendable_count; + + bool failed:1; +}; + +struct mdbox_map_atomic_context { + struct mdbox_map *map; + struct mail_index_transaction *sync_trans; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *sync_view; + + bool map_refreshed:1; + bool locked:1; + bool success:1; + bool failed:1; +}; + +int mdbox_map_view_lookup_rec(struct mdbox_map *map, + struct mail_index_view *view, uint32_t seq, + struct dbox_mail_lookup_rec *rec_r); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c new file mode 100644 index 0000000..513db02 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c @@ -0,0 +1,1502 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "ostream.h" +#include "mkdir-parents.h" +#include "unlink-old-files.h" +#include "mailbox-list-private.h" +#include "mdbox-storage.h" +#include "mdbox-file.h" +#include "mdbox-map-private.h" + +#include <dirent.h> + +#define MAX_BACKWARDS_LOOKUPS 10 + +#define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10) +#define DBOX_FORCE_PURGE_MIN_RATIO 0.5 + +#define MAP_STORAGE(map) (&(map)->storage->storage.storage) + +struct mdbox_map_transaction_context { + struct mdbox_map_atomic_context *atomic; + struct mail_index_transaction *trans; + + bool changed:1; + bool committed:1; +}; + +static int mdbox_map_generate_uid_validity(struct mdbox_map *map); + +void mdbox_map_set_corrupted(struct mdbox_map *map, const char *format, ...) +{ + va_list args; + + va_start(args, format); + mail_storage_set_critical(MAP_STORAGE(map), + "mdbox map %s corrupted: %s", + map->index->filepath, + t_strdup_vprintf(format, args)); + va_end(args); + + mdbox_storage_set_corrupted(map->storage); +} + +struct mdbox_map * +mdbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list) +{ + struct mdbox_map *map; + const char *root, *index_root; + + root = mailbox_list_get_root_forced(root_list, MAILBOX_LIST_PATH_TYPE_DIR); + index_root = mailbox_list_get_root_forced(root_list, MAILBOX_LIST_PATH_TYPE_INDEX); + + map = i_new(struct mdbox_map, 1); + map->storage = storage; + map->set = storage->set; + map->path = i_strconcat(root, "/"MDBOX_GLOBAL_DIR_NAME, NULL); + map->index_path = + i_strconcat(index_root, "/"MDBOX_GLOBAL_DIR_NAME, NULL); + map->index = mail_index_alloc(storage->storage.storage.event, + map->index_path, + MDBOX_GLOBAL_INDEX_PREFIX); + mail_index_set_fsync_mode(map->index, + MAP_STORAGE(map)->set->parsed_fsync_mode, 0); + mail_index_set_lock_method(map->index, + MAP_STORAGE(map)->set->parsed_lock_method, + mail_storage_get_lock_timeout(MAP_STORAGE(map), UINT_MAX)); + map->root_list = root_list; + map->map_ext_id = mail_index_ext_register(map->index, "map", + sizeof(struct mdbox_map_mail_index_header), + sizeof(struct mdbox_map_mail_index_record), + sizeof(uint32_t)); + map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0, + sizeof(uint16_t), sizeof(uint16_t)); + return map; +} + +void mdbox_map_deinit(struct mdbox_map **_map) +{ + struct mdbox_map *map = *_map; + + *_map = NULL; + + if (map->view != NULL) { + mail_index_view_close(&map->view); + mail_index_close(map->index); + } + mail_index_free(&map->index); + i_free(map->index_path); + i_free(map->path); + i_free(map); +} + +static int mdbox_map_mkdir_storage(struct mdbox_map *map) +{ + if (mailbox_list_mkdir_root(map->root_list, map->path, + MAILBOX_LIST_PATH_TYPE_DIR) < 0) { + mail_storage_copy_list_error(MAP_STORAGE(map), map->root_list); + return -1; + } + + if (strcmp(map->path, map->index_path) != 0 && + mailbox_list_mkdir_root(map->root_list, map->index_path, + MAILBOX_LIST_PATH_TYPE_INDEX) < 0) { + mail_storage_copy_list_error(MAP_STORAGE(map), map->root_list); + return -1; + } + return 0; +} + +static void mdbox_map_cleanup(struct mdbox_map *map) +{ + unsigned int interval = + MAP_STORAGE(map)->set->mail_temp_scan_interval; + struct stat st; + + if (stat(map->path, &st) < 0) + return; + + /* check once in a while if there are temp files to clean up */ + if (interval == 0) { + /* disabled */ + } else if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { + /* there haven't been any changes to this directory since we + last checked it. */ + } else if (st.st_atime < ioloop_time - (time_t)interval) { + /* time to scan */ + (void)unlink_old_files(map->path, DBOX_TEMP_FILE_PREFIX, + ioloop_time - DBOX_TMP_DELETE_SECS); + } +} + +static int mdbox_map_open_internal(struct mdbox_map *map, bool create_missing) +{ + enum mail_index_open_flags open_flags; + struct mailbox_permissions perm; + int ret = 0; + + if (map->view != NULL) { + /* already opened */ + return 1; + } + + mailbox_list_get_root_permissions(map->root_list, &perm); + mail_index_set_permissions(map->index, perm.file_create_mode, + perm.file_create_gid, + perm.file_create_gid_origin); + + open_flags = MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY | + mail_storage_settings_to_index_flags(MAP_STORAGE(map)->set); + if (create_missing) { + if ((ret = mdbox_map_mkdir_storage(map)) < 0) + return -1; + if (ret > 0) { + /* storage/ directory already existed. + the index should exist also. */ + } else { + open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; + } + } + ret = mail_index_open(map->index, open_flags); + if (ret == 0 && create_missing) { + /* storage/ already existed, but indexes didn't. we'll need to + take extra steps to make sure we won't overwrite any m.* + files that may already exist. */ + map->verify_existing_file_ids = TRUE; + open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; + ret = mail_index_open(map->index, open_flags); + } + if (ret < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + return -1; + } + if (ret == 0) { + /* index not found - for now just return failure */ + i_assert(!create_missing); + return 0; + } + + map->view = mail_index_view_open(map->index); + mdbox_map_cleanup(map); + + if (mail_index_get_header(map->view)->uid_validity == 0) { + if (mdbox_map_generate_uid_validity(map) < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + mail_index_close(map->index); + return -1; + } + if (mdbox_map_refresh(map) < 0) { + mail_index_close(map->index); + return -1; + } + } + return 1; +} + +int mdbox_map_open(struct mdbox_map *map) +{ + return mdbox_map_open_internal(map, FALSE); +} + +int mdbox_map_open_or_create(struct mdbox_map *map) +{ + return mdbox_map_open_internal(map, TRUE) <= 0 ? -1 : 0; +} + +int mdbox_map_refresh(struct mdbox_map *map) +{ + struct mail_index_view_sync_ctx *ctx; + bool delayed_expunges, fscked; + int ret = 0; + + /* some open files may have read partially written mails. now that + map syncing makes the new mails visible, we need to make sure the + partial data is flushed out of memory */ + mdbox_files_sync_input(map->storage); + + if (mail_index_refresh(map->view->index) < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + return -1; + } + if (mail_index_view_have_transactions(map->view)) { + /* can't sync when there are transactions */ + return 0; + } + + ctx = mail_index_view_sync_begin(map->view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + fscked = mail_index_reset_fscked(map->view->index); + if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { + mail_storage_set_index_error(MAP_STORAGE(map), map->index); + ret = -1; + } + if (fscked) + mdbox_storage_set_corrupted(map->storage); + return ret; +} + +bool mdbox_map_is_fscked(struct mdbox_map *map) +{ + const struct mail_index_header *hdr; + + if (map->view == NULL) { + /* map isn't opened yet. don't bother. */ + return FALSE; + } + + hdr = mail_index_get_header(map->view); + return (hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0; +} + +static void +mdbox_map_get_ext_hdr(struct mdbox_map *map, struct mail_index_view *view, + struct mdbox_map_mail_index_header *hdr_r) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size); + i_zero(hdr_r); + memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r))); +} + +uint32_t mdbox_map_get_rebuild_count(struct mdbox_map *map) +{ + struct mdbox_map_mail_index_header hdr; + + mdbox_map_get_ext_hdr(map, map->view, &hdr); + return hdr.rebuild_count; +} + +static int +mdbox_map_lookup_seq(struct mdbox_map *map, uint32_t seq, + const struct mdbox_map_mail_index_record **rec_r) +{ + const struct mdbox_map_mail_index_record *rec; + const void *data; + uint32_t uid; + + mail_index_lookup_ext(map->view, seq, map->map_ext_id, &data, NULL); + rec = data; + + if (rec == NULL || rec->file_id == 0) { + mail_index_lookup_uid(map->view, seq, &uid); + mdbox_map_set_corrupted(map, "file_id=0 for map_uid=%u", uid); + return -1; + } + *rec_r = rec; + return 0; +} + +static int +mdbox_map_get_seq(struct mdbox_map *map, uint32_t map_uid, uint32_t *seq_r) +{ + if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) { + /* not found - try again after a refresh */ + if (mdbox_map_refresh(map) < 0) + return -1; + if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) + return 0; + } + return 1; +} + +int mdbox_map_lookup(struct mdbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r) +{ + const struct mdbox_map_mail_index_record *rec; + uint32_t seq; + int ret; + + if (mdbox_map_open_or_create(map) < 0) + return -1; + + if ((ret = mdbox_map_get_seq(map, map_uid, &seq)) <= 0) + return ret; + + if (mdbox_map_lookup_seq(map, seq, &rec) < 0) + return -1; + *file_id_r = rec->file_id; + *offset_r = rec->offset; + return 1; +} + +int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r) +{ + uint32_t seq; + int ret; + + if (mdbox_map_open_or_create(map) < 0) + return -1; + + if ((ret = mdbox_map_get_seq(map, map_uid, &seq)) <= 0) + return ret; + + return mdbox_map_lookup_seq_full(map, seq, rec_r, refcount_r); +} + +int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r) +{ + const struct mdbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + + if (mdbox_map_lookup_seq(map, seq, &rec) < 0) + return -1; + *rec_r = *rec; + + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing ref extension"); + return -1; + } + ref16_p = data; + *refcount_r = *ref16_p; + return 1; +} + +uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq) +{ + uint32_t uid; + + mail_index_lookup_uid(map->view, seq, &uid); + return uid; +} + +unsigned int mdbox_map_get_messages_count(struct mdbox_map *map) +{ + return mail_index_view_get_messages_count(map->view); +} + +int mdbox_map_view_lookup_rec(struct mdbox_map *map, + struct mail_index_view *view, uint32_t seq, + struct dbox_mail_lookup_rec *rec_r) +{ + const uint16_t *ref16_p; + const void *data; + + i_zero(rec_r); + mail_index_lookup_uid(view, seq, &rec_r->map_uid); + + mail_index_lookup_ext(view, seq, map->map_ext_id, &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing map extension"); + return -1; + } + memcpy(&rec_r->rec, data, sizeof(rec_r->rec)); + + mail_index_lookup_ext(view, seq, map->ref_ext_id, &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing ref extension"); + return -1; + } + ref16_p = data; + rec_r->refcount = *ref16_p; + return 0; +} + +int mdbox_map_get_file_msgs(struct mdbox_map *map, uint32_t file_id, + ARRAY_TYPE(mdbox_map_file_msg) *recs) +{ + const struct mail_index_header *hdr; + struct dbox_mail_lookup_rec rec; + struct mdbox_map_file_msg msg; + uint32_t seq; + + if (mdbox_map_refresh(map) < 0) + return -1; + hdr = mail_index_get_header(map->view); + + i_zero(&msg); + for (seq = 1; seq <= hdr->messages_count; seq++) { + if (mdbox_map_view_lookup_rec(map, map->view, seq, &rec) < 0) + return -1; + + if (rec.rec.file_id == file_id) { + msg.map_uid = rec.map_uid; + msg.offset = rec.rec.offset; + msg.refcount = rec.refcount; + array_push_back(recs, &msg); + } + } + return 0; +} + +int mdbox_map_get_zero_ref_files(struct mdbox_map *map, + ARRAY_TYPE(seq_range) *file_ids_r) +{ + const struct mail_index_header *hdr; + const struct mdbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + uint32_t seq; + bool expunged; + int ret; + + if ((ret = mdbox_map_open(map)) <= 0) { + /* no map / internal error */ + return ret; + } + if (mdbox_map_refresh(map) < 0) + return -1; + + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + if (data != NULL && !expunged) { + ref16_p = data; + if (*ref16_p != 0) + continue; + } + + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data != NULL && !expunged) { + rec = data; + seq_range_array_add(file_ids_r, rec->file_id); + } + } + return 0; +} + +struct mdbox_map_atomic_context *mdbox_map_atomic_begin(struct mdbox_map *map) +{ + struct mdbox_map_atomic_context *atomic; + + atomic = i_new(struct mdbox_map_atomic_context, 1); + atomic->map = map; + return atomic; +} + +static void +mdbox_map_sync_handle(struct mdbox_map *map, + struct mail_index_sync_ctx *sync_ctx) +{ + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + uoff_t offset1, offset2; + + mail_index_sync_get_offsets(sync_ctx, &seq1, &offset1, &seq2, &offset2); + if (offset1 != offset2 || seq1 != seq2) { + /* something had crashed. need a full resync. */ + i_warning("mdbox %s: Inconsistency in map index " + "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")", + map->path, seq1, offset1, seq2, offset2); + mdbox_storage_set_corrupted(map->storage); + } + while (mail_index_sync_next(sync_ctx, &sync_rec)) ; +} + +int mdbox_map_atomic_lock(struct mdbox_map_atomic_context *atomic, + const char *reason) +{ + int ret; + + if (atomic->locked) + return 0; + + if (mdbox_map_open_or_create(atomic->map) < 0) + return -1; + + /* use syncing to lock the transaction log, so that we always see + log's head_offset = tail_offset */ + ret = mail_index_sync_begin(atomic->map->index, &atomic->sync_ctx, + &atomic->sync_view, &atomic->sync_trans, + MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET); + if (mail_index_reset_fscked(atomic->map->index)) + mdbox_storage_set_corrupted(atomic->map->storage); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_index_error(MAP_STORAGE(atomic->map), + atomic->map->index); + return -1; + } + mail_index_sync_set_reason(atomic->sync_ctx, reason); + atomic->locked = TRUE; + /* reset refresh state so that if it's wanted to be done locked, + it gets the latest changes */ + atomic->map_refreshed = FALSE; + mdbox_map_sync_handle(atomic->map, atomic->sync_ctx); + return 0; +} + +bool mdbox_map_atomic_is_locked(struct mdbox_map_atomic_context *atomic) +{ + return atomic->locked; +} + +void mdbox_map_atomic_set_failed(struct mdbox_map_atomic_context *atomic) +{ + atomic->success = FALSE; + atomic->failed = TRUE; +} + +void mdbox_map_atomic_set_success(struct mdbox_map_atomic_context *atomic) +{ + if (!atomic->failed) + atomic->success = TRUE; +} + +void mdbox_map_atomic_unset_fscked(struct mdbox_map_atomic_context *atomic) +{ + mail_index_unset_fscked(atomic->sync_trans); +} + +int mdbox_map_atomic_finish(struct mdbox_map_atomic_context **_atomic) +{ + struct mdbox_map_atomic_context *atomic = *_atomic; + int ret = 0; + + *_atomic = NULL; + + if (atomic->sync_ctx == NULL) { + /* not locked */ + i_assert(!atomic->locked); + } else if (atomic->success) { + if (mail_index_sync_commit(&atomic->sync_ctx) < 0) { + mail_storage_set_index_error(MAP_STORAGE(atomic->map), + atomic->map->index); + ret = -1; + } + } else { + mail_index_sync_rollback(&atomic->sync_ctx); + } + i_free(atomic); + return ret; +} + +struct mdbox_map_transaction_context * +mdbox_map_transaction_begin(struct mdbox_map_atomic_context *atomic, + bool external) +{ + struct mdbox_map_transaction_context *ctx; + enum mail_index_transaction_flags flags = + MAIL_INDEX_TRANSACTION_FLAG_FSYNC; + bool success; + + if (external) + flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + + ctx = i_new(struct mdbox_map_transaction_context, 1); + ctx->atomic = atomic; + if (atomic->locked && atomic->map_refreshed) { + /* already refreshed within a lock, don't do it again */ + success = TRUE; + } else { + success = mdbox_map_open(atomic->map) > 0 && + mdbox_map_refresh(atomic->map) == 0; + } + + if (success) { + atomic->map_refreshed = TRUE; + ctx->trans = mail_index_transaction_begin(atomic->map->view, + flags); + } + return ctx; +} + +int mdbox_map_transaction_commit(struct mdbox_map_transaction_context *ctx, + const char *reason) +{ + i_assert(!ctx->committed); + + ctx->committed = TRUE; + if (!ctx->changed) + return 0; + + if (mdbox_map_atomic_lock(ctx->atomic, reason) < 0) + return -1; + + if (mail_index_transaction_commit(&ctx->trans) < 0) { + mail_storage_set_index_error(MAP_STORAGE(ctx->atomic->map), + ctx->atomic->map->index); + return -1; + } + mdbox_map_atomic_set_success(ctx->atomic); + return 0; +} + +void mdbox_map_transaction_free(struct mdbox_map_transaction_context **_ctx) +{ + struct mdbox_map_transaction_context *ctx = *_ctx; + + *_ctx = NULL; + + if (ctx->trans != NULL) + mail_index_transaction_rollback(&ctx->trans); + i_free(ctx); +} + +int mdbox_map_update_refcount(struct mdbox_map_transaction_context *ctx, + uint32_t map_uid, int diff) +{ + struct mdbox_map *map = ctx->atomic->map; + const void *data; + uint32_t seq; + int old_diff, new_diff; + + if (unlikely(ctx->trans == NULL)) + return -1; + + if (!mail_index_lookup_seq(map->view, map_uid, &seq)) { + /* we can't refresh map here since view has a + transaction open. */ + if (diff > 0) { + /* the message was probably just purged */ + mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_EXPUNGED, + "Some of the requested messages no longer exist."); + } else { + mdbox_map_set_corrupted(map, + "refcount update lost map_uid=%u", map_uid); + } + return -1; + } + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, &data, NULL); + old_diff = data == NULL ? 0 : *((const uint16_t *)data); + ctx->changed = TRUE; + new_diff = mail_index_atomic_inc_ext(ctx->trans, seq, + map->ref_ext_id, diff); + if (old_diff + new_diff < 0) { + mdbox_map_set_corrupted(map, "map_uid=%u refcount too low", + map_uid); + return -1; + } + if (old_diff + new_diff >= 32768 && new_diff > 0) { + /* we're getting close to the 64k limit. fail early + to make it less likely that two processes increase + the refcount enough times to cross the limit */ + mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_LIMIT, + t_strdup_printf("Message has been copied too many times (%d + %d)", + old_diff, new_diff)); + return -1; + } + return 0; +} + +int mdbox_map_update_refcounts(struct mdbox_map_transaction_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, int diff) +{ + const uint32_t *uidp; + unsigned int i, count; + + if (unlikely(ctx->trans == NULL)) + return -1; + + count = array_count(map_uids); + for (i = 0; i < count; i++) { + uidp = array_idx(map_uids, i); + if (mdbox_map_update_refcount(ctx, *uidp, diff) < 0) + return -1; + } + return 0; +} + +int mdbox_map_remove_file_id(struct mdbox_map *map, uint32_t file_id) +{ + struct mdbox_map_atomic_context *atomic; + struct mdbox_map_transaction_context *map_trans; + const struct mail_index_header *hdr; + const struct mdbox_map_mail_index_record *rec; + const void *data; + uint32_t seq; + int ret = 0; + + /* make sure the map is refreshed, otherwise we might be expunging + messages that have already been moved to other files. */ + + /* we need a per-file transaction, otherwise we can't refresh the map */ + atomic = mdbox_map_atomic_begin(map); + map_trans = mdbox_map_transaction_begin(atomic, TRUE); + + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, NULL); + if (data == NULL) { + mdbox_map_set_corrupted(map, "missing map extension"); + ret = -1; + break; + } + + rec = data; + if (rec->file_id == file_id) { + map_trans->changed = TRUE; + mail_index_expunge(map_trans->trans, seq); + } + } + if (ret == 0) + ret = mdbox_map_transaction_commit(map_trans, "removing file"); + mdbox_map_transaction_free(&map_trans); + if (mdbox_map_atomic_finish(&atomic) < 0) + ret = -1; + return ret; +} + +struct mdbox_map_append_context * +mdbox_map_append_begin(struct mdbox_map_atomic_context *atomic) +{ + struct mdbox_map_append_context *ctx; + + ctx = i_new(struct mdbox_map_append_context, 1); + ctx->atomic = atomic; + ctx->map = atomic->map; + ctx->first_new_file_id = (uint32_t)-1; + i_array_init(&ctx->file_appends, 64); + i_array_init(&ctx->files, 64); + i_array_init(&ctx->appends, 128); + + if (mdbox_map_open_or_create(atomic->map) < 0) + ctx->failed = TRUE; + else { + /* refresh the map so we can try appending to the + latest files */ + if (mdbox_map_refresh(atomic->map) == 0) + atomic->map_refreshed = TRUE; + else + ctx->failed = TRUE; + } + return ctx; +} + +static time_t day_begin_stamp(unsigned int interval) +{ + struct tm tm; + time_t stamp; + unsigned int unit = 1; + + if (interval == 0) + return 0; + + /* get the beginning of day/hour/minute depending on how large + the interval is */ + tm = *localtime(&ioloop_time); + if (interval >= 60) { + tm.tm_sec = 0; + unit = 60; + if (interval >= 3600) { + tm.tm_min = 0; + unit = 3600; + if (interval >= 3600*24) { + tm.tm_hour = 0; + unit = 3600*24; + } + } + } + stamp = mktime(&tm); + if (stamp == (time_t)-1) + i_panic("mktime(today) failed"); + + return stamp - (interval - unit); +} + +static bool dbox_try_open(struct dbox_file *file, bool want_altpath) +{ + bool notfound; + + if (want_altpath) { + if (dbox_file_open(file, ¬found) <= 0) + return FALSE; + } else { + if (dbox_file_open_primary(file, ¬found) <= 0) + return FALSE; + } + if (notfound) + return FALSE; + + if (file->lock != NULL) { + /* already locked, we're possibly in the middle of purging it + in which case we really don't want to write there. */ + return FALSE; + } + if (dbox_file_is_in_alt(file) != want_altpath) { + /* different alt location than what we want, can't use it */ + return FALSE; + } + return TRUE; +} + +static bool dbox_file_is_ok_at(struct dbox_file *file, uoff_t offset) +{ + bool last; + int ret; + + if (dbox_file_seek(file, offset) == 0) + return FALSE; + + while ((ret = dbox_file_seek_next(file, &offset, &last)) > 0); + if (ret == 0 && !last) + return FALSE; + return TRUE; +} + +static bool +mdbox_map_file_try_append(struct mdbox_map_append_context *ctx, + bool want_altpath, + const struct mdbox_map_mail_index_record *rec, + time_t stamp, uoff_t mail_size, + struct dbox_file_append_context **file_append_r, + struct ostream **output_r, bool *retry_later_r) +{ + struct mdbox_map *map = ctx->map; + struct mdbox_storage *storage = map->storage; + struct dbox_file *file; + struct dbox_file_append_context *file_append; + struct stat st; + bool file_too_old = FALSE; + int ret; + + *file_append_r = NULL; + *output_r = NULL; + *retry_later_r = FALSE; + + file = mdbox_file_init(storage, rec->file_id); + if (!dbox_try_open(file, want_altpath)) { + dbox_file_unref(&file); + return TRUE; + } + + if (file->create_time < stamp) + file_too_old = TRUE; + else if ((ret = dbox_file_try_lock(file)) <= 0) { + /* locking failed */ + *retry_later_r = ret == 0; + } else if (stat(file->cur_path, &st) < 0) { + if (errno != ENOENT) + i_error("stat(%s) failed: %m", file->cur_path); + /* the file was unlinked between opening and locking it. */ + } else if (st.st_size != rec->offset + rec->size && + /* check if there's any garbage at the end of file. + note that there may be valid messages added by another + session before we locked it (but after we refreshed + map index). */ + !dbox_file_is_ok_at(file, rec->offset + rec->size)) { + /* error message was already logged */ + } else { + file_append = dbox_file_append_init(file); + if (dbox_file_get_append_stream(file_append, output_r) <= 0) { + /* couldn't append to this file */ + } else if ((*output_r)->offset + mail_size > map->set->mdbox_rotate_size) { + /* file was too large after all */ + } else { + /* success */ + *file_append_r = file_append; + return TRUE; + } + dbox_file_append_rollback(&file_append); + } + + /* failure */ + dbox_file_unlock(file); + dbox_file_unref(&file); + return !file_too_old; +} + +static bool +mdbox_map_is_appending(struct mdbox_map_append_context *ctx, uint32_t file_id) +{ + struct dbox_file_append_context *const *file_appends; + unsigned int i, count; + + /* there shouldn't be many files open, don't bother with anything + faster. */ + file_appends = array_get(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)file_appends[i]->file; + + if (mfile->file_id == file_id) + return TRUE; + } + return FALSE; +} + +static struct dbox_file_append_context * +mdbox_map_find_existing_append(struct mdbox_map_append_context *ctx, + uoff_t mail_size, bool want_altpath, + struct ostream **output_r) +{ + struct mdbox_map *map = ctx->map; + struct dbox_file_append_context *const *file_appends, *append; + struct mdbox_file *mfile; + unsigned int i, count; + uoff_t append_offset; + + /* first try to use files already used in this append */ + file_appends = array_get(&ctx->file_appends, &count); + for (i = count; i > ctx->files_nonappendable_count; i--) { + append = file_appends[i-1]; + + if (dbox_file_is_in_alt(append->file) != want_altpath) + continue; + if (append->file->fd == -1) { + /* already closed it (below). we might be able to still + fit some small mail there, but that's too much + trouble */ + continue; + } + + append_offset = append->output->offset; + if (append_offset + mail_size <= map->set->mdbox_rotate_size && + dbox_file_get_append_stream(append, output_r) > 0) + return append; + + /* can't append to this file anymore. if we created this file, + close it so we don't waste fds. if we didn't, we can't close + it without also losing our lock too early. */ + mfile = (struct mdbox_file *)append->file; + if (mfile->file_id == 0 && dbox_file_append_flush(append) == 0) + dbox_file_close(append->file); + } + ctx->files_nonappendable_count = count; + return NULL; +} + +static int +mdbox_map_find_primary_files(struct mdbox_map_append_context *ctx, + ARRAY_TYPE(seq_range) *file_ids_r) +{ + struct mdbox_storage *dstorage = ctx->map->storage; + struct mail_storage *storage = &dstorage->storage.storage; + DIR *dir; + struct dirent *d; + uint32_t file_id; + int ret = 0; + + /* we want to quickly find the latest alt file, but we also want to + avoid accessing the alt storage as much as possible. typically most + of the older mails would be in alt storage, so we'll just put the + few m.* files in primary storage to checked_file_ids array. other + files are then known to exist in alt storage. */ + dir = opendir(dstorage->storage_dir); + if (dir == NULL) { + mail_storage_set_critical(storage, + "opendir(%s) failed: %m", dstorage->storage_dir); + return -1; + } + + for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) { + if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, + strlen(MDBOX_MAIL_FILE_PREFIX)) != 0) + continue; + if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX), + &file_id) < 0) + continue; + + seq_range_array_add(file_ids_r, file_id); + } + if (errno != 0) { + mail_storage_set_critical(storage, + "readdir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(storage, + "closedir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + return ret; +} + +static int +mdbox_map_find_appendable_file(struct mdbox_map_append_context *ctx, + uoff_t mail_size, bool want_altpath, + struct dbox_file_append_context **file_append_r, + struct ostream **output_r) +{ + struct mdbox_map *map = ctx->map; + ARRAY_TYPE(seq_range) checked_file_ids; + const struct mail_index_header *hdr; + const struct mdbox_map_mail_index_record *rec; + unsigned int backwards_lookup_count; + uint32_t seq, seq1, uid; + time_t stamp; + bool retry_later; + + if (mail_size >= map->set->mdbox_rotate_size) + return 0; + + /* try to find an existing appendable file */ + stamp = day_begin_stamp(map->set->mdbox_rotate_interval); + hdr = mail_index_get_header(map->view); + + backwards_lookup_count = 0; + t_array_init(&checked_file_ids, 16); + + if (want_altpath) { + /* we want to save to alt storage. */ + if (mdbox_map_find_primary_files(ctx, &checked_file_ids) < 0) + return -1; + } + + for (seq = hdr->messages_count; seq > 0; seq--) { + if (mdbox_map_lookup_seq(map, seq, &rec) < 0) + return -1; + + if (seq_range_exists(&checked_file_ids, rec->file_id)) + continue; + seq_range_array_add(&checked_file_ids, rec->file_id); + + if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) { + /* we've wasted enough time here */ + break; + } + + /* first lookup: this should be enough usually, but we can't + be sure until after locking. also if messages were recently + moved, this message might not be the last one in the file. */ + if (rec->offset + rec->size + mail_size >= + map->set->mdbox_rotate_size) + continue; + + if (mdbox_map_is_appending(ctx, rec->file_id)) { + /* already checked this */ + continue; + } + + mail_index_lookup_uid(map->view, seq, &uid); + if (!mdbox_map_file_try_append(ctx, want_altpath, rec, + stamp, mail_size, file_append_r, + output_r, &retry_later)) { + /* file is too old. the rest of the files are too. */ + break; + } + /* NOTE: we've now refreshed map view. there are no guarantees + about sequences anymore. */ + if (*file_append_r != NULL) + return 1; + /* FIXME: use retry_later somehow */ + if (uid == 1 || + !mail_index_lookup_seq_range(map->view, 1, uid-1, + &seq1, &seq)) + break; + seq++; + } + return 0; +} + +int mdbox_map_append_next(struct mdbox_map_append_context *ctx, + uoff_t mail_size, enum mdbox_map_append_flags flags, + struct dbox_file_append_context **file_append_ctx_r, + struct ostream **output_r) +{ + struct dbox_file *file; + struct mdbox_map_append *append; + struct dbox_file_append_context *file_append; + bool existing, want_altpath; + int ret; + + if (ctx->failed) + return -1; + + want_altpath = (flags & DBOX_MAP_APPEND_FLAG_ALT) != 0; + file_append = mdbox_map_find_existing_append(ctx, mail_size, + want_altpath, output_r); + if (file_append != NULL) { + ret = 1; + existing = TRUE; + } else { + ret = mdbox_map_find_appendable_file(ctx, mail_size, want_altpath, + &file_append, output_r); + existing = FALSE; + } + if (ret > 0) + file = file_append->file; + else if (ret < 0) + return -1; + else { + /* create a new file */ + file = (flags & DBOX_MAP_APPEND_FLAG_ALT) == 0 ? + mdbox_file_init(ctx->map->storage, 0) : + mdbox_file_init_new_alt(ctx->map->storage); + file_append = dbox_file_append_init(file); + + ret = dbox_file_get_append_stream(file_append, output_r); + if (ret <= 0) { + i_assert(ret < 0); + dbox_file_append_rollback(&file_append); + dbox_file_unref(&file); + return -1; + } + } + + append = array_append_space(&ctx->appends); + append->file_append = file_append; + append->offset = (*output_r)->offset; + append->size = (uint32_t)-1; + if (!existing) { + i_assert(file_append->first_append_offset == 0); + file_append->first_append_offset = file_append->output->offset; + array_push_back(&ctx->file_appends, &file_append); + array_push_back(&ctx->files, &file); + } + *file_append_ctx_r = file_append; + return 0; +} + +static void +mdbox_map_append_close_if_unneeded(struct mdbox_map *map, + struct dbox_file_append_context *append_ctx) +{ + struct mdbox_file *mfile = + (struct mdbox_file *)append_ctx->file; + uoff_t end_offset = append_ctx->output->offset; + + /* if this file is now large enough not to fit any other + mails and we created it, close its fd since it's not + needed anymore. */ + if (end_offset > map->set->mdbox_rotate_size && + mfile->file_id == 0 && + dbox_file_append_flush(append_ctx) == 0) + dbox_file_close(append_ctx->file); +} + +void mdbox_map_append_finish(struct mdbox_map_append_context *ctx) +{ + struct mdbox_map_append *appends, *last; + unsigned int count; + uoff_t cur_offset; + + appends = array_get_modifiable(&ctx->appends, &count); + i_assert(count > 0); + last = &appends[count-1]; + i_assert(last->size == (uint32_t)-1); + + cur_offset = last->file_append->output->offset; + i_assert(cur_offset >= last->offset); + last->size = cur_offset - last->offset; + dbox_file_append_checkpoint(last->file_append); + + mdbox_map_append_close_if_unneeded(ctx->map, last->file_append); +} + +void mdbox_map_append_abort(struct mdbox_map_append_context *ctx) +{ + struct mdbox_map_append *appends; + unsigned int count; + + appends = array_get_modifiable(&ctx->appends, &count); + i_assert(count > 0 && appends[count-1].size == (uint32_t)-1); + array_delete(&ctx->appends, count-1, 1); +} + +static int +mdbox_find_highest_file_id(struct mdbox_map *map, uint32_t *file_id_r) +{ + const size_t prefix_len = strlen(MDBOX_MAIL_FILE_PREFIX); + DIR *dir; + struct dirent *d; + unsigned int id, highest_id = 0; + + dir = opendir(map->path); + if (dir == NULL) { + i_error("opendir(%s) failed: %m", map->path); + return -1; + } + while ((d = readdir(dir)) != NULL) { + if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, prefix_len) == 0 && + str_to_uint(d->d_name + prefix_len, &id) == 0) { + if (highest_id < id) + highest_id = id; + } + } + (void)closedir(dir); + + *file_id_r = highest_id; + return 0; +} + +static int +mdbox_map_assign_file_ids(struct mdbox_map_append_context *ctx, + bool separate_transaction, const char *reason) +{ + struct dbox_file_append_context *const *file_appends; + unsigned int i, count; + struct mdbox_map_mail_index_header hdr; + uint32_t first_file_id, file_id, existing_id; + + /* start the syncing. we'll need it even if there are no file ids to + be assigned. */ + if (mdbox_map_atomic_lock(ctx->atomic, reason) < 0) + return -1; + + mdbox_map_get_ext_hdr(ctx->map, ctx->atomic->sync_view, &hdr); + file_id = hdr.highest_file_id + 1; + + if (ctx->map->verify_existing_file_ids) { + /* storage/ directory had been already created but + without indexes. scan to see if there exists a higher + m.* file id than what is in header, so we won't + accidentally overwrite any existing files. */ + if (mdbox_find_highest_file_id(ctx->map, &existing_id) < 0) + return -1; + if (file_id < existing_id+1) + file_id = existing_id+1; + } + + /* assign file_ids for newly created files */ + first_file_id = file_id; + file_appends = array_get(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)file_appends[i]->file; + + if (dbox_file_append_flush(file_appends[i]) < 0) + return -1; + + if (mfile->file_id == 0) { + if (mdbox_file_assign_file_id(mfile, file_id++) < 0) + return -1; + } + } + + ctx->trans = !separate_transaction ? NULL : + mail_index_transaction_begin(ctx->map->view, + MAIL_INDEX_TRANSACTION_FLAG_FSYNC); + + /* update the highest used file_id */ + if (first_file_id != file_id) { + file_id--; + mail_index_update_header_ext(ctx->trans != NULL ? ctx->trans : + ctx->atomic->sync_trans, + ctx->map->map_ext_id, + 0, &file_id, sizeof(file_id)); + } + return 0; +} + +int mdbox_map_append_assign_map_uids(struct mdbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r) +{ + const struct mdbox_map_append *appends; + const struct mail_index_header *hdr; + struct mdbox_map_mail_index_record rec; + unsigned int i, count; + ARRAY_TYPE(seq_range) uids; + const struct seq_range *range; + uint32_t seq; + uint16_t ref16; + int ret = 0; + + if (array_count(&ctx->appends) == 0) { + *first_map_uid_r = 0; + *last_map_uid_r = 0; + return 0; + } + + if (mdbox_map_assign_file_ids(ctx, TRUE, "saving - assign uids") < 0) + return -1; + + /* append map records to index */ + i_zero(&rec); + ref16 = 1; + appends = array_get(&ctx->appends, &count); + for (i = 0; i < count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)appends[i].file_append->file; + + i_assert(appends[i].offset <= (uint32_t)-1); + i_assert(appends[i].size <= (uint32_t)-1); + + rec.file_id = mfile->file_id; + rec.offset = appends[i].offset; + rec.size = appends[i].size; + + mail_index_append(ctx->trans, 0, &seq); + mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id, + &rec, NULL); + mail_index_update_ext(ctx->trans, seq, ctx->map->ref_ext_id, + &ref16, NULL); + } + + /* assign map UIDs for appended records */ + hdr = mail_index_get_header(ctx->atomic->sync_view); + t_array_init(&uids, 1); + mail_index_append_finish_uids(ctx->trans, hdr->next_uid, &uids); + range = array_front(&uids); + i_assert(range[0].seq2 - range[0].seq1 + 1 == count); + + if (hdr->uid_validity == 0) { + /* we don't really care about uidvalidity, but it can't be 0 */ + uint32_t uid_validity = ioloop_time; + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + if (mail_index_transaction_commit(&ctx->trans) < 0) { + mail_storage_set_index_error(MAP_STORAGE(ctx->map), + ctx->map->index); + return -1; + } + + *first_map_uid_r = range[0].seq1; + *last_map_uid_r = range[0].seq2; + return ret; +} + +int mdbox_map_append_move(struct mdbox_map_append_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, + const ARRAY_TYPE(seq_range) *expunge_map_uids) +{ + const struct mdbox_map_append *appends; + struct mdbox_map_mail_index_record rec; + struct seq_range_iter iter; + const uint32_t *uids; + unsigned int i, j, map_uids_count, appends_count; + uint32_t uid, seq, next_uid; + + /* map is locked by this call */ + if (mdbox_map_assign_file_ids(ctx, FALSE, "purging - update uids") < 0) + return -1; + + i_zero(&rec); + appends = array_get(&ctx->appends, &appends_count); + + next_uid = mail_index_get_header(ctx->atomic->sync_view)->next_uid; + uids = array_get(map_uids, &map_uids_count); + for (i = j = 0; i < map_uids_count; i++) { + struct mdbox_file *mfile = + (struct mdbox_file *)appends[j].file_append->file; + + i_assert(j < appends_count); + rec.file_id = mfile->file_id; + rec.offset = appends[j].offset; + rec.size = appends[j].size; + j++; + + if (!mail_index_lookup_seq(ctx->atomic->sync_view, + uids[i], &seq)) { + /* We wrote the email to the new m.* file, but another + process already expunged it and purged it. Deleting + the email from the new m.* file would be problematic + at this point, so just add the mail back to the map + with refcount=0 and the next purge will remove it. */ + mail_index_append(ctx->atomic->sync_trans, + next_uid++, &seq); + } + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->map->map_ext_id, &rec, NULL); + } + + seq_range_array_iter_init(&iter, expunge_map_uids); i = 0; + while (seq_range_array_iter_nth(&iter, i++, &uid)) { + if (!mail_index_lookup_seq(ctx->atomic->sync_view, uid, &seq)) + i_unreached(); + mail_index_expunge(ctx->atomic->sync_trans, seq); + } + return 0; +} + +int mdbox_map_append_flush(struct mdbox_map_append_context *ctx) +{ + struct dbox_file_append_context **file_appends; + unsigned int i, count; + + i_assert(ctx->trans == NULL); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (dbox_file_append_flush(file_appends[i]) < 0) + return -1; + } + return 0; +} + +int mdbox_map_append_commit(struct mdbox_map_append_context *ctx) +{ + struct dbox_file_append_context **file_appends; + unsigned int i, count; + + i_assert(ctx->trans == NULL); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (dbox_file_append_commit(&file_appends[i]) < 0) + return -1; + } + mdbox_map_atomic_set_success(ctx->atomic); + return 0; +} + +void mdbox_map_append_free(struct mdbox_map_append_context **_ctx) +{ + struct mdbox_map_append_context *ctx = *_ctx; + struct dbox_file_append_context **file_appends; + struct dbox_file **files; + unsigned int i, count; + + *_ctx = NULL; + + if (ctx->trans != NULL) + mail_index_transaction_rollback(&ctx->trans); + + file_appends = array_get_modifiable(&ctx->file_appends, &count); + for (i = 0; i < count; i++) { + if (file_appends[i] != NULL) + dbox_file_append_rollback(&file_appends[i]); + } + + files = array_get_modifiable(&ctx->files, &count); + for (i = 0; i < count; i++) { + dbox_file_unlock(files[i]); + dbox_file_unref(&files[i]); + } + + array_free(&ctx->appends); + array_free(&ctx->file_appends); + array_free(&ctx->files); + i_free(ctx); +} + +static int mdbox_map_generate_uid_validity(struct mdbox_map *map) +{ + const struct mail_index_header *hdr; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + uint32_t uid_validity; + int ret; + + /* do this inside syncing, so that we're locked and there are no + race conditions */ + ret = mail_index_sync_begin(map->index, &sync_ctx, &view, &trans, + MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET); + if (ret <= 0) { + i_assert(ret != 0); + return -1; + } + mdbox_map_sync_handle(map, sync_ctx); + + hdr = mail_index_get_header(map->view); + if (hdr->uid_validity != 0) { + /* someone else beat us to it */ + } else { + uid_validity = ioloop_time; + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + mail_index_sync_set_reason(sync_ctx, "uidvalidity initialization"); + return mail_index_sync_commit(&sync_ctx); +} + +uint32_t mdbox_map_get_uid_validity(struct mdbox_map *map) +{ + uint32_t uid_validity; + + i_assert(map->view != NULL); + + uid_validity = mail_index_get_header(map->view)->uid_validity; + if (uid_validity == 0) + mdbox_map_set_corrupted(map, "lost uidvalidity"); + return uid_validity; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.h b/src/lib-storage/index/dbox-multi/mdbox-map.h new file mode 100644 index 0000000..9571f0e --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-map.h @@ -0,0 +1,144 @@ +#ifndef MDBOX_MAP_H +#define MDBOX_MAP_H + +#include "seq-range-array.h" + +struct dbox_file_append_context; +struct mdbox_map_append_context; +struct mdbox_storage; + +enum mdbox_map_append_flags { + DBOX_MAP_APPEND_FLAG_ALT = 0x01 +}; + +struct mdbox_map_mail_index_header { + uint32_t highest_file_id; + /* increased every time storage is rebuilt */ + uint32_t rebuild_count; +}; + +struct mdbox_map_mail_index_record { + uint32_t file_id; + uint32_t offset; + uint32_t size; /* including pre/post metadata */ +}; + +struct mdbox_map_file_msg { + uint32_t map_uid; + uint32_t offset; + uint32_t refcount; +}; +ARRAY_DEFINE_TYPE(mdbox_map_file_msg, struct mdbox_map_file_msg); + +struct mdbox_map * +mdbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list); +void mdbox_map_deinit(struct mdbox_map **map); + +/* Open the map. Returns 1 if ok, 0 if map doesn't exist, -1 if error. */ +int mdbox_map_open(struct mdbox_map *map); +/* Open or create the map. This is done automatically for most operations. + Returns 0 if ok, -1 if error. */ +int mdbox_map_open_or_create(struct mdbox_map *map); +/* Refresh the map. Returns 0 if ok, -1 if error. */ +int mdbox_map_refresh(struct mdbox_map *map); +/* Returns TRUE if map has been fsck'd. */ +bool mdbox_map_is_fscked(struct mdbox_map *map); + +/* Return the current rebuild counter */ +uint32_t mdbox_map_get_rebuild_count(struct mdbox_map *map); + +/* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID + is already expunged, -1 if error. */ +int mdbox_map_lookup(struct mdbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r); +/* Like mdbox_map_lookup(), but look up everything. */ +int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r); +/* Like mdbox_map_lookup_full(), but look up with sequence. */ +int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r); +/* Return map UID for the map sequence. */ +uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq); +/* Returns the total number of messages in the map. */ +unsigned int mdbox_map_get_messages_count(struct mdbox_map *map); + +/* Get all messages from file */ +int mdbox_map_get_file_msgs(struct mdbox_map *map, uint32_t file_id, + ARRAY_TYPE(mdbox_map_file_msg) *recs); + +/* Begin atomic context. There can be multiple transactions/appends within the + same atomic context. */ +struct mdbox_map_atomic_context *mdbox_map_atomic_begin(struct mdbox_map *map); +/* Lock the map immediately. */ +int mdbox_map_atomic_lock(struct mdbox_map_atomic_context *atomic, + const char *reason); +/* Returns TRUE if map is locked */ +bool mdbox_map_atomic_is_locked(struct mdbox_map_atomic_context *atomic); +/* When finish() is called, rollback the changes. If data was already written + to map's transaction log, this desyncs the map and causes a rebuild */ +void mdbox_map_atomic_set_failed(struct mdbox_map_atomic_context *atomic); +/* Mark this atomic as having succeeded. This is internally done if + transaction or append is committed within this atomic, but not when the + atomic is used standalone. */ +void mdbox_map_atomic_set_success(struct mdbox_map_atomic_context *atomic); +/* Remove fsck'd flag. */ +void mdbox_map_atomic_unset_fscked(struct mdbox_map_atomic_context *atomic); +/* Commit/rollback changes within this atomic context. */ +int mdbox_map_atomic_finish(struct mdbox_map_atomic_context **atomic); + +struct mdbox_map_transaction_context * +mdbox_map_transaction_begin(struct mdbox_map_atomic_context *atomic, + bool external); +/* Write transaction to map and leave it locked. Call _free() to update tail + offset and unlock. */ +int mdbox_map_transaction_commit(struct mdbox_map_transaction_context *ctx, + const char *reason); +void mdbox_map_transaction_free(struct mdbox_map_transaction_context **ctx); + +int mdbox_map_update_refcount(struct mdbox_map_transaction_context *ctx, + uint32_t map_uid, int diff); +int mdbox_map_update_refcounts(struct mdbox_map_transaction_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, int diff); +int mdbox_map_remove_file_id(struct mdbox_map *map, uint32_t file_id); + +/* Return all files containing messages with zero refcount. */ +int mdbox_map_get_zero_ref_files(struct mdbox_map *map, + ARRAY_TYPE(seq_range) *file_ids_r); + +struct mdbox_map_append_context * +mdbox_map_append_begin(struct mdbox_map_atomic_context *atomic); +/* Request file for saving a new message with given size (if available). If an + existing file can be used, the record is locked and updated in index. + Returns 0 if ok, -1 if error. */ +int mdbox_map_append_next(struct mdbox_map_append_context *ctx, uoff_t mail_size, + enum mdbox_map_append_flags flags, + struct dbox_file_append_context **file_append_ctx_r, + struct ostream **output_r); +/* Finished saving the last mail. Saves the message size. */ +void mdbox_map_append_finish(struct mdbox_map_append_context *ctx); +/* Abort saving the last mail. */ +void mdbox_map_append_abort(struct mdbox_map_append_context *ctx); +/* Assign map UIDs to all appended msgs to multi-files. */ +int mdbox_map_append_assign_map_uids(struct mdbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r); +/* The appends are existing messages that were simply moved to a new file. + map_uids contains the moved messages' map UIDs. */ +int mdbox_map_append_move(struct mdbox_map_append_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, + const ARRAY_TYPE(seq_range) *expunge_map_uids); +/* Flush/fsync appends. */ +int mdbox_map_append_flush(struct mdbox_map_append_context *ctx); +/* Returns 0 if ok, -1 if error. */ +int mdbox_map_append_commit(struct mdbox_map_append_context *ctx); +void mdbox_map_append_free(struct mdbox_map_append_context **ctx); + +/* Returns map's uidvalidity */ +uint32_t mdbox_map_get_uid_validity(struct mdbox_map *map); + +void mdbox_map_set_corrupted(struct mdbox_map *map, const char *format, ...) + ATTR_FORMAT(2, 3); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c new file mode 100644 index 0000000..8461074 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c @@ -0,0 +1,690 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "hash.h" +#include "dbox-attachment.h" +#include "mdbox-storage.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-file.h" +#include "mdbox-map.h" +#include "mdbox-sync.h" + +#include <dirent.h> + +/* + Altmoving works like: + + 1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox + code and map UID's alt-refcount is updated. It won't be written to disk. + 2. mdbox_purge() is called, which checks if map UID's refcount equals + to its alt-refcount. If it does, it's moved to alt storage. Moving to + primary storage is done if _ALT flag was removed from any message. +*/ + +enum mdbox_msg_action { + MDBOX_MSG_ACTION_MOVE_TO_ALT = 1, + MDBOX_MSG_ACTION_MOVE_FROM_ALT +}; + +struct mdbox_purge_context { + pool_t pool; + struct mdbox_storage *storage; + + uint32_t lowest_primary_file_id; + /* list of file_ids that exist in primary storage. this list is looked + up while there is no locking, so it may not be accurate anymore by + the time it's used. */ + ARRAY_TYPE(seq_range) primary_file_ids; + /* list of file_ids that we need to purge */ + ARRAY_TYPE(seq_range) purge_file_ids; + + /* uint32_t map_uid => enum mdbox_msg_action action */ + HASH_TABLE(void *, void *) altmoves; + bool have_altmoves; + + struct mdbox_map_atomic_context *atomic; + struct mdbox_map_append_context *append_ctx; +}; + +static int mdbox_map_file_msg_offset_cmp(const struct mdbox_map_file_msg *m1, + const struct mdbox_map_file_msg *m2) +{ + if (m1->offset < m2->offset) + return -1; + else if (m1->offset > m2->offset) + return 1; + else + return 0; +} + +static int +mdbox_file_read_metadata_hdr(struct dbox_file *file, + struct dbox_metadata_header *meta_hdr_r) +{ + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(file->input, &data, &size, + sizeof(*meta_hdr_r)); + if (ret <= 0) { + i_assert(ret == -1); + if (file->input->stream_errno == 0) { + dbox_file_set_corrupted(file, "missing metadata"); + return 0; + } + mail_storage_set_critical(&file->storage->storage, + "read(%s) failed: %s", file->cur_path, + i_stream_get_error(file->input)); + return -1; + } + + memcpy(meta_hdr_r, data, sizeof(*meta_hdr_r)); + if (memcmp(meta_hdr_r->magic_post, DBOX_MAGIC_POST, + sizeof(meta_hdr_r->magic_post)) != 0) { + dbox_file_set_corrupted(file, "invalid metadata magic"); + return 0; + } + i_stream_skip(file->input, sizeof(*meta_hdr_r)); + return 1; +} + +static int +mdbox_file_metadata_copy(struct dbox_file *file, struct ostream *output) +{ + struct dbox_metadata_header meta_hdr; + const char *line; + size_t buf_size; + int ret; + + if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0) + return ret; + + o_stream_nsend(output, &meta_hdr, sizeof(meta_hdr)); + buf_size = i_stream_get_max_buffer_size(file->input); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == '\0') { + /* end of metadata */ + break; + } + o_stream_nsend_str(output, line); + o_stream_nsend(output, "\n", 1); + } + i_stream_set_max_buffer_size(file->input, buf_size); + + if (line == NULL) { + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return 0; + } + o_stream_nsend(output, "\n", 1); + return 1; +} + +static int +mdbox_metadata_get_extrefs(struct dbox_file *file, pool_t ext_refs_pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + struct dbox_metadata_header meta_hdr; + const char *line; + size_t buf_size; + int ret; + + /* skip and ignore the header */ + if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0) + return ret; + + buf_size = i_stream_get_max_buffer_size(file->input); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == '\0') { + /* end of metadata */ + break; + } + if (*line == DBOX_METADATA_EXT_REF) T_BEGIN { + if (!index_attachment_parse_extrefs(line+1, ext_refs_pool, + extrefs)) { + i_warning("%s: Ignoring corrupted extref: %s", + file->cur_path, line); + } + } T_END; + } + i_stream_set_max_buffer_size(file->input, buf_size); + + if (line == NULL) { + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return 0; + } + return 1; +} + +static bool +mdbox_purge_want_altpath(struct mdbox_purge_context *ctx, + struct dbox_file *file, uint32_t map_uid) +{ + enum mdbox_msg_action action; + void *value; + + if (dbox_file_is_in_alt(file)) + return TRUE; + + if (!ctx->have_altmoves) + return FALSE; + + value = hash_table_lookup(ctx->altmoves, POINTER_CAST(map_uid)); + action = POINTER_CAST_TO(value, enum mdbox_msg_action); + return action == MDBOX_MSG_ACTION_MOVE_TO_ALT; +} + +static int +mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file, + const struct mdbox_map_file_msg *msg) +{ + struct dbox_file_append_context *out_file_append; + struct istream *input; + struct ostream *output; + enum mdbox_map_append_flags append_flags; + uoff_t msg_size; + int ret; + + if (ctx->append_ctx == NULL) + ctx->append_ctx = mdbox_map_append_begin(ctx->atomic); + + append_flags = !mdbox_purge_want_altpath(ctx, file, msg->map_uid) ? 0 : + DBOX_MAP_APPEND_FLAG_ALT; + msg_size = file->msg_header_size + file->cur_physical_size; + if (mdbox_map_append_next(ctx->append_ctx, file->cur_physical_size, + append_flags, &out_file_append, &output) < 0) + return -1; + + i_assert(file != out_file_append->file); + + input = i_stream_create_limit(file->input, msg_size); + o_stream_nsend_istream(output, input); + if (o_stream_flush(output) < 0) { + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %s", + out_file_append->file->cur_path, + o_stream_get_error(output)); + ret = -1; + } else if (input->v_offset != msg_size) { + i_assert(input->v_offset < msg_size); + i_assert(i_stream_read_eof(file->input)); + + dbox_file_set_corrupted(file, "truncated message at EOF"); + ret = 0; + } else { + ret = 1; + } + i_stream_unref(&input); + + if (ret > 0) { + /* copy metadata */ + if ((ret = mdbox_file_metadata_copy(file, output)) <= 0) + return ret; + + mdbox_map_append_finish(ctx->append_ctx); + } + return ret; +} + +static int +mdbox_file_purge_check_refcounts(struct mdbox_purge_context *ctx, + const ARRAY_TYPE(mdbox_map_file_msg) *msgs_arr) +{ + struct mdbox_map *map = ctx->storage->map; + struct mdbox_map_mail_index_record rec; + uint16_t refcount; + const struct mdbox_map_file_msg *msgs; + unsigned int i, count; + int ret; + + if (mdbox_map_atomic_lock(ctx->atomic, "purging check") < 0) + return -1; + + msgs = array_get(msgs_arr, &count); + for (i = 0; i < count; i++) { + if (msgs[i].refcount != 0) + continue; + + ret = mdbox_map_lookup_full(map, msgs[i].map_uid, &rec, + &refcount); + if (ret <= 0) { + if (ret < 0) + return -1; + mdbox_map_set_corrupted(map, + "Purging unexpectedly lost map_uid=%u", + msgs[i].map_uid); + return -1; + } + if (refcount > 0) + return 0; + } + return 1; +} + +static int +mdbox_purge_attachments(struct mdbox_purge_context *ctx, + const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr) +{ + struct dbox_storage *storage = &ctx->storage->storage; + const struct mail_attachment_extref *extref; + int ret = 0; + + array_foreach(extrefs_arr, extref) { + if (index_attachment_delete(&storage->storage, + storage->attachment_fs, + extref->path) < 0) + ret = -1; + } + return ret; +} + +static int +mdbox_file_purge(struct mdbox_purge_context *ctx, struct dbox_file *file, + uint32_t file_id) +{ + struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage; + struct stat st; + ARRAY_TYPE(mdbox_map_file_msg) msgs_arr; + const struct mdbox_map_file_msg *msgs; + ARRAY_TYPE(seq_range) expunged_map_uids; + ARRAY_TYPE(uint32_t) copied_map_uids; + ARRAY_TYPE(mail_attachment_extref) ext_refs; + pool_t ext_refs_pool; + unsigned int i, count; + uoff_t offset; + int ret; + + i_assert(ctx->atomic == NULL); + i_assert(ctx->append_ctx == NULL); + + if ((ret = dbox_file_try_lock(file)) <= 0) + return ret; + + /* make sure the file still exists. another process may have already + deleted it. */ + if (stat(file->cur_path, &st) < 0) { + dbox_file_unlock(file); + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(&file->storage->storage, + "stat(%s) failed: %m", file->cur_path); + return -1; + } + + /* get list of map UIDs that exist in this file (again has to be done + after locking) */ + i_array_init(&msgs_arr, 128); + if (mdbox_map_get_file_msgs(dstorage->map, file_id, + &msgs_arr) < 0) { + array_free(&msgs_arr); + dbox_file_unlock(file); + return -1; + } + /* sort messages by their offset */ + array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp); + + ext_refs_pool = pool_alloconly_create("mdbox purge ext refs", 1024); + ctx->atomic = mdbox_map_atomic_begin(ctx->storage->map); + msgs = array_get(&msgs_arr, &count); + i_array_init(&ext_refs, 32); + i_array_init(&copied_map_uids, I_MIN(count, 1)); + i_array_init(&expunged_map_uids, I_MIN(count, 1)); + offset = file->file_header_size; + for (i = 0; i < count; i++) { + if ((ret = dbox_file_seek(file, offset)) <= 0) + break; + + if (msgs[i].offset != offset) { + /* map doesn't match file's actual contents */ + dbox_file_set_corrupted(file, + "purging found mismatched offsets " + "(%"PRIuUOFF_T" vs %u, %u/%u)", + offset, msgs[i].offset, i, count); + ret = 0; + break; + } + + if (msgs[i].refcount == 0) { + /* skip over expunged message */ + i_stream_seek(file->input, offset + + file->msg_header_size + + file->cur_physical_size); + /* skip metadata */ + ret = mdbox_metadata_get_extrefs(file, ext_refs_pool, + &ext_refs); + if (ret <= 0) + break; + seq_range_array_add(&expunged_map_uids, + msgs[i].map_uid); + } else { + /* non-expunged message. write it to output file. */ + i_stream_seek(file->input, offset); + ret = mdbox_purge_save_msg(ctx, file, &msgs[i]); + if (ret <= 0) + break; + array_push_back(&copied_map_uids, &msgs[i].map_uid); + } + offset = file->input->v_offset; + } + if (offset != (uoff_t)st.st_size && ret > 0) { + /* file has more messages than what map tells us */ + dbox_file_set_corrupted(file, + "more messages available than in map " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", offset, st.st_size); + ret = 0; + } + if (ret > 0 && ctx->append_ctx != NULL) { + /* flush writes before locking the map */ + if (mdbox_map_append_flush(ctx->append_ctx) < 0) + ret = -1; + } + + if (ret <= 0) + ret = -1; + else { + /* it's possible that one of the messages we purged was + just copied to another mailbox. the only way to prevent that + would be to keep map locked during the purge, but that could + keep it locked for too long. instead we'll check here if + there are such copies, and if there are cancel this file's + purge. */ + ret = mdbox_file_purge_check_refcounts(ctx, &msgs_arr); + } + array_free(&msgs_arr); msgs = NULL; + + if (ret <= 0) { + /* failed */ + } else if (ctx->append_ctx == NULL) { + /* everything purged from this file */ + ret = 1; + } else { + /* assign new file_id + offset to moved messages */ + if (mdbox_map_append_move(ctx->append_ctx, &copied_map_uids, + &expunged_map_uids) < 0 || + mdbox_map_append_commit(ctx->append_ctx) < 0) + ret = -1; + else + ret = 1; + } + if (ctx->append_ctx != NULL) + mdbox_map_append_free(&ctx->append_ctx); + (void)mdbox_map_atomic_finish(&ctx->atomic); + + /* unlink only after unlocking map, so readers don't see it + temporarily vanished */ + if (ret > 0) { + (void)dbox_file_unlink(file); + if (mdbox_map_remove_file_id(ctx->storage->map, file_id) < 0) + ret = -1; + } else { + dbox_file_unlock(file); + } + array_free(&copied_map_uids); + array_free(&expunged_map_uids); + + (void)mdbox_purge_attachments(ctx, &ext_refs); + array_free(&ext_refs); + pool_unref(&ext_refs_pool); + return ret; +} + +void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->box); + ARRAY_TYPE(uint32_t) *dest; + uint32_t map_uid; + + /* we'll assume here that alt flag won't be changed multiple times + for the same mail. it shouldn't happen with current code, and + checking for it would just slow down the code. + + so the way it works currently is just that map_uids are added to + an array, which is later sorted and processed further. note that + it's still possible that the same map_uid exists in the array + multiple times. */ + if (mdbox_mail_lookup(mbox, mbox->box.view, mail->seq, &map_uid) < 0) + return; + + dest = move_to_alt ? &mbox->storage->move_to_alt_map_uids : + &mbox->storage->move_from_alt_map_uids; + + if (!array_is_created(dest)) + i_array_init(dest, 256); + array_push_back(dest, &map_uid); +} + +static struct mdbox_purge_context * +mdbox_purge_alloc(struct mdbox_storage *storage) +{ + struct mdbox_purge_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("mdbox purge context", 1024*32); + ctx = p_new(pool, struct mdbox_purge_context, 1); + ctx->pool = pool; + ctx->storage = storage; + ctx->lowest_primary_file_id = (uint32_t)-1; + i_array_init(&ctx->primary_file_ids, 64); + i_array_init(&ctx->purge_file_ids, 64); + hash_table_create_direct(&ctx->altmoves, pool, 0); + return ctx; +} + +static void mdbox_purge_free(struct mdbox_purge_context **_ctx) +{ + struct mdbox_purge_context *ctx = *_ctx; + + *_ctx = NULL; + + hash_table_destroy(&ctx->altmoves); + array_free(&ctx->primary_file_ids); + array_free(&ctx->purge_file_ids); + pool_unref(&ctx->pool); +} + +static int mdbox_purge_get_primary_files(struct mdbox_purge_context *ctx) +{ + struct mdbox_storage *dstorage = ctx->storage; + struct mail_storage *storage = &dstorage->storage.storage; + DIR *dir; + struct dirent *d; + string_t *path; + unsigned int file_id; + size_t dir_len; + int ret = 0; + + if (!array_is_created(&dstorage->move_to_alt_map_uids) && + !array_is_created(&dstorage->move_from_alt_map_uids)) { + /* we don't need to do alt moving, don't bother getting list + of primary files */ + return 0; + } + + dir = opendir(dstorage->storage_dir); + if (dir == NULL) { + if (errno == ENOENT) { + /* no storage directory at all yet */ + return 0; + } + mail_storage_set_critical(storage, + "opendir(%s) failed: %m", dstorage->storage_dir); + return -1; + } + + path = t_str_new(256); + str_append(path, dstorage->storage_dir); + str_append_c(path, '/'); + dir_len = str_len(path); + + for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) { + if (!str_begins(d->d_name, MDBOX_MAIL_FILE_PREFIX)) + continue; + if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX), + &file_id) < 0) + continue; + + str_truncate(path, dir_len); + str_append(path, d->d_name); + seq_range_array_add(&ctx->primary_file_ids, file_id); + } + if (array_count(&ctx->primary_file_ids) > 0) { + const struct seq_range *range = + array_front(&ctx->primary_file_ids); + ctx->lowest_primary_file_id = range[0].seq1; + } + if (errno != 0) { + mail_storage_set_critical(storage, + "readdir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(storage, + "closedir(%s) failed: %m", dstorage->storage_dir); + ret = -1; + } + return ret; +} + +static int uint32_t_cmp(const uint32_t *u1, const uint32_t *u2) +{ + if (*u1 < *u2) + return -1; + if (*u1 > *u2) + return 1; + return 0; +} + +static int mdbox_altmove_add_files(struct mdbox_purge_context *ctx) +{ + struct mdbox_storage *dstorage = ctx->storage; + const uint32_t *map_uids; + unsigned int i, count, alt_refcount = 0; + struct mdbox_map_mail_index_record cur_rec; + enum mdbox_msg_action action; + uint32_t cur_map_uid; + uint16_t cur_refcount = 0; + uoff_t offset; + int ret = 0; + + /* first add move-to-alt actions */ + if (array_is_created(&dstorage->move_to_alt_map_uids)) { + array_sort(&dstorage->move_to_alt_map_uids, uint32_t_cmp); + map_uids = array_get(&dstorage->move_to_alt_map_uids, &count); + } else { + map_uids = NULL; + count = 0; + } + cur_map_uid = 0; + for (i = 0; i < count; i++) { + if (cur_map_uid != map_uids[i]) { + cur_map_uid = map_uids[i]; + if (mdbox_map_lookup_full(dstorage->map, cur_map_uid, + &cur_rec, &cur_refcount) < 0) { + cur_refcount = (uint16_t)-1; + ret = -1; + } + alt_refcount = 1; + } else { + alt_refcount++; + } + + if (alt_refcount == cur_refcount && + seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) { + /* all instances marked as moved to alt storage */ + action = MDBOX_MSG_ACTION_MOVE_TO_ALT; + hash_table_insert(ctx->altmoves, + POINTER_CAST(cur_map_uid), + POINTER_CAST(action)); + seq_range_array_add(&ctx->purge_file_ids, + cur_rec.file_id); + } + } + + /* next add move-from-alt actions. they override move-to-alt actions + in case there happen to be any conflicts (shouldn't). only a single + move-from-alt record is needed to do the move. */ + if (array_is_created(&dstorage->move_from_alt_map_uids)) + map_uids = array_get(&dstorage->move_from_alt_map_uids, &count); + else { + map_uids = NULL; + count = 0; + } + cur_map_uid = 0; + for (i = 0; i < count; i++) { + if (cur_map_uid == map_uids[i]) + continue; + cur_map_uid = map_uids[i]; + + if (mdbox_map_lookup(dstorage->map, cur_map_uid, + &cur_rec.file_id, &offset) < 0) { + ret = -1; + continue; + } + if (seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) { + /* already in primary storage */ + continue; + } + + action = MDBOX_MSG_ACTION_MOVE_FROM_ALT; + hash_table_update(ctx->altmoves, POINTER_CAST(cur_map_uid), + POINTER_CAST(action)); + seq_range_array_add(&ctx->purge_file_ids, cur_rec.file_id); + } + ctx->have_altmoves = hash_table_count(ctx->altmoves) > 0; + return ret; +} + +int mdbox_purge(struct mail_storage *_storage) +{ + struct mdbox_storage *storage = (struct mdbox_storage *)_storage; + struct mdbox_purge_context *ctx; + struct dbox_file *file; + struct seq_range_iter iter; + unsigned int i = 0; + uint32_t file_id; + bool deleted; + int ret; + + ctx = mdbox_purge_alloc(storage); + ret = mdbox_map_get_zero_ref_files(storage->map, &ctx->purge_file_ids); + if (storage->alt_storage_dir != NULL) { + if (mdbox_purge_get_primary_files(ctx) < 0) + ret = -1; + else { + /* add files that can be altmoved */ + if (mdbox_altmove_add_files(ctx) < 0) + ret = -1; + } + } + + seq_range_array_iter_init(&iter, &ctx->purge_file_ids); i = 0; + while (ret == 0 && + seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN { + file = mdbox_file_init(storage, file_id); + if (dbox_file_open(file, &deleted) > 0 && !deleted) { + if (mdbox_file_purge(ctx, file, file_id) < 0) + ret = -1; + } else { + if (mdbox_map_remove_file_id(storage->map, file_id) < 0) + ret = -1; + } + dbox_file_unref(&file); + } T_END; + mdbox_purge_free(&ctx); + + if (storage->corrupted) { + /* purging found corrupted files */ + (void)mdbox_storage_rebuild(storage); + ret = -1; + } + return ret; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c new file mode 100644 index 0000000..c0dcf5b --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c @@ -0,0 +1,491 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "fdatasync-path.h" +#include "hex-binary.h" +#include "hex-dec.h" +#include "str.h" +#include "istream.h" +#include "istream-crlf.h" +#include "ostream.h" +#include "write-full.h" +#include "index-mail.h" +#include "index-pop3-uidl.h" +#include "mail-copy.h" +#include "dbox-save.h" +#include "mdbox-storage.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" + + +struct dbox_save_mail { + struct dbox_file_append_context *file_append; + uint32_t seq; + uint32_t append_offset; + time_t save_date; + bool written_to_disk; +}; + +struct mdbox_save_context { + struct dbox_save_context ctx; + + struct mdbox_mailbox *mbox; + struct mdbox_sync_context *sync_ctx; + + struct dbox_file *cur_file; + struct dbox_file_append_context *cur_file_append; + struct mdbox_map_append_context *append_ctx; + + ARRAY_TYPE(uint32_t) copy_map_uids; + struct mdbox_map_atomic_context *atomic; + struct mdbox_map_transaction_context *map_trans; + + ARRAY(struct dbox_save_mail) mails; +}; + +#define MDBOX_SAVECTX(s) container_of(DBOX_SAVECTX(s), struct mdbox_save_context, ctx) + +static struct dbox_file * +mdbox_copy_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(t->save_ctx); + const struct mdbox_mail_index_record *rec; + const void *data; + uint32_t file_id; + + mail_index_lookup_ext(t->view, seq, ctx->mbox->ext_id, &data, NULL); + rec = data; + + if (mdbox_map_lookup(ctx->mbox->storage->map, rec->map_uid, + &file_id, offset_r) < 0) + i_unreached(); + + return mdbox_file_init(ctx->mbox->storage, file_id); +} + +struct dbox_file * +mdbox_save_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(t->save_ctx); + const struct dbox_save_mail *mails, *mail; + unsigned int count; + + mails = array_get(&ctx->mails, &count); + i_assert(count > 0); + i_assert(seq >= mails[0].seq); + + mail = &mails[seq - mails[0].seq]; + i_assert(mail->seq == seq); + + if (mail->file_append == NULL) { + /* copied mail */ + return mdbox_copy_file_get_file(t, seq, offset_r); + } + + /* saved mail */ + i_assert(mail->written_to_disk); + if (dbox_file_append_flush(mail->file_append) < 0) + ctx->ctx.failed = TRUE; + + mail->file_append->file->refcount++; + *offset_r = mail->append_offset; + return mail->file_append->file; +} + +struct mail_save_context * +mdbox_save_alloc(struct mailbox_transaction_context *t) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(t->box); + struct mdbox_save_context *ctx; + + i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); + + if (t->save_ctx != NULL) { + /* use the existing allocated structure */ + ctx = MDBOX_SAVECTX(t->save_ctx); + ctx->cur_file = NULL; + ctx->ctx.failed = FALSE; + ctx->ctx.finished = FALSE; + ctx->ctx.dbox_output = NULL; + ctx->cur_file_append = NULL; + return &ctx->ctx.ctx; + } + + ctx = i_new(struct mdbox_save_context, 1); + ctx->ctx.ctx.transaction = t; + ctx->ctx.trans = t->itrans; + ctx->mbox = mbox; + ctx->atomic = mdbox_map_atomic_begin(mbox->storage->map); + ctx->append_ctx = mdbox_map_append_begin(ctx->atomic); + i_array_init(&ctx->mails, 32); + t->save_ctx = &ctx->ctx.ctx; + return t->save_ctx; +} + +int mdbox_save_begin(struct mail_save_context *_ctx, struct istream *input) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct dbox_save_mail *save_mail; + uoff_t mail_size, append_offset; + + /* get the size of the mail to be saved, if possible */ + if (i_stream_get_size(input, TRUE, &mail_size) <= 0) { + /* we couldn't find out the exact size. fallback to non-exact, + maybe it'll give something useful. the mail size is used + only to figure out if it's causing mdbox file to grow + too large. */ + if (i_stream_get_size(input, FALSE, &mail_size) <= 0) + mail_size = 0; + } + if (mdbox_map_append_next(ctx->append_ctx, mail_size, 0, + &ctx->cur_file_append, + &ctx->ctx.dbox_output) < 0) { + ctx->ctx.failed = TRUE; + return -1; + } + i_assert(ctx->ctx.dbox_output->offset <= (uint32_t)-1); + append_offset = ctx->ctx.dbox_output->offset; + + ctx->cur_file = ctx->cur_file_append->file; + dbox_save_begin(&ctx->ctx, input); + + save_mail = array_append_space(&ctx->mails); + save_mail->file_append = ctx->cur_file_append; + save_mail->seq = ctx->ctx.seq; + save_mail->append_offset = append_offset; + return ctx->ctx.failed ? -1 : 0; +} + +static int mdbox_save_mail_write_metadata(struct mdbox_save_context *ctx, + struct dbox_save_mail *mail) +{ + struct dbox_file *file = mail->file_append->file; + struct dbox_message_header dbox_msg_hdr; + uoff_t message_size; + guid_128_t guid_128; + + i_assert(file->msg_header_size == sizeof(dbox_msg_hdr)); + + message_size = ctx->ctx.dbox_output->offset - + mail->append_offset - mail->file_append->file->msg_header_size; + + dbox_save_write_metadata(&ctx->ctx.ctx, ctx->ctx.dbox_output, + message_size, ctx->mbox->box.name, guid_128); + /* save the 128bit GUID to index so if the map index gets corrupted + we can still find the message */ + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_128, NULL); + + dbox_msg_header_fill(&dbox_msg_hdr, message_size); + if (o_stream_pwrite(ctx->ctx.dbox_output, &dbox_msg_hdr, + sizeof(dbox_msg_hdr), mail->append_offset) < 0) { + dbox_file_set_syscall_error(file, "pwrite()"); + return -1; + } + mail->written_to_disk = TRUE; + mail->save_date = ctx->ctx.ctx.data.save_date; + return 0; +} + +static int mdbox_save_finish_write(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx; + struct dbox_save_mail *mail; + + ctx->ctx.finished = TRUE; + if (ctx->ctx.dbox_output == NULL) + return -1; + + dbox_save_end(&ctx->ctx); + + mail = array_back_modifiable(&ctx->mails); + if (!ctx->ctx.failed) T_BEGIN { + if (mdbox_save_mail_write_metadata(ctx, mail) < 0) + ctx->ctx.failed = TRUE; + else + mdbox_map_append_finish(ctx->append_ctx); + } T_END; + + if (mail->file_append->file->input != NULL) { + /* if we try to read the saved mail before unlocking file, + make sure the input stream doesn't have stale data */ + i_stream_sync(mail->file_append->file->input); + } + i_stream_unref(&ctx->ctx.input); + + if (ctx->ctx.failed) { + index_storage_save_abort_last(&ctx->ctx.ctx, ctx->ctx.seq); + mdbox_map_append_abort(ctx->append_ctx); + array_pop_back(&ctx->mails); + return -1; + } + return 0; +} + +int mdbox_save_finish(struct mail_save_context *ctx) +{ + int ret; + + ret = mdbox_save_finish_write(ctx); + index_save_context_free(ctx); + return ret; +} + +void mdbox_save_cancel(struct mail_save_context *_ctx) +{ + struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx); + + ctx->failed = TRUE; + (void)mdbox_save_finish(_ctx); +} + +static void +mdbox_save_set_map_uids(struct mdbox_save_context *ctx, + uint32_t first_map_uid, uint32_t last_map_uid) +{ + struct mdbox_mailbox *mbox = ctx->mbox; + struct mail_index_view *view = ctx->ctx.ctx.transaction->view; + const struct mdbox_mail_index_record *old_rec; + struct mdbox_mail_index_record rec; + const struct dbox_save_mail *mails; + unsigned int i, count; + const void *data; + uint32_t next_map_uid = first_map_uid; + + mdbox_update_header(mbox, ctx->ctx.trans, NULL); + + i_zero(&rec); + mails = array_get(&ctx->mails, &count); + for (i = 0; i < count; i++) { + mail_index_lookup_ext(view, mails[i].seq, mbox->ext_id, + &data, NULL); + old_rec = data; + if (old_rec != NULL && old_rec->map_uid != 0) { + /* message was copied. keep the existing map uid */ + continue; + } + + if (mails[i].save_date > 0) + rec.save_date = mails[i].save_date; + else + rec.save_date = ioloop_time; + rec.map_uid = next_map_uid++; + mail_index_update_ext(ctx->ctx.trans, mails[i].seq, + mbox->ext_id, &rec, NULL); + } + i_assert(next_map_uid == last_map_uid + 1); +} + +int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct mailbox_transaction_context *_t = _ctx->transaction; + const struct mail_index_header *hdr; + uint32_t first_map_uid, last_map_uid; + + i_assert(ctx->ctx.finished); + + /* flush/fsync writes to m.* files before locking the map */ + if (mdbox_map_append_flush(ctx->append_ctx) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* make sure the map gets locked */ + if (mdbox_map_atomic_lock(ctx->atomic, "saving") < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + /* lock the mailbox after map to avoid deadlocks. if we've noticed + any corruption, deal with it later, otherwise we won't have + up-to-date atomic->sync_view */ + if (mdbox_sync_begin(ctx->mbox, MDBOX_SYNC_FLAG_NO_PURGE | + MDBOX_SYNC_FLAG_FORCE | + MDBOX_SYNC_FLAG_FSYNC | + MDBOX_SYNC_FLAG_NO_REBUILD, ctx->atomic, + &ctx->sync_ctx) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + i_assert(ctx->sync_ctx != NULL); + + /* assign map UIDs for newly saved messages after we've successfully + acquired all the locks. the transaction is now very unlikely to + fail. the UIDs are written to the transaction log immediately within + this function, but the map is left locked. */ + if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + + /* update dbox header flags */ + dbox_save_update_header_flags(&ctx->ctx, ctx->sync_ctx->sync_view, + ctx->mbox->hdr_ext_id, offsetof(struct mdbox_index_header, flags)); + + /* assign UIDs for new messages */ + hdr = mail_index_get_header(ctx->sync_ctx->sync_view); + mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, + &_t->changes->saved_uids); + + if (ctx->ctx.highest_pop3_uidl_seq != 0) { + const struct dbox_save_mail *mails; + struct seq_range_iter iter; + unsigned int highest_pop3_uidl_idx; + uint32_t uid; + + mails = array_front(&ctx->mails); + highest_pop3_uidl_idx = + ctx->ctx.highest_pop3_uidl_seq - mails[0].seq; + i_assert(mails[highest_pop3_uidl_idx].seq == ctx->ctx.highest_pop3_uidl_seq); + + seq_range_array_iter_init(&iter, &_t->changes->saved_uids); + if (!seq_range_array_iter_nth(&iter, highest_pop3_uidl_idx, &uid)) + i_unreached(); + index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid); + } + + /* save map UIDs to mailbox index */ + if (first_map_uid != 0) + mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid); + + /* increase map's refcount for copied mails */ + if (array_is_created(&ctx->copy_map_uids)) { + ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE); + if (mdbox_map_update_refcounts(ctx->map_trans, + &ctx->copy_map_uids, 1) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + mail_index_sync_set_reason(ctx->sync_ctx->index_sync_ctx, "copying"); + } else { + mail_index_sync_set_reason(ctx->sync_ctx->index_sync_ctx, "saving"); + } + + _t->changes->uid_validity = hdr->uid_validity; + return 0; +} + +void mdbox_transaction_save_commit_post(struct mail_save_context *_ctx, + struct mail_index_transaction_commit_result *result) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct mail_storage *storage = _ctx->transaction->box->storage; + + _ctx->transaction = NULL; /* transaction is already freed */ + + mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx, + result); + + /* finish writing the mailbox APPENDs */ + if (mdbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) { + /* commit refcount increases for copied mails */ + if (ctx->map_trans != NULL) { + if (mdbox_map_transaction_commit(ctx->map_trans, "copy refcount updates") < 0) + mdbox_map_atomic_set_failed(ctx->atomic); + } + /* flush file append writes */ + if (mdbox_map_append_commit(ctx->append_ctx) < 0) + mdbox_map_atomic_set_failed(ctx->atomic); + } + mdbox_map_append_free(&ctx->append_ctx); + /* update the sync tail offset, everything else + was already written at this point. */ + (void)mdbox_map_atomic_finish(&ctx->atomic); + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + const char *box_path = mailbox_get_path(&ctx->mbox->box); + + if (fdatasync_path(box_path) < 0) { + mail_set_critical(_ctx->dest_mail, + "fdatasync_path(%s) failed: %m", box_path); + } + } + mdbox_transaction_save_rollback(_ctx); +} + +void mdbox_transaction_save_rollback(struct mail_save_context *_ctx) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + + if (!ctx->ctx.finished) + mdbox_save_cancel(&ctx->ctx.ctx); + if (ctx->append_ctx != NULL) + mdbox_map_append_free(&ctx->append_ctx); + if (ctx->map_trans != NULL) + mdbox_map_transaction_free(&ctx->map_trans); + if (ctx->atomic != NULL) + (void)mdbox_map_atomic_finish(&ctx->atomic); + if (array_is_created(&ctx->copy_map_uids)) + array_free(&ctx->copy_map_uids); + + if (ctx->sync_ctx != NULL) + (void)mdbox_sync_finish(&ctx->sync_ctx, FALSE); + + array_free(&ctx->mails); + i_free(ctx); +} + +int mdbox_copy(struct mail_save_context *_ctx, struct mail *mail) +{ + struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx); + struct dbox_save_mail *save_mail; + struct mdbox_mailbox *src_mbox; + struct mdbox_mail_index_record rec; + const void *guid_data; + guid_128_t wanted_guid; + + ctx->ctx.finished = TRUE; + + if (mail->box->storage != _ctx->transaction->box->storage || + _ctx->transaction->box->disable_reflink_copy_to) + return mail_storage_copy(_ctx, mail); + src_mbox = MDBOX_MAILBOX(mail->box); + + i_zero(&rec); + rec.save_date = ioloop_time; + if (mdbox_mail_lookup(src_mbox, mail->transaction->view, mail->seq, + &rec.map_uid) < 0) { + index_save_context_free(_ctx); + return -1; + } + + mail_index_lookup_ext(mail->transaction->view, mail->seq, + src_mbox->guid_ext_id, &guid_data, NULL); + if (guid_data == NULL || guid_128_is_empty(guid_data)) { + /* missing GUID, something's broken. don't copy using + refcounting. */ + return mail_storage_copy(_ctx, mail); + } else if (_ctx->data.guid != NULL && + (guid_128_from_string(_ctx->data.guid, wanted_guid) < 0 || + memcmp(guid_data, wanted_guid, sizeof(wanted_guid)) != 0)) { + /* GUID change requested. we can't do it with refcount + copying */ + return mail_storage_copy(_ctx, mail); + } + + /* remember the map_uid so we can later increase its refcount */ + if (!array_is_created(&ctx->copy_map_uids)) + i_array_init(&ctx->copy_map_uids, 32); + array_push_back(&ctx->copy_map_uids, &rec.map_uid); + + /* add message to mailbox index */ + dbox_save_add_to_index(&ctx->ctx); + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->ext_id, &rec, NULL); + + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_data, NULL); + index_copy_cache_fields(_ctx, mail, ctx->ctx.seq); + + save_mail = array_append_space(&ctx->mails); + save_mail->seq = ctx->ctx.seq; + + mail_set_seq_saving(_ctx->dest_mail, ctx->ctx.seq); + index_save_context_free(_ctx); + return 0; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-settings.c b/src/lib-storage/index/dbox-multi/mdbox-settings.c new file mode 100644 index 0000000..dc9e989 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.c @@ -0,0 +1,43 @@ +/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "settings-parser.h" +#include "mail-storage-settings.h" +#include "mdbox-settings.h" + +#include <stddef.h> + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct mdbox_settings) + +static const struct setting_define mdbox_setting_defines[] = { + DEF(BOOL, mdbox_preallocate_space), + DEF(SIZE, mdbox_rotate_size), + DEF(TIME, mdbox_rotate_interval), + + SETTING_DEFINE_LIST_END +}; + +static const struct mdbox_settings mdbox_default_settings = { + .mdbox_preallocate_space = FALSE, + .mdbox_rotate_size = 10*1024*1024, + .mdbox_rotate_interval = 0 +}; + +static const struct setting_parser_info mdbox_setting_parser_info = { + .module_name = "mdbox", + .defines = mdbox_setting_defines, + .defaults = &mdbox_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct mdbox_settings), + + .parent_offset = SIZE_MAX, + .parent = &mail_user_setting_parser_info +}; + +const struct setting_parser_info *mdbox_get_setting_parser_info(void) +{ + return &mdbox_setting_parser_info; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-settings.h b/src/lib-storage/index/dbox-multi/mdbox-settings.h new file mode 100644 index 0000000..353da69 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.h @@ -0,0 +1,12 @@ +#ifndef MDBOX_SETTINGS_H +#define MDBOX_SETTINGS_H + +struct mdbox_settings { + bool mdbox_preallocate_space; + uoff_t mdbox_rotate_size; + unsigned int mdbox_rotate_interval; +}; + +const struct setting_parser_info *mdbox_get_setting_parser_info(void); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c new file mode 100644 index 0000000..4277a1a --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c @@ -0,0 +1,1005 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "hash.h" +#include "str.h" +#include "mail-cache.h" +#include "index-rebuild.h" +#include "mail-namespace.h" +#include "mailbox-list-private.h" +#include "mdbox-storage.h" +#include "mdbox-file.h" +#include "mdbox-map-private.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" + +#include <dirent.h> +#include <unistd.h> + +#define REBUILD_MAX_REFCOUNT 32768 + +struct mdbox_rebuild_msg { + struct mdbox_rebuild_msg *guid_hash_next; + + guid_128_t guid_128; + uint32_t file_id; + uint32_t offset; + uint32_t rec_size; + uoff_t mail_size; + uint32_t map_uid; + + uint16_t refcount; + bool seen_zero_ref_in_map:1; +}; + +struct rebuild_msg_mailbox { + struct mailbox *box; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + uint32_t next_uid; +}; + +struct mdbox_storage_rebuild_context { + struct mdbox_storage *storage; + struct mdbox_map_atomic_context *atomic; + pool_t pool; + + struct mdbox_map_mail_index_header orig_map_hdr; + HASH_TABLE(uint8_t *, struct mdbox_rebuild_msg *) guid_hash; + ARRAY(struct mdbox_rebuild_msg *) msgs; + ARRAY_TYPE(seq_range) seen_file_ids; + + uint32_t rebuild_count; + uint32_t highest_file_id; + + struct mailbox_list *default_list; + + struct rebuild_msg_mailbox prev_msg; + + bool have_pop3_uidls:1; + bool have_pop3_orders:1; +}; + +static struct mdbox_storage_rebuild_context * +mdbox_storage_rebuild_init(struct mdbox_storage *storage, + struct mdbox_map_atomic_context *atomic) +{ + struct mdbox_storage_rebuild_context *ctx; + + i_assert(!storage->rebuilding_storage); + + ctx = i_new(struct mdbox_storage_rebuild_context, 1); + ctx->storage = storage; + ctx->atomic = atomic; + ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256); + hash_table_create(&ctx->guid_hash, ctx->pool, 0, + guid_128_hash, guid_128_cmp); + i_array_init(&ctx->msgs, 512); + i_array_init(&ctx->seen_file_ids, 128); + + ctx->storage->rebuilding_storage = TRUE; + return ctx; +} + +static void +mdbox_storage_rebuild_deinit(struct mdbox_storage_rebuild_context *ctx) +{ + i_assert(ctx->storage->rebuilding_storage); + + ctx->storage->rebuilding_storage = FALSE; + + hash_table_destroy(&ctx->guid_hash); + pool_unref(&ctx->pool); + array_free(&ctx->seen_file_ids); + array_free(&ctx->msgs); + i_free(ctx); +} + +static int +mdbox_rebuild_msg_offset_cmp(struct mdbox_rebuild_msg *const *m1, + struct mdbox_rebuild_msg *const *m2) +{ + if ((*m1)->file_id < (*m2)->file_id) + return -1; + if ((*m1)->file_id > (*m2)->file_id) + return 1; + + if ((*m1)->offset < (*m2)->offset) + return -1; + if ((*m1)->offset > (*m2)->offset) + return 1; + + if ((*m1)->rec_size < (*m2)->rec_size) + return -1; + if ((*m1)->rec_size > (*m2)->rec_size) + return 1; + return 0; +} + +static int mdbox_rebuild_msg_uid_cmp(struct mdbox_rebuild_msg *const *m1, + struct mdbox_rebuild_msg *const *m2) +{ + if ((*m1)->map_uid < (*m2)->map_uid) + return -1; + if ((*m1)->map_uid > (*m2)->map_uid) + return 1; + return 0; +} + +static void rebuild_scan_metadata(struct mdbox_storage_rebuild_context *ctx, + struct dbox_file *file) +{ + if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_UIDL) != NULL) + ctx->have_pop3_uidls = TRUE; + if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_ORDER) != NULL) + ctx->have_pop3_orders = TRUE; +} + +static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx, + struct dbox_file *file, uint32_t file_id) +{ + const char *guid; + uint8_t *guid_p; + struct mdbox_rebuild_msg *rec, *old_rec; + uoff_t offset, prev_offset; + bool last, first, fixed = FALSE; + int ret; + + dbox_file_seek_rewind(file); + prev_offset = 0; + while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) { + if (ret > 0) { + if ((ret = dbox_file_metadata_read(file)) < 0) + break; + } + + if (ret == 0) { + /* file is corrupted. fix it and retry. */ + if (fixed || last) + break; + first = prev_offset == 0; + if (prev_offset == 0) { + /* use existing file header if it was ok */ + prev_offset = offset; + } + if ((ret = dbox_file_fix(file, prev_offset)) < 0) + break; + if (ret == 0) { + /* file was deleted */ + return 1; + } + fixed = TRUE; + if (!first) { + /* seek to the offset where we last left off */ + ret = dbox_file_seek(file, prev_offset); + if (ret <= 0) + break; + } + continue; + } + prev_offset = offset; + + guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID); + if (guid == NULL || *guid == '\0') { + dbox_file_set_corrupted(file, + "Message is missing GUID"); + ret = 0; + break; + } + rebuild_scan_metadata(ctx, file); + + rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1); + rec->file_id = file_id; + rec->offset = offset; + rec->rec_size = file->input->v_offset - offset; + rec->mail_size = dbox_file_get_plaintext_size(file); + mail_generate_guid_128_hash(guid, rec->guid_128); + i_assert(!guid_128_is_empty(rec->guid_128)); + array_push_back(&ctx->msgs, &rec); + + guid_p = rec->guid_128; + old_rec = hash_table_lookup(ctx->guid_hash, guid_p); + if (old_rec == NULL) + hash_table_insert(ctx->guid_hash, guid_p, rec); + else if (rec->mail_size == old_rec->mail_size) { + /* two mails' GUID and size are the same, which quite + likely means that their contents are the same as + well. we'll compare the mail sizes instead of the + record sizes, because the records' metadata may + differ. + + save this duplicate mail with refcount=0 to the map, + so it will eventually be purged. */ + rec->seen_zero_ref_in_map = TRUE; + } else { + /* duplicate GUID, but not a duplicate message. */ + i_error("mdbox %s: Duplicate GUID %s in " + "m.%u:%u (size=%"PRIuUOFF_T") and m.%u:%u " + "(size=%"PRIuUOFF_T")", + ctx->storage->storage_dir, guid, + old_rec->file_id, old_rec->offset, old_rec->mail_size, + rec->file_id, rec->offset, rec->mail_size); + rec->guid_hash_next = old_rec->guid_hash_next; + old_rec->guid_hash_next = rec; + } + } + if (ret < 0) + return -1; + else if (ret == 0 && !last) + return 0; + else + return 1; +} + +static int +rebuild_rename_file(struct mdbox_storage_rebuild_context *ctx, + const char *dir, const char **fname_p, uint32_t *file_id_r) +{ + const char *old_path, *new_path, *fname = *fname_p; + + old_path = t_strconcat(dir, "/", fname, NULL); + do { + new_path = t_strdup_printf("%s/"MDBOX_MAIL_FILE_FORMAT, + dir, ++ctx->highest_file_id); + /* use link()+unlink() instead of rename() to make sure we + don't overwrite any files. */ + if (link(old_path, new_path) == 0) { + i_unlink(old_path); + *fname_p = strrchr(new_path, '/') + 1; + *file_id_r = ctx->highest_file_id; + return 0; + } + } while (errno == EEXIST); + + i_error("link(%s, %s) failed: %m", old_path, new_path); + return -1; +} + +static int rebuild_add_file(struct mdbox_storage_rebuild_context *ctx, + const char *dir, const char *fname) +{ + struct dbox_file *file; + uint32_t file_id; + const char *id_str, *ext; + bool deleted; + int ret = 0; + + id_str = fname + strlen(MDBOX_MAIL_FILE_PREFIX); + if (str_to_uint32(id_str, &file_id) < 0 || file_id == 0) { + /* m.*.broken files are created by file fixing + m.*.lock files are created if flock() isn't available */ + ext = strrchr(id_str, '.'); + if (ext == NULL || (strcmp(ext, ".broken") != 0 && + strcmp(ext, ".lock") != 0)) { + i_warning("mdbox rebuild: " + "Skipping file with missing ID: %s/%s", + dir, fname); + } + return 0; + } + if (!seq_range_exists(&ctx->seen_file_ids, file_id)) { + if (ctx->highest_file_id < file_id) + ctx->highest_file_id = file_id; + } else { + /* duplicate file. either readdir() returned it twice + (unlikely) or it exists in both alt and primary storage. + to make sure we don't lose any mails from either of the + files, give this file a new ID and rename it. */ + if (rebuild_rename_file(ctx, dir, &fname, &file_id) < 0) + return -1; + } + seq_range_array_add(&ctx->seen_file_ids, file_id); + + file = mdbox_file_init(ctx->storage, file_id); + if ((ret = dbox_file_open(file, &deleted)) > 0 && !deleted) + ret = rebuild_file_mails(ctx, file, file_id); + if (ret == 0) + i_error("mdbox rebuild: Failed to fix file %s/%s", dir, fname); + dbox_file_unref(&file); + return ret < 0 ? -1 : 0; +} + +static void +rebuild_add_missing_map_uids(struct mdbox_storage_rebuild_context *ctx, + uint32_t next_uid) +{ + struct mdbox_rebuild_msg **msgs; + struct mdbox_map_mail_index_record rec; + unsigned int i, count; + uint32_t seq; + + i_zero(&rec); + msgs = array_get_modifiable(&ctx->msgs, &count); + for (i = 0; i < count; i++) { + if (msgs[i]->map_uid != 0) + continue; + + rec.file_id = msgs[i]->file_id; + rec.offset = msgs[i]->offset; + rec.size = msgs[i]->rec_size; + + msgs[i]->map_uid = next_uid++; + mail_index_append(ctx->atomic->sync_trans, + msgs[i]->map_uid, &seq); + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->storage->map->map_ext_id, + &rec, NULL); + } +} + +static void rebuild_apply_map(struct mdbox_storage_rebuild_context *ctx) +{ + struct mdbox_map *map = ctx->storage->map; + const struct mail_index_header *hdr; + struct mdbox_rebuild_msg **pos; + struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg; + struct dbox_mail_lookup_rec rec; + uint32_t seq; + + array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp); + /* msgs now contains a list of all messages that exists in m.* files, + sorted by file_id,offset */ + + hdr = mail_index_get_header(ctx->atomic->sync_view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + if (mdbox_map_view_lookup_rec(map, ctx->atomic->sync_view, + seq, &rec) < 0) { + /* map or ref extension is missing from the index. + Just ignore the file entirely. (Don't try to + continue with other records, since they'll fail + as well, and each failure logs the same error.) */ + i_assert(seq == 1); + break; + } + + /* look up the rebuild msg record for this message based on + the (file_id, offset, size) triplet */ + search_msg.file_id = rec.rec.file_id; + search_msg.offset = rec.rec.offset; + search_msg.rec_size = rec.rec.size; + pos = array_bsearch(&ctx->msgs, &search_msgp, + mdbox_rebuild_msg_offset_cmp); + if (pos == NULL || (*pos)->map_uid != 0) { + /* map record points to nonexistent or + a duplicate message. */ + mail_index_expunge(ctx->atomic->sync_trans, seq); + } else { + /* remember this message's map_uid */ + (*pos)->map_uid = rec.map_uid; + if (rec.refcount == 0) + (*pos)->seen_zero_ref_in_map = TRUE; + } + } + rebuild_add_missing_map_uids(ctx, hdr->next_uid); + + /* afterwards we're interested in looking up map_uids. + re-sort the messages to make it easier. */ + array_sort(&ctx->msgs, mdbox_rebuild_msg_uid_cmp); +} + +static struct mdbox_rebuild_msg * +rebuild_lookup_map_uid(struct mdbox_storage_rebuild_context *ctx, + uint32_t map_uid) +{ + struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg; + struct mdbox_rebuild_msg **pos; + + search_msg.map_uid = map_uid; + pos = array_bsearch(&ctx->msgs, &search_msgp, + mdbox_rebuild_msg_uid_cmp); + return pos == NULL ? NULL : *pos; +} + +static bool +guid_hash_have_map_uid(struct mdbox_rebuild_msg **recp, uint32_t map_uid) +{ + struct mdbox_rebuild_msg *rec; + + for (rec = *recp; rec != NULL; rec = rec->guid_hash_next) { + if (rec->map_uid == map_uid) { + *recp = rec; + return TRUE; + } + } + return FALSE; +} + +static void +rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx, + struct index_rebuild_context *rebuild_ctx, + struct mdbox_mailbox *mbox, + struct mail_index_view *view, + struct mail_index_transaction *trans) +{ + struct mdbox_mail_index_record new_dbox_rec; + const struct mail_index_header *hdr; + struct mdbox_rebuild_msg *rec; + const void *data; + const uint8_t *guid_p; + uint32_t old_seq, new_seq, uid, map_uid; + + /* Rebuild the mailbox's index. Note that index is reset at this point, + so although we can still access the old messages, we'll need to + append anything we want to keep as new messages. */ + hdr = mail_index_get_header(view); + for (old_seq = 1; old_seq <= hdr->messages_count; old_seq++) { + mail_index_lookup_ext(view, old_seq, mbox->ext_id, + &data, NULL); + if (data == NULL) { + i_zero(&new_dbox_rec); + map_uid = 0; + } else { + memcpy(&new_dbox_rec, data, sizeof(new_dbox_rec)); + map_uid = new_dbox_rec.map_uid; + } + + mail_index_lookup_ext(view, old_seq, mbox->guid_ext_id, + &data, NULL); + guid_p = data; + + /* see if we can find this message based on + 1) GUID, 2) map_uid */ + rec = guid_p == NULL ? NULL : + hash_table_lookup(ctx->guid_hash, guid_p); + if (rec == NULL) { + /* multi-dbox message that wasn't found with GUID. + either it's lost or GUID has been corrupted. we can + still try to look it up using map_uid. */ + rec = map_uid == 0 ? NULL : + rebuild_lookup_map_uid(ctx, map_uid); + map_uid = rec == NULL ? 0 : rec->map_uid; + } else if (!guid_hash_have_map_uid(&rec, map_uid)) { + /* message's GUID and map_uid point to different + physical messages. assume that GUID is correct and + map_uid is wrong. */ + map_uid = rec->map_uid; + } else { + /* everything was ok. use this specific record's + map_uid to avoid duplicating mails in case the same + GUID exists multiple times */ + } + + if (rec != NULL && + rec->refcount < REBUILD_MAX_REFCOUNT) T_BEGIN { + /* keep this message. add it to mailbox index. */ + i_assert(map_uid != 0); + rec->refcount++; + + mail_index_lookup_uid(view, old_seq, &uid); + mail_index_append(trans, uid, &new_seq); + index_rebuild_index_metadata(rebuild_ctx, + new_seq, uid); + + new_dbox_rec.map_uid = map_uid; + mail_index_update_ext(trans, new_seq, mbox->ext_id, + &new_dbox_rec, NULL); + mail_index_update_ext(trans, new_seq, mbox->guid_ext_id, + rec->guid_128, NULL); + } T_END; + } +} + +static void +mdbox_rebuild_get_header(struct mail_index_view *view, uint32_t hdr_ext_id, + struct mdbox_index_header *hdr_r, bool *need_resize_r) +{ + const void *data; + size_t data_size; + + mail_index_get_header_ext(view, hdr_ext_id, &data, &data_size); + i_zero(hdr_r); + memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r))); + *need_resize_r = data_size < sizeof(*hdr_r); +} + +static void mdbox_header_update(struct mdbox_storage_rebuild_context *ctx, + struct index_rebuild_context *rebuild_ctx, + struct mdbox_mailbox *mbox) +{ + struct mdbox_index_header hdr, backup_hdr; + bool need_resize, need_resize_backup; + + mdbox_rebuild_get_header(rebuild_ctx->view, mbox->hdr_ext_id, + &hdr, &need_resize); + if (rebuild_ctx->backup_view == NULL) { + i_zero(&backup_hdr); + need_resize = TRUE; + } else { + mdbox_rebuild_get_header(rebuild_ctx->backup_view, + mbox->hdr_ext_id, &backup_hdr, + &need_resize_backup); + } + + /* make sure we have valid mailbox guid */ + if (guid_128_is_empty(hdr.mailbox_guid)) { + if (!guid_128_is_empty(backup_hdr.mailbox_guid)) { + memcpy(hdr.mailbox_guid, backup_hdr.mailbox_guid, + sizeof(hdr.mailbox_guid)); + } else { + guid_128_generate(hdr.mailbox_guid); + } + } + + /* update map's uid-validity */ + hdr.map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map); + + if (ctx->have_pop3_uidls) + hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS; + if (ctx->have_pop3_orders) + hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS; + + /* and write changes */ + if (need_resize) { + mail_index_ext_resize_hdr(rebuild_ctx->trans, mbox->hdr_ext_id, + sizeof(hdr)); + } + mail_index_update_header_ext(rebuild_ctx->trans, mbox->hdr_ext_id, 0, + &hdr, sizeof(hdr)); +} + +static int +rebuild_mailbox(struct mdbox_storage_rebuild_context *ctx, + struct mail_namespace *ns, const char *vname) +{ + struct mailbox *box; + struct mdbox_mailbox *mbox; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + struct index_rebuild_context *rebuild_ctx; + enum mail_error error; + int ret; + + box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY | + MAILBOX_FLAG_IGNORE_ACLS); + if (box->storage != &ctx->storage->storage.storage) { + /* the namespace has multiple storages. */ + mailbox_free(&box); + return 0; + } + if (mailbox_open(box) < 0) { + error = mailbox_get_last_mail_error(box); + i_error("Couldn't open mailbox '%s': %s", + vname, mailbox_get_last_internal_error(box, NULL)); + mailbox_free(&box); + if (error == MAIL_ERROR_TEMP) + return -1; + /* non-temporary error, ignore */ + return 0; + } + mbox = MDBOX_MAILBOX(box); + + ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans, + MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES); + if (ret <= 0) { + i_assert(ret != 0); + mailbox_set_index_error(box); + mailbox_free(&box); + return -1; + } + + rebuild_ctx = index_index_rebuild_init(&mbox->box, view, trans); + mdbox_header_update(ctx, rebuild_ctx, mbox); + rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans); + index_index_rebuild_deinit(&rebuild_ctx, dbox_get_uidvalidity_next); + mail_index_unset_fscked(trans); + + mail_index_sync_set_reason(sync_ctx, "mdbox storage rebuild"); + if (mail_index_sync_commit(&sync_ctx) < 0) { + mailbox_set_index_error(box); + ret = -1; + } + + mailbox_free(&box); + return ret < 0 ? -1 : 0; +} + +static int +rebuild_namespace_mailboxes(struct mdbox_storage_rebuild_context *ctx, + struct mail_namespace *ns) +{ + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + int ret = 0; + + if (ctx->default_list == NULL || + (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) + ctx->default_list = ns->list; + + iter = mailbox_list_iter_init(ns->list, "*", + MAILBOX_LIST_ITER_RAW_LIST | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & (MAILBOX_NONEXISTENT | + MAILBOX_NOSELECT)) == 0) { + T_BEGIN { + ret = rebuild_mailbox(ctx, ns, info->vname); + } T_END; + if (ret < 0) { + ret = -1; + break; + } + } + } + if (mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int rebuild_mailboxes(struct mdbox_storage_rebuild_context *ctx) +{ + struct mail_storage *storage = &ctx->storage->storage.storage; + struct mail_namespace *ns; + + for (ns = storage->user->namespaces; ns != NULL; ns = ns->next) { + if (ns->storage == storage && ns->alias_for == NULL) { + if (rebuild_namespace_mailboxes(ctx, ns) < 0) + return -1; + } + } + if (ctx->default_list == NULL) + i_panic("No namespace found for storage=%s", storage->name); + return 0; +} + +static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg) +{ + mail_index_sync_set_reason(msg->sync_ctx, "mdbox storage rebuild"); + if (mail_index_sync_commit(&msg->sync_ctx) < 0) + return -1; + mailbox_free(&msg->box); + i_zero(msg); + return 0; +} + +static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx, + struct mdbox_rebuild_msg *msg) +{ + struct mail_storage *storage = &ctx->storage->storage.storage; + struct dbox_file *file; + const struct mail_index_header *hdr; + struct mdbox_mail_index_record dbox_rec; + const char *mailbox = NULL; + struct mailbox *box; + struct mdbox_mailbox *mbox; + enum mail_error error; + bool deleted, created; + int ret; + uint32_t seq; + + /* first see if message contains the mailbox it was originally + saved to */ + file = mdbox_file_init(ctx->storage, msg->file_id); + ret = dbox_file_open(file, &deleted); + if (ret > 0 && !deleted) + ret = dbox_file_seek(file, msg->offset); + if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) { + mailbox = dbox_file_metadata_get(file, + DBOX_METADATA_ORIG_MAILBOX); + if (mailbox != NULL) { + mailbox = mailbox_list_get_vname(ctx->default_list, mailbox); + mailbox = t_strdup(mailbox); + } + rebuild_scan_metadata(ctx, file); + } + dbox_file_unref(&file); + if (ret <= 0 || deleted) { + if (ret < 0) + return -1; + /* we shouldn't get here, so apparently we couldn't fix + something. just ignore the mail.. */ + return 0; + } + + if (mailbox == NULL) + mailbox = "INBOX"; + + /* we have the destination mailbox. now open it and add the message + there. */ + created = FALSE; + box = ctx->prev_msg.box != NULL && + strcmp(mailbox, ctx->prev_msg.box->vname) == 0 ? + ctx->prev_msg.box : NULL; + while (box == NULL) { + box = mailbox_alloc(ctx->default_list, mailbox, + MAILBOX_FLAG_READONLY | + MAILBOX_FLAG_IGNORE_ACLS); + i_assert(box->storage == storage); + if (mailbox_open(box) == 0) + break; + + error = mailbox_get_last_mail_error(box); + if (error == MAIL_ERROR_NOTFOUND && !created) { + /* mailbox doesn't exist currently? see if creating + it helps. */ + created = TRUE; + (void)mailbox_create(box, NULL, FALSE); + mailbox_free(&box); + continue; + } + + mailbox_free(&box); + if (error == MAIL_ERROR_TEMP) + return -1; + + if (strcmp(mailbox, "INBOX") != 0) { + /* see if we can save to INBOX instead. */ + mailbox = "INBOX"; + } else { + /* this shouldn't happen */ + return -1; + } + } + mbox = MDBOX_MAILBOX(box); + + /* switch the mailbox cache if necessary */ + if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) { + if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) + return -1; + } + if (ctx->prev_msg.box == NULL) { + ret = mail_index_sync_begin(box->index, + &ctx->prev_msg.sync_ctx, + &ctx->prev_msg.view, + &ctx->prev_msg.trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mailbox_set_index_error(box); + mailbox_free(&box); + return -1; + } + ctx->prev_msg.box = box; + hdr = mail_index_get_header(ctx->prev_msg.view); + ctx->prev_msg.next_uid = hdr->next_uid; + } + + /* add the new message */ + i_zero(&dbox_rec); + dbox_rec.map_uid = msg->map_uid; + dbox_rec.save_date = ioloop_time; + mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq); + mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->ext_id, + &dbox_rec, NULL); + mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id, + msg->guid_128, NULL); + + i_assert(msg->refcount == 0); + msg->refcount++; + return 0; +} + +static int rebuild_handle_zero_refs(struct mdbox_storage_rebuild_context *ctx) +{ + struct mdbox_rebuild_msg **msgs; + unsigned int i, count; + + /* if we have messages at this point which have refcount=0, they're + either already expunged or they were somehow lost for some reason. + we'll need to figure out what to do about them. */ + msgs = array_get_modifiable(&ctx->msgs, &count); + for (i = 0; i < count; i++) { + if (msgs[i]->refcount != 0) + continue; + + if (msgs[i]->seen_zero_ref_in_map) { + /* we've seen the map record, trust it. */ + continue; + } + /* either map record was lost for this message or the message + was lost from its mailbox. safest way to handle this is to + restore the message. */ + if (rebuild_restore_msg(ctx, msgs[i]) < 0) + return -1; + } + if (ctx->prev_msg.box != NULL) { + if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0) + return -1; + } + return 0; +} + +static void rebuild_update_refcounts(struct mdbox_storage_rebuild_context *ctx) +{ + const struct mail_index_header *hdr; + const void *data; + struct mdbox_rebuild_msg **msgs; + const uint16_t *ref16_p; + uint32_t seq, map_uid; + unsigned int i, count; + + /* update refcounts for existing map records */ + msgs = array_get_modifiable(&ctx->msgs, &count); + hdr = mail_index_get_header(ctx->atomic->sync_view); + for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) { + mail_index_lookup_uid(ctx->atomic->sync_view, seq, &map_uid); + if (map_uid != msgs[i]->map_uid) { + /* we've already expunged this map record */ + i_assert(map_uid < msgs[i]->map_uid); + continue; + } + + mail_index_lookup_ext(ctx->atomic->sync_view, seq, + ctx->storage->map->ref_ext_id, + &data, NULL); + ref16_p = data; + if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) { + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->storage->map->ref_ext_id, + &msgs[i]->refcount, NULL); + } + i++; + } + + /* update refcounts for newly created map records */ + for (; i < count; i++, seq++) { + mail_index_update_ext(ctx->atomic->sync_trans, seq, + ctx->storage->map->ref_ext_id, + &msgs[i]->refcount, NULL); + } +} + +static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx) +{ + struct mdbox_map_mail_index_header map_hdr; + + i_assert(ctx->default_list != NULL); + + if (rebuild_handle_zero_refs(ctx) < 0) + return -1; + rebuild_update_refcounts(ctx); + + /* update map header */ + map_hdr = ctx->orig_map_hdr; + map_hdr.highest_file_id = ctx->highest_file_id; + map_hdr.rebuild_count = ++ctx->rebuild_count; + + mail_index_update_header_ext(ctx->atomic->sync_trans, + ctx->storage->map->map_ext_id, + 0, &map_hdr, sizeof(map_hdr)); + return 0; +} + +static int +mdbox_storage_rebuild_scan_dir(struct mdbox_storage_rebuild_context *ctx, + const char *storage_dir, bool alt) +{ + DIR *dir; + struct dirent *d; + int ret = 0; + + dir = opendir(storage_dir); + if (dir == NULL) { + if (alt && errno == ENOENT) + return 0; + + mail_storage_set_critical(&ctx->storage->storage.storage, + "opendir(%s) failed: %m", storage_dir); + return -1; + } + for (errno = 0; (d = readdir(dir)) != NULL && ret == 0; errno = 0) { + if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, + strlen(MDBOX_MAIL_FILE_PREFIX)) == 0) T_BEGIN { + ret = rebuild_add_file(ctx, storage_dir, d->d_name); + } T_END; + } + if (ret == 0 && errno != 0) { + mail_storage_set_critical(&ctx->storage->storage.storage, + "readdir(%s) failed: %m", storage_dir); + ret = -1; + } + if (closedir(dir) < 0) { + mail_storage_set_critical(&ctx->storage->storage.storage, + "closedir(%s) failed: %m", storage_dir); + ret = -1; + } + return ret; +} + +static int +mdbox_storage_rebuild_scan_prepare(struct mdbox_storage_rebuild_context *ctx) +{ + const void *data; + size_t data_size; + + if (mdbox_map_open_or_create(ctx->storage->map) < 0) + return -1; + + /* begin by locking the map, so that other processes can't try to + rebuild at the same time. */ + if (mdbox_map_atomic_lock(ctx->atomic, "mdbox storage rebuild") < 0) + return -1; + + /* fsck the map just in case its UIDs are broken */ + if (mail_index_fsck(ctx->storage->map->index) < 0) { + mail_storage_set_index_error(&ctx->storage->storage.storage, + ctx->storage->map->index); + return -1; + } + + /* get old map header */ + mail_index_get_header_ext(ctx->atomic->sync_view, + ctx->storage->map->map_ext_id, + &data, &data_size); + i_zero(&ctx->orig_map_hdr); + memcpy(&ctx->orig_map_hdr, data, + I_MIN(data_size, sizeof(ctx->orig_map_hdr))); + ctx->highest_file_id = ctx->orig_map_hdr.highest_file_id; + + /* get storage rebuild counter after locking */ + ctx->rebuild_count = mdbox_map_get_rebuild_count(ctx->storage->map); + if (ctx->rebuild_count != ctx->storage->corrupted_rebuild_count && + ctx->storage->corrupted) { + /* storage was already rebuilt by someone else */ + return 0; + } + return 1; +} + +static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx) +{ + i_warning("mdbox %s: rebuilding indexes", ctx->storage->storage_dir); + + if (mdbox_storage_rebuild_scan_dir(ctx, ctx->storage->storage_dir, + FALSE) < 0) + return -1; + if (ctx->storage->alt_storage_dir != NULL) { + if (mdbox_storage_rebuild_scan_dir(ctx, + ctx->storage->alt_storage_dir, TRUE) < 0) + return -1; + } + + rebuild_apply_map(ctx); + if (rebuild_mailboxes(ctx) < 0 || + rebuild_finish(ctx) < 0) { + mdbox_map_atomic_set_failed(ctx->atomic); + return -1; + } + return 0; +} + +int mdbox_storage_rebuild_in_context(struct mdbox_storage *storage, + struct mdbox_map_atomic_context *atomic) +{ + struct mdbox_storage_rebuild_context *ctx; + int ret; + + if (dbox_verify_alt_storage(storage->map->root_list) < 0) { + mail_storage_set_critical(&storage->storage.storage, + "mdbox rebuild: Alt storage %s not mounted, aborting", + storage->alt_storage_dir); + mdbox_map_atomic_set_failed(atomic); + return -1; + } + + ctx = mdbox_storage_rebuild_init(storage, atomic); + if ((ret = mdbox_storage_rebuild_scan_prepare(ctx)) > 0) { + struct event_reason *reason = event_reason_begin("mdbox:rebuild"); + ret = mdbox_storage_rebuild_scan(ctx); + event_reason_end(&reason); + } + mdbox_storage_rebuild_deinit(ctx); + + if (ret == 0) { + storage->corrupted = FALSE; + storage->corrupted_rebuild_count = 0; + } + return ret; +} + +int mdbox_storage_rebuild(struct mdbox_storage *storage) +{ + struct mdbox_map_atomic_context *atomic; + int ret; + + atomic = mdbox_map_atomic_begin(storage->map); + ret = mdbox_storage_rebuild_in_context(storage, atomic); + mdbox_map_atomic_set_success(atomic); + mdbox_map_atomic_unset_fscked(atomic); + if (mdbox_map_atomic_finish(&atomic) < 0) + ret = -1; + return ret; +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h new file mode 100644 index 0000000..1194d29 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h @@ -0,0 +1,10 @@ +#ifndef MDBOX_STORAGE_REBUILD_H +#define MDBOX_STORAGE_REBUILD_H + +struct mdbox_map_atomic_context; + +int mdbox_storage_rebuild_in_context(struct mdbox_storage *storage, + struct mdbox_map_atomic_context *atomic); +int mdbox_storage_rebuild(struct mdbox_storage *storage); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.c b/src/lib-storage/index/dbox-multi/mdbox-storage.c new file mode 100644 index 0000000..e7e7f60 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c @@ -0,0 +1,530 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "master-service.h" +#include "mail-index-modseq.h" +#include "mail-index-alloc-cache.h" +#include "mailbox-log.h" +#include "mailbox-list-private.h" +#include "index-pop3-uidl.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-storage.h" + +extern struct mail_storage mdbox_storage; +extern struct mailbox mdbox_mailbox; + +static struct event_category event_category_mdbox = { + .name = "mdbox", + .parent = &event_category_storage, +}; + +static struct mail_storage *mdbox_storage_alloc(void) +{ + struct mdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mdbox storage", 2048); + storage = p_new(pool, struct mdbox_storage, 1); + storage->storage.v = mdbox_dbox_storage_vfuncs; + storage->storage.storage = mdbox_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +int mdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, const char **error_r) +{ + struct mdbox_storage *storage = MDBOX_STORAGE(_storage); + const char *dir; + + storage->set = mail_namespace_get_driver_settings(ns, _storage); + storage->preallocate_space = storage->set->mdbox_preallocate_space; + + if (*ns->list->set.mailbox_dir_name == '\0') { + *error_r = "mdbox: MAILBOXDIR must not be empty"; + return -1; + } + + _storage->unique_root_dir = + p_strdup(_storage->pool, ns->list->set.root_dir); + + dir = mailbox_list_get_root_forced(ns->list, MAILBOX_LIST_PATH_TYPE_DIR); + storage->storage_dir = p_strconcat(_storage->pool, dir, + "/"MDBOX_GLOBAL_DIR_NAME, NULL); + if (ns->list->set.alt_dir != NULL) { + storage->alt_storage_dir = p_strconcat(_storage->pool, + ns->list->set.alt_dir, + "/"MDBOX_GLOBAL_DIR_NAME, NULL); + } + i_array_init(&storage->open_files, 64); + + storage->map = mdbox_map_init(storage, ns->list); + return dbox_storage_create(_storage, ns, error_r); +} + +void mdbox_storage_destroy(struct mail_storage *_storage) +{ + struct mdbox_storage *storage = MDBOX_STORAGE(_storage); + + mdbox_files_free(storage); + mdbox_map_deinit(&storage->map); + timeout_remove(&storage->to_close_unused_files); + + if (array_is_created(&storage->move_from_alt_map_uids)) + array_free(&storage->move_from_alt_map_uids); + if (array_is_created(&storage->move_to_alt_map_uids)) + array_free(&storage->move_to_alt_map_uids); + array_free(&storage->open_files); + dbox_storage_destroy(_storage); +} + +static const char * +mdbox_storage_find_root_dir(const struct mail_namespace *ns) +{ + bool debug = ns->mail_set->mail_debug; + const char *home, *path; + + if (ns->owner != NULL && + mail_user_get_home(ns->owner, &home) > 0) { + path = t_strconcat(home, "/mdbox", NULL); + if (access(path, R_OK|W_OK|X_OK) == 0) { + if (debug) + i_debug("mdbox: root exists (%s)", path); + return path; + } + if (debug) + i_debug("mdbox: access(%s, rwx): failed: %m", path); + } + return NULL; +} + +static bool mdbox_storage_autodetect(const struct mail_namespace *ns, + struct mailbox_list_settings *set) +{ + bool debug = ns->mail_set->mail_debug; + struct stat st; + const char *path, *root_dir; + + if (set->root_dir != NULL) + root_dir = set->root_dir; + else { + root_dir = mdbox_storage_find_root_dir(ns); + if (root_dir == NULL) { + if (debug) + i_debug("mdbox: couldn't find root dir"); + return FALSE; + } + } + + path = t_strconcat(root_dir, "/"MDBOX_GLOBAL_DIR_NAME, NULL); + if (stat(path, &st) < 0) { + if (debug) + i_debug("mdbox autodetect: stat(%s) failed: %m", path); + return FALSE; + } + + if (!S_ISDIR(st.st_mode)) { + if (debug) + i_debug("mdbox autodetect: %s not a directory", path); + return FALSE; + } + + set->root_dir = root_dir; + dbox_storage_get_list_settings(ns, set); + return TRUE; +} + +static struct mailbox * +mdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mdbox_mailbox *mbox; + struct index_mailbox_context *ibox; + pool_t pool; + + /* dbox can't work without index files */ + flags &= ENUM_NEGATE(MAILBOX_FLAG_NO_INDEX_FILES); + + pool = pool_alloconly_create("mdbox mailbox", 1024*3); + mbox = p_new(pool, struct mdbox_mailbox, 1); + mbox->box = mdbox_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mdbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + ibox = INDEX_STORAGE_CONTEXT(&mbox->box); + ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY; + + mbox->storage = MDBOX_STORAGE(storage); + return &mbox->box; +} + +int mdbox_mailbox_open(struct mailbox *box) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + time_t path_ctime; + + if (dbox_mailbox_check_existence(box, &path_ctime) < 0) + return -1; + if (dbox_mailbox_open(box, path_ctime) < 0) + return -1; + + mbox->ext_id = + mail_index_ext_register(mbox->box.index, "mdbox", 0, + sizeof(struct mdbox_mail_index_record), + sizeof(uint32_t)); + mbox->hdr_ext_id = + mail_index_ext_register(mbox->box.index, "mdbox-hdr", + sizeof(struct mdbox_index_header), 0, 0); + mbox->guid_ext_id = + mail_index_ext_register(mbox->box.index, "guid", + 0, GUID_128_SIZE, 1); + return 0; +} + +static void mdbox_mailbox_close(struct mailbox *box) +{ + struct mdbox_storage *mstorage = MDBOX_STORAGE(box->storage); + + if (mstorage->corrupted && !mstorage->rebuilding_storage) + (void)mdbox_storage_rebuild(mstorage); + + index_storage_mailbox_close(box); +} + +int mdbox_read_header(struct mdbox_mailbox *mbox, + struct mdbox_index_header *hdr, bool *need_resize_r) +{ + const void *data; + size_t data_size; + + i_assert(mbox->box.opened); + + mail_index_get_header_ext(mbox->box.view, mbox->hdr_ext_id, + &data, &data_size); + if (data_size < MDBOX_INDEX_HEADER_MIN_SIZE && + (!mbox->creating || data_size != 0)) { + mailbox_set_critical(&mbox->box, + "mdbox: Invalid dbox header size: %zu", + data_size); + mdbox_storage_set_corrupted(mbox->storage); + return -1; + } + i_zero(hdr); + if (data_size > 0) + memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr))); + *need_resize_r = data_size < sizeof(*hdr); + return 0; +} + +void mdbox_update_header(struct mdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update) +{ + struct mdbox_index_header hdr, new_hdr; + bool need_resize; + + if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) { + i_zero(&hdr); + need_resize = TRUE; + } + + new_hdr = hdr; + + if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) { + memcpy(new_hdr.mailbox_guid, update->mailbox_guid, + sizeof(new_hdr.mailbox_guid)); + } else if (guid_128_is_empty(new_hdr.mailbox_guid)) { + guid_128_generate(new_hdr.mailbox_guid); + } + + new_hdr.map_uid_validity = + mdbox_map_get_uid_validity(mbox->storage->map); + if (need_resize) { + mail_index_ext_resize_hdr(trans, mbox->hdr_ext_id, + sizeof(new_hdr)); + } + if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) { + mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0, + &new_hdr, sizeof(new_hdr)); + } +} + +static int ATTR_NULL(2, 3) +mdbox_write_index_header(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + struct mail_index_transaction *new_trans = NULL; + struct mail_index_view *view; + const struct mail_index_header *hdr; + uint32_t uid_validity, uid_next; + + if (mdbox_map_open_or_create(mbox->storage->map) < 0) + return -1; + + if (trans == NULL) { + new_trans = mail_index_transaction_begin(box->view, 0); + trans = new_trans; + } + + view = mail_index_view_open(box->index); + hdr = mail_index_get_header(view); + uid_validity = hdr->uid_validity; + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + else if (uid_validity == 0) { + /* set uidvalidity */ + uid_validity = dbox_get_uidvalidity_next(box->list); + } + + if (hdr->uid_validity != uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + if (update != NULL && hdr->next_uid < update->min_next_uid) { + uid_next = update->min_next_uid; + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + } + if (update != NULL && update->min_first_recent_uid != 0 && + hdr->first_recent_uid < update->min_first_recent_uid) { + uint32_t first_recent_uid = update->min_first_recent_uid; + + mail_index_update_header(trans, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + if (update != NULL && update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(view) < update->min_highest_modseq) { + mail_index_modseq_enable(box->index); + mail_index_update_highest_modseq(trans, + update->min_highest_modseq); + } + mail_index_view_close(&view); + + if (box->inbox_user && box->creating) { + /* initialize pop3-uidl header when creating mailbox + (not on mailbox_update()) */ + index_pop3_uidl_set_max_uid(box, trans, 0); + } + + mdbox_update_header(mbox, trans, update); + if (new_trans != NULL) { + if (mail_index_transaction_commit(&new_trans) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + return 0; +} + +int mdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + int ret; + + mbox->creating = TRUE; + ret = mdbox_write_index_header(box, update, trans); + mbox->creating = FALSE; + return ret; +} + +void mdbox_storage_set_corrupted(struct mdbox_storage *storage) +{ + if (storage->corrupted) { + /* already set it corrupted (possibly recursing back here) */ + return; + } + + storage->corrupted = TRUE; + storage->corrupted_rebuild_count = (uint32_t)-1; + + if (mdbox_map_open(storage->map) > 0 && + mdbox_map_refresh(storage->map) == 0) { + storage->corrupted_rebuild_count = + mdbox_map_get_rebuild_count(storage->map); + } +} + +static const char * +mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) +{ + return ""; +} + +void mdbox_set_mailbox_corrupted(struct mailbox *box) +{ + struct mdbox_storage *mstorage = MDBOX_STORAGE(box->storage); + + mdbox_storage_set_corrupted(mstorage); +} + +void mdbox_set_file_corrupted(struct dbox_file *file) +{ + struct mdbox_storage *mstorage = MDBOX_DBOX_STORAGE(file->storage); + + mdbox_storage_set_corrupted(mstorage); +} + +static int +mdbox_mailbox_get_guid(struct mdbox_mailbox *mbox, guid_128_t guid_r) +{ + const struct mail_index_header *idx_hdr; + struct mdbox_index_header hdr; + bool need_resize; + int ret = 0; + + i_assert(!mbox->creating); + + /* there's a race condition between mkdir and getting the mailbox GUID. + normally this is handled by mdbox syncing, but GUID can be looked up + without syncing. when we detect this situation we'll try to finish + creating the indexes first, which usually means just waiting for + the sync lock to get unlocked by the other process creating them. */ + idx_hdr = mail_index_get_header(mbox->box.view); + if (idx_hdr->uid_validity == 0 && idx_hdr->next_uid == 1) { + if (dbox_mailbox_create_indexes(&mbox->box, NULL) < 0) + return -1; + } + + if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) + i_zero(&hdr); + + if (guid_128_is_empty(hdr.mailbox_guid)) { + /* regenerate it */ + if (mdbox_write_index_header(&mbox->box, NULL, NULL) < 0 || + mdbox_read_header(mbox, &hdr, &need_resize) < 0) + ret = -1; + } + if (ret == 0) + memcpy(guid_r, hdr.mailbox_guid, GUID_128_SIZE); + return ret; +} + +static int +mdbox_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + if ((items & MAILBOX_METADATA_GUID) != 0) { + if (mdbox_mailbox_get_guid(mbox, metadata_r->guid) < 0) + return -1; + } + return 0; +} + +static int +mdbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) +{ + if (!box->opened) { + if (mailbox_open(box) < 0) + return -1; + } + if (mdbox_write_index_header(box, update, NULL) < 0) + return -1; + return index_storage_mailbox_update_common(box, update); +} + +struct mail_storage mdbox_storage = { + .name = MDBOX_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + .event_category = &event_category_mdbox, + + .v = { + mdbox_get_setting_parser_info, + mdbox_storage_alloc, + mdbox_storage_create, + mdbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + mdbox_storage_autodetect, + mdbox_mailbox_alloc, + mdbox_purge, + mail_storage_list_index_rebuild, + } +}; + +struct mailbox mdbox_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mdbox_mailbox_open, + mdbox_mailbox_close, + index_storage_mailbox_free, + dbox_mailbox_create, + mdbox_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mdbox_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + mdbox_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + dbox_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + index_storage_search_next_match_mail, + mdbox_save_alloc, + mdbox_save_begin, + dbox_save_continue, + mdbox_save_finish, + mdbox_save_cancel, + mdbox_copy, + mdbox_transaction_save_commit_pre, + mdbox_transaction_save_commit_post, + mdbox_transaction_save_rollback, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs = { + mdbox_file_unrefed, + mdbox_file_create_fd, + mdbox_mail_open, + mdbox_mailbox_create_indexes, + mdbox_get_attachment_path_suffix, + mdbox_set_mailbox_corrupted, + mdbox_set_file_corrupted +}; diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.h b/src/lib-storage/index/dbox-multi/mdbox-storage.h new file mode 100644 index 0000000..ea99532 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h @@ -0,0 +1,118 @@ +#ifndef MDBOX_STORAGE_H +#define MDBOX_STORAGE_H + +#include "index-storage.h" +#include "dbox-storage.h" +#include "mdbox-settings.h" + +#define MDBOX_STORAGE_NAME "mdbox" +#define MDBOX_DELETED_STORAGE_NAME "mdbox_deleted" +#define MDBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index" +#define MDBOX_GLOBAL_DIR_NAME "storage" +#define MDBOX_MAIL_FILE_PREFIX "m." +#define MDBOX_MAIL_FILE_FORMAT MDBOX_MAIL_FILE_PREFIX"%u" +#define MDBOX_MAX_OPEN_UNUSED_FILES 2 +#define MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS 30 + +#define MDBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t)) +struct mdbox_index_header { + uint32_t map_uid_validity; + guid_128_t mailbox_guid; + uint8_t flags; /* enum dbox_index_header_flags */ + uint8_t unused[3]; +}; + +struct mdbox_storage { + struct dbox_storage storage; + const struct mdbox_settings *set; + + /* paths for storage directories */ + const char *storage_dir, *alt_storage_dir; + struct mdbox_map *map; + + ARRAY(struct mdbox_file *) open_files; + struct timeout *to_close_unused_files; + + ARRAY_TYPE(uint32_t) move_to_alt_map_uids; + ARRAY_TYPE(uint32_t) move_from_alt_map_uids; + + /* if non-zero, storage should be rebuilt (except if rebuild_count + has changed from this value) */ + uint32_t corrupted_rebuild_count; + + bool corrupted:1; + bool rebuilding_storage:1; + bool preallocate_space:1; +}; + +struct mdbox_mail_index_record { + uint32_t map_uid; + /* UNIX timestamp of when the message was saved/copied to this + mailbox */ + uint32_t save_date; +}; + +struct mdbox_mailbox { + struct mailbox box; + struct mdbox_storage *storage; + + uint32_t map_uid_validity; + uint32_t ext_id, hdr_ext_id, guid_ext_id; + + bool mdbox_deleted_synced:1; + bool creating:1; +}; + +#define MDBOX_DBOX_STORAGE(s) container_of(s, struct mdbox_storage, storage) +#define MDBOX_STORAGE(s) MDBOX_DBOX_STORAGE(DBOX_STORAGE(s)) +#define MDBOX_MAILBOX(s) container_of(s, struct mdbox_mailbox, box) + +extern struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs; +extern struct mail_vfuncs mdbox_mail_vfuncs; + +int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, + struct dbox_file **file_r); + +/* Get map_uid for wanted message. */ +int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view, + uint32_t seq, uint32_t *map_uid_r); +uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); +int mdbox_read_header(struct mdbox_mailbox *mbox, + struct mdbox_index_header *hdr, bool *need_resize_r); +void mdbox_update_header(struct mdbox_mailbox *mbox, + struct mail_index_transaction *trans, + const struct mailbox_update *update) ATTR_NULL(3); +int mdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans); + +struct mail_save_context * +mdbox_save_alloc(struct mailbox_transaction_context *_t); +int mdbox_save_begin(struct mail_save_context *ctx, struct istream *input); +int mdbox_save_finish(struct mail_save_context *ctx); +void mdbox_save_cancel(struct mail_save_context *ctx); + +struct dbox_file * +mdbox_save_file_get_file(struct mailbox_transaction_context *t, + uint32_t seq, uoff_t *offset_r); + +int mdbox_transaction_save_commit_pre(struct mail_save_context *ctx); +void mdbox_transaction_save_commit_post(struct mail_save_context *ctx, + struct mail_index_transaction_commit_result *result); +void mdbox_transaction_save_rollback(struct mail_save_context *ctx); + +int mdbox_copy(struct mail_save_context *ctx, struct mail *mail); + +void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt); +int mdbox_purge(struct mail_storage *storage); + +int mdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, const char **error_r); +void mdbox_storage_destroy(struct mail_storage *_storage); +int mdbox_mailbox_open(struct mailbox *box); + +void mdbox_storage_set_corrupted(struct mdbox_storage *storage); +void mdbox_set_mailbox_corrupted(struct mailbox *box); +void mdbox_set_file_corrupted(struct dbox_file *file); + +#endif diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.c b/src/lib-storage/index/dbox-multi/mdbox-sync.c new file mode 100644 index 0000000..0c2d2d4 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c @@ -0,0 +1,377 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +/* + Expunging works like: + + 1. Lock map index by beginning a map sync. + 2. Write map UID refcount changes to map index (=> tail != head). + 3. Expunge messages from mailbox index. + 4. Finish map sync, which updates tail=head and unlocks map index. + + If something crashes after 2 but before 4 is finished, tail != head and + reader can do a full resync to figure out what got broken. +*/ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "str.h" +#include "mdbox-storage.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mailbox-recent-flags.h" + + +/* returns -1 on error, 1 on success, 0 if guid is empty/missing */ +static int +dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq, + const guid_128_t guid_128) +{ + const void *data; + uint32_t uid; + + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mail_index_lookup_ext(ctx->sync_view, seq, + ctx->mbox->guid_ext_id, &data, NULL); + + if ((data == NULL) || guid_128_is_empty(data)) + return 0; + + if (guid_128_is_empty(guid_128) || + memcmp(data, guid_128, GUID_128_SIZE) == 0) + return 1; + + mailbox_set_critical(&ctx->mbox->box, + "Expunged GUID mismatch for UID %u: %s vs %s", + uid, guid_128_to_string(data), guid_128_to_string(guid_128)); + mdbox_storage_set_corrupted(ctx->mbox->storage); + return -1; +} + +static int mdbox_sync_expunge(struct mdbox_sync_context *ctx, uint32_t seq, + const guid_128_t guid_128) +{ + uint32_t map_uid; + int ret; + + if (seq_range_array_add(&ctx->expunged_seqs, seq)) { + /* already marked as expunged in this sync */ + return 0; + } + + ret = dbox_sync_verify_expunge_guid(ctx, seq, guid_128); + if (ret <= 0) + return ret; + if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0) + return -1; + if (mdbox_map_update_refcount(ctx->map_trans, map_uid, -1) < 0) + return -1; + return 0; +} + +static int mdbox_sync_rec(struct mdbox_sync_context *ctx, + const struct mail_index_sync_rec *sync_rec) +{ + uint32_t seq, seq1, seq2; + + if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + /* not interested */ + return 0; + } + + if (!mail_index_lookup_seq_range(ctx->sync_view, + sync_rec->uid1, sync_rec->uid2, + &seq1, &seq2)) { + /* already expunged everything. nothing to do. */ + return 0; + } + + for (seq = seq1; seq <= seq2; seq++) { + if (mdbox_sync_expunge(ctx, seq, sync_rec->guid_128) < 0) + return -1; + } + return 0; +} + +static int dbox_sync_mark_expunges(struct mdbox_sync_context *ctx) +{ + enum mail_index_transaction_flags flags = + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + struct mailbox *box = &ctx->mbox->box; + struct mail_index_transaction *trans; + struct seq_range_iter iter; + unsigned int n; + const void *data; + uint32_t seq, uid; + + /* use a separate transaction here so that we can commit the changes + during map transaction */ + trans = mail_index_transaction_begin(ctx->sync_view, flags); + seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &seq)) { + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mail_index_lookup_ext(ctx->sync_view, seq, + ctx->mbox->guid_ext_id, &data, NULL); + if ((data == NULL) || guid_128_is_empty(data)) + mail_index_expunge(trans, seq); + else + mail_index_expunge_guid(trans, seq, data); + } + if (mail_index_transaction_commit(&trans) < 0) + return -1; + + box->tmp_sync_view = ctx->sync_view; + seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &seq)) { + mail_index_lookup_uid(ctx->sync_view, seq, &uid); + mailbox_sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE); + } + box->tmp_sync_view = NULL; + return 0; +} + +static int mdbox_sync_index(struct mdbox_sync_context *ctx) +{ + struct mailbox *box = &ctx->mbox->box; + const struct mail_index_header *hdr; + struct mail_index_sync_rec sync_rec; + uint32_t seq1, seq2; + int ret = 0; + + hdr = mail_index_get_header(ctx->sync_view); + if (hdr->uid_validity == 0) { + /* newly created index file */ + if (hdr->next_uid == 1) { + /* could be just a race condition where we opened the + mailbox between mkdir and index creation. fix this + silently. */ + if (mdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0) + return -1; + return 1; + } + mailbox_set_critical(box, "Broken index: missing UIDVALIDITY"); + return 0; + } + + /* mark the newly seen messages as recent */ + if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, + hdr->next_uid, &seq1, &seq2)) { + mailbox_recent_flags_set_seqs(&ctx->mbox->box, ctx->sync_view, + seq1, seq2); + } + + /* handle syncing records without map being locked. */ + if (mdbox_map_atomic_is_locked(ctx->atomic)) { + ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE); + i_array_init(&ctx->expunged_seqs, 64); + } + while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) { + if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0) + break; + } + + /* write refcount changes to map index. transaction commit updates the + log head, while tail is left behind. */ + if (mdbox_map_atomic_is_locked(ctx->atomic)) { + if (ret == 0) + ret = mdbox_map_transaction_commit(ctx->map_trans, "mdbox syncing"); + /* write changes to mailbox index */ + if (ret == 0) + ret = dbox_sync_mark_expunges(ctx); + + /* finish the map changes and unlock the map. this also updates + map's tail -> head. */ + if (ret < 0) + mdbox_map_atomic_set_failed(ctx->atomic); + mdbox_map_transaction_free(&ctx->map_trans); + ctx->expunged_count = seq_range_count(&ctx->expunged_seqs); + array_free(&ctx->expunged_seqs); + } + + mailbox_sync_notify(box, 0, 0); + + return ret == 0 ? 1 : + (ctx->mbox->storage->corrupted ? 0 : -1); +} + +static int mdbox_sync_try_begin(struct mdbox_sync_context *ctx, + enum mail_index_sync_flags sync_flags) +{ + struct mdbox_mailbox *mbox = ctx->mbox; + int ret; + + ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, sync_flags); + if (mail_index_reset_fscked(mbox->box.index)) + mdbox_storage_set_corrupted(mbox->storage); + if (ret <= 0) + return ret; /* error / nothing to do */ + + if (!mdbox_map_atomic_is_locked(ctx->atomic) && + mail_index_sync_has_expunges(ctx->index_sync_ctx)) { + /* we have expunges, so we need to write to map. + it needs to be locked before mailbox index. */ + mail_index_sync_set_reason(ctx->index_sync_ctx, "mdbox expunge check"); + mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&ctx->mbox->box); + + if (mdbox_map_atomic_lock(ctx->atomic, "mdbox syncing with expunges") < 0) + return -1; + return mdbox_sync_try_begin(ctx, sync_flags); + } + return 1; +} + +int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, + struct mdbox_map_atomic_context *atomic, + struct mdbox_sync_context **ctx_r) +{ + const struct mail_index_header *hdr = + mail_index_get_header(mbox->box.view); + struct mdbox_sync_context *ctx; + const char *reason; + enum mail_index_sync_flags sync_flags; + int ret; + bool rebuild, storage_rebuilt = FALSE; + + *ctx_r = NULL; + + /* avoid race conditions with mailbox creation, don't check for dbox + headers until syncing has locked the mailbox */ + rebuild = mbox->storage->corrupted || + (hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0 || + mdbox_map_is_fscked(mbox->storage->map) || + (flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0; + if (rebuild && (flags & MDBOX_SYNC_FLAG_NO_REBUILD) == 0) { + if (mdbox_storage_rebuild_in_context(mbox->storage, atomic) < 0) + return -1; + mailbox_recent_flags_reset(&mbox->box); + storage_rebuilt = TRUE; + } + + ctx = i_new(struct mdbox_sync_context, 1); + ctx->mbox = mbox; + ctx->flags = flags; + ctx->atomic = atomic; + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if (!rebuild && (flags & MDBOX_SYNC_FLAG_FORCE) == 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + if ((flags & MDBOX_SYNC_FLAG_FSYNC) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC; + /* don't write unnecessary dirty flag updates */ + sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; + + ret = mdbox_sync_try_begin(ctx, sync_flags); + if (ret <= 0) { + /* failed / nothing to do */ + index_storage_expunging_deinit(&mbox->box); + i_free(ctx); + return ret; + } + + if ((ret = mdbox_sync_index(ctx)) <= 0) { + mail_index_sync_set_reason(ctx->index_sync_ctx, + ret < 0 ? "mdbox syncing failed" : + "mdbox syncing found corruption"); + mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&mbox->box); + i_free_and_null(ctx); + + if (ret < 0) + return -1; + + /* corrupted */ + if (storage_rebuilt) { + mailbox_set_critical(&mbox->box, + "mdbox: Storage keeps breaking"); + return -1; + } + + /* we'll need to rebuild storage. + try again from the beginning. */ + mdbox_storage_set_corrupted(mbox->storage); + if ((flags & MDBOX_SYNC_FLAG_NO_REBUILD) != 0) { + mailbox_set_critical(&mbox->box, + "mdbox: Can't rebuild storage"); + return -1; + } + return mdbox_sync_begin(mbox, flags, atomic, ctx_r); + } + index_storage_expunging_deinit(&mbox->box); + + if (!mdbox_map_atomic_is_locked(ctx->atomic)) + reason = "mdbox synced"; + else { + /* may be 0 msgs, but that still informs that the map + was locked */ + reason = t_strdup_printf("mdbox synced - %u msgs expunged", + ctx->expunged_count); + } + mail_index_sync_set_reason(ctx->index_sync_ctx, reason); + + *ctx_r = ctx; + return 0; +} + +int mdbox_sync_finish(struct mdbox_sync_context **_ctx, bool success) +{ + struct mdbox_sync_context *ctx = *_ctx; + struct mail_storage *storage = &ctx->mbox->storage->storage.storage; + int ret = success ? 0 : -1; + + *_ctx = NULL; + + if (success) { + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { + mailbox_set_index_error(&ctx->mbox->box); + ret = -1; + } + } else { + mail_index_sync_rollback(&ctx->index_sync_ctx); + } + + if (storage->rebuild_list_index) + ret = mail_storage_list_index_rebuild_and_set_uncorrupted(storage); + + i_free(ctx); + return ret; +} + +int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags) +{ + struct mdbox_sync_context *sync_ctx; + struct mdbox_map_atomic_context *atomic; + int ret; + + atomic = mdbox_map_atomic_begin(mbox->storage->map); + ret = mdbox_sync_begin(mbox, flags, atomic, &sync_ctx); + if (ret == 0 && sync_ctx != NULL) + ret = mdbox_sync_finish(&sync_ctx, TRUE); + if (ret == 0) + mdbox_map_atomic_set_success(atomic); + if (mdbox_map_atomic_finish(&atomic) < 0) + ret = -1; + return ret; +} + +struct mailbox_sync_context * +mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box); + enum mdbox_sync_flags mdbox_sync_flags = 0; + int ret = 0; + + if (mail_index_reset_fscked(box->index)) + mdbox_storage_set_corrupted(mbox->storage); + if (index_mailbox_want_full_sync(&mbox->box, flags) || + mbox->storage->corrupted) { + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) + mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD; + ret = mdbox_sync(mbox, mdbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +} diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.h b/src/lib-storage/index/dbox-multi/mdbox-sync.h new file mode 100644 index 0000000..a9bc565 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.h @@ -0,0 +1,37 @@ +#ifndef MDBOX_SYNC_H +#define MDBOX_SYNC_H + +struct mailbox; +struct mdbox_mailbox; + +enum mdbox_sync_flags { + MDBOX_SYNC_FLAG_FORCE = 0x01, + MDBOX_SYNC_FLAG_FSYNC = 0x02, + MDBOX_SYNC_FLAG_FORCE_REBUILD = 0x04, + MDBOX_SYNC_FLAG_NO_PURGE = 0x08, + MDBOX_SYNC_FLAG_NO_REBUILD = 0x10 +}; + +struct mdbox_sync_context { + struct mdbox_mailbox *mbox; + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mdbox_map_transaction_context *map_trans; + struct mdbox_map_atomic_context *atomic; + enum mdbox_sync_flags flags; + + ARRAY_TYPE(seq_range) expunged_seqs; + unsigned int expunged_count; +}; + +int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags, + struct mdbox_map_atomic_context *atomic, + struct mdbox_sync_context **ctx_r); +int mdbox_sync_finish(struct mdbox_sync_context **ctx, bool success); +int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags); + +struct mailbox_sync_context * +mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); + +#endif |