diff options
Diffstat (limited to '')
70 files changed, 31742 insertions, 0 deletions
diff --git a/src/lib-index/Makefile.am b/src/lib-index/Makefile.am new file mode 100644 index 0000000..f203201 --- /dev/null +++ b/src/lib-index/Makefile.am @@ -0,0 +1,150 @@ +noinst_LTLIBRARIES = libindex.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-mail + +libindex_la_SOURCES = \ + mail-cache.c \ + mail-cache-decisions.c \ + mail-cache-fields.c \ + mail-cache-lookup.c \ + mail-cache-purge.c \ + mail-cache-transaction.c \ + mail-cache-sync-update.c \ + mail-index.c \ + mail-index-alloc-cache.c \ + mail-index-dummy-view.c \ + mail-index-fsck.c \ + mail-index-lock.c \ + mail-index-map.c \ + mail-index-map-hdr.c \ + mail-index-map-read.c \ + mail-index-modseq.c \ + mail-index-transaction.c \ + mail-index-transaction-export.c \ + mail-index-transaction-finish.c \ + mail-index-transaction-sort-appends.c \ + mail-index-transaction-update.c \ + mail-index-transaction-view.c \ + mail-index-strmap.c \ + mail-index-sync.c \ + mail-index-sync-ext.c \ + mail-index-sync-keywords.c \ + mail-index-sync-update.c \ + mail-index-util.c \ + mail-index-view.c \ + mail-index-view-sync.c \ + mail-index-write.c \ + mail-transaction-log.c \ + mail-transaction-log-append.c \ + mail-transaction-log-file.c \ + mail-transaction-log-modseq.c \ + mail-transaction-log-view.c \ + mailbox-log.c + +headers = \ + mail-cache.h \ + mail-cache-private.h \ + mail-index.h \ + mail-index-alloc-cache.h \ + mail-index-modseq.h \ + mail-index-private.h \ + mail-index-strmap.h \ + mail-index-sync-private.h \ + mail-index-transaction-private.h \ + mail-index-util.h \ + mail-index-view-private.h \ + mail-transaction-log.h \ + mail-transaction-log-private.h \ + mail-transaction-log-view-private.h \ + mailbox-log.h + +test_programs = \ + test-mail-cache \ + test-mail-cache-fields \ + test-mail-cache-purge \ + test-mail-index \ + test-mail-index-map \ + test-mail-index-modseq \ + test-mail-index-sync-ext \ + test-mail-index-transaction-finish \ + test-mail-index-transaction-update \ + test-mail-index-write \ + test-mail-transaction-log-append \ + test-mail-transaction-log-file \ + test-mail-transaction-log-view + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + mail-index-util.lo \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_deps = $(noinst_LTLIBRARIES) $(test_libs) + +test_mail_cache_SOURCES = test-mail-cache-common.c test-mail-cache.c +test_mail_cache_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_cache_DEPENDENCIES = $(test_deps) + +test_mail_cache_fields_SOURCES = test-mail-cache-common.c test-mail-cache-fields.c +test_mail_cache_fields_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_cache_fields_DEPENDENCIES = $(test_deps) + +test_mail_cache_purge_SOURCES = test-mail-cache-common.c test-mail-cache-purge.c +test_mail_cache_purge_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_cache_purge_DEPENDENCIES = $(test_deps) + +test_mail_index_SOURCES = test-mail-index.c +test_mail_index_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_DEPENDENCIES = $(test_deps) + +test_mail_index_map_SOURCES = test-mail-index-map.c +test_mail_index_map_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_map_DEPENDENCIES = $(test_deps) + +test_mail_index_modseq_SOURCES = test-mail-index-modseq.c +test_mail_index_modseq_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_modseq_DEPENDENCIES = $(test_deps) + +test_mail_index_sync_ext_SOURCES = test-mail-index-sync-ext.c +test_mail_index_sync_ext_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_sync_ext_DEPENDENCIES = $(test_deps) + +test_mail_index_transaction_finish_SOURCES = test-mail-index-transaction-finish.c +test_mail_index_transaction_finish_LDADD = mail-index-transaction-finish.lo $(test_libs) +test_mail_index_transaction_finish_DEPENDENCIES = $(test_deps) + +test_mail_index_transaction_update_SOURCES = test-mail-index-transaction-update.c +test_mail_index_transaction_update_LDADD = mail-index-transaction-update.lo $(test_libs) +test_mail_index_transaction_update_DEPENDENCIES = $(test_deps) + +test_mail_index_write_SOURCES = test-mail-index-write.c +test_mail_index_write_LDADD = mail-index-write.lo $(test_libs) +test_mail_index_write_DEPENDENCIES = $(test_deps) + +test_mail_transaction_log_append_SOURCES = test-mail-transaction-log-append.c +test_mail_transaction_log_append_LDADD = mail-transaction-log-append.lo $(test_libs) +test_mail_transaction_log_append_DEPENDENCIES = $(test_deps) + +test_mail_transaction_log_file_SOURCES = test-mail-transaction-log-file.c +test_mail_transaction_log_file_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_transaction_log_file_DEPENDENCIES = $(test_deps) + +test_mail_transaction_log_view_SOURCES = test-mail-transaction-log-view.c +test_mail_transaction_log_view_LDADD = mail-transaction-log-view.lo $(test_libs) +test_mail_transaction_log_view_DEPENDENCIES = $(test_deps) + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + +noinst_HEADERS = \ + test-mail-cache.h \ + test-mail-index.h diff --git a/src/lib-index/Makefile.in b/src/lib-index/Makefile.in new file mode 100644 index 0000000..a25cb29 --- /dev/null +++ b/src/lib-index/Makefile.in @@ -0,0 +1,1285 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/lib-index +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \ + $(pkginc_lib_HEADERS) $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-mail-cache$(EXEEXT) \ + test-mail-cache-fields$(EXEEXT) test-mail-cache-purge$(EXEEXT) \ + test-mail-index$(EXEEXT) test-mail-index-map$(EXEEXT) \ + test-mail-index-modseq$(EXEEXT) \ + test-mail-index-sync-ext$(EXEEXT) \ + test-mail-index-transaction-finish$(EXEEXT) \ + test-mail-index-transaction-update$(EXEEXT) \ + test-mail-index-write$(EXEEXT) \ + test-mail-transaction-log-append$(EXEEXT) \ + test-mail-transaction-log-file$(EXEEXT) \ + test-mail-transaction-log-view$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +LTLIBRARIES = $(noinst_LTLIBRARIES) +libindex_la_LIBADD = +am_libindex_la_OBJECTS = mail-cache.lo mail-cache-decisions.lo \ + mail-cache-fields.lo mail-cache-lookup.lo mail-cache-purge.lo \ + mail-cache-transaction.lo mail-cache-sync-update.lo \ + mail-index.lo mail-index-alloc-cache.lo \ + mail-index-dummy-view.lo mail-index-fsck.lo mail-index-lock.lo \ + mail-index-map.lo mail-index-map-hdr.lo mail-index-map-read.lo \ + mail-index-modseq.lo mail-index-transaction.lo \ + mail-index-transaction-export.lo \ + mail-index-transaction-finish.lo \ + mail-index-transaction-sort-appends.lo \ + mail-index-transaction-update.lo \ + mail-index-transaction-view.lo mail-index-strmap.lo \ + mail-index-sync.lo mail-index-sync-ext.lo \ + mail-index-sync-keywords.lo mail-index-sync-update.lo \ + mail-index-util.lo mail-index-view.lo mail-index-view-sync.lo \ + mail-index-write.lo mail-transaction-log.lo \ + mail-transaction-log-append.lo mail-transaction-log-file.lo \ + mail-transaction-log-modseq.lo mail-transaction-log-view.lo \ + mailbox-log.lo +libindex_la_OBJECTS = $(am_libindex_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am_test_mail_cache_OBJECTS = test-mail-cache-common.$(OBJEXT) \ + test-mail-cache.$(OBJEXT) +test_mail_cache_OBJECTS = $(am_test_mail_cache_OBJECTS) +am_test_mail_cache_fields_OBJECTS = test-mail-cache-common.$(OBJEXT) \ + test-mail-cache-fields.$(OBJEXT) +test_mail_cache_fields_OBJECTS = $(am_test_mail_cache_fields_OBJECTS) +am_test_mail_cache_purge_OBJECTS = test-mail-cache-common.$(OBJEXT) \ + test-mail-cache-purge.$(OBJEXT) +test_mail_cache_purge_OBJECTS = $(am_test_mail_cache_purge_OBJECTS) +am_test_mail_index_OBJECTS = test-mail-index.$(OBJEXT) +test_mail_index_OBJECTS = $(am_test_mail_index_OBJECTS) +am_test_mail_index_map_OBJECTS = test-mail-index-map.$(OBJEXT) +test_mail_index_map_OBJECTS = $(am_test_mail_index_map_OBJECTS) +am_test_mail_index_modseq_OBJECTS = test-mail-index-modseq.$(OBJEXT) +test_mail_index_modseq_OBJECTS = $(am_test_mail_index_modseq_OBJECTS) +am_test_mail_index_sync_ext_OBJECTS = \ + test-mail-index-sync-ext.$(OBJEXT) +test_mail_index_sync_ext_OBJECTS = \ + $(am_test_mail_index_sync_ext_OBJECTS) +am_test_mail_index_transaction_finish_OBJECTS = \ + test-mail-index-transaction-finish.$(OBJEXT) +test_mail_index_transaction_finish_OBJECTS = \ + $(am_test_mail_index_transaction_finish_OBJECTS) +am_test_mail_index_transaction_update_OBJECTS = \ + test-mail-index-transaction-update.$(OBJEXT) +test_mail_index_transaction_update_OBJECTS = \ + $(am_test_mail_index_transaction_update_OBJECTS) +am_test_mail_index_write_OBJECTS = test-mail-index-write.$(OBJEXT) +test_mail_index_write_OBJECTS = $(am_test_mail_index_write_OBJECTS) +am_test_mail_transaction_log_append_OBJECTS = \ + test-mail-transaction-log-append.$(OBJEXT) +test_mail_transaction_log_append_OBJECTS = \ + $(am_test_mail_transaction_log_append_OBJECTS) +am_test_mail_transaction_log_file_OBJECTS = \ + test-mail-transaction-log-file.$(OBJEXT) +test_mail_transaction_log_file_OBJECTS = \ + $(am_test_mail_transaction_log_file_OBJECTS) +am_test_mail_transaction_log_view_OBJECTS = \ + test-mail-transaction-log-view.$(OBJEXT) +test_mail_transaction_log_view_OBJECTS = \ + $(am_test_mail_transaction_log_view_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/mail-cache-decisions.Plo \ + ./$(DEPDIR)/mail-cache-fields.Plo \ + ./$(DEPDIR)/mail-cache-lookup.Plo \ + ./$(DEPDIR)/mail-cache-purge.Plo \ + ./$(DEPDIR)/mail-cache-sync-update.Plo \ + ./$(DEPDIR)/mail-cache-transaction.Plo \ + ./$(DEPDIR)/mail-cache.Plo \ + ./$(DEPDIR)/mail-index-alloc-cache.Plo \ + ./$(DEPDIR)/mail-index-dummy-view.Plo \ + ./$(DEPDIR)/mail-index-fsck.Plo \ + ./$(DEPDIR)/mail-index-lock.Plo \ + ./$(DEPDIR)/mail-index-map-hdr.Plo \ + ./$(DEPDIR)/mail-index-map-read.Plo \ + ./$(DEPDIR)/mail-index-map.Plo \ + ./$(DEPDIR)/mail-index-modseq.Plo \ + ./$(DEPDIR)/mail-index-strmap.Plo \ + ./$(DEPDIR)/mail-index-sync-ext.Plo \ + ./$(DEPDIR)/mail-index-sync-keywords.Plo \ + ./$(DEPDIR)/mail-index-sync-update.Plo \ + ./$(DEPDIR)/mail-index-sync.Plo \ + ./$(DEPDIR)/mail-index-transaction-export.Plo \ + ./$(DEPDIR)/mail-index-transaction-finish.Plo \ + ./$(DEPDIR)/mail-index-transaction-sort-appends.Plo \ + ./$(DEPDIR)/mail-index-transaction-update.Plo \ + ./$(DEPDIR)/mail-index-transaction-view.Plo \ + ./$(DEPDIR)/mail-index-transaction.Plo \ + ./$(DEPDIR)/mail-index-util.Plo \ + ./$(DEPDIR)/mail-index-view-sync.Plo \ + ./$(DEPDIR)/mail-index-view.Plo \ + ./$(DEPDIR)/mail-index-write.Plo ./$(DEPDIR)/mail-index.Plo \ + ./$(DEPDIR)/mail-transaction-log-append.Plo \ + ./$(DEPDIR)/mail-transaction-log-file.Plo \ + ./$(DEPDIR)/mail-transaction-log-modseq.Plo \ + ./$(DEPDIR)/mail-transaction-log-view.Plo \ + ./$(DEPDIR)/mail-transaction-log.Plo \ + ./$(DEPDIR)/mailbox-log.Plo \ + ./$(DEPDIR)/test-mail-cache-common.Po \ + ./$(DEPDIR)/test-mail-cache-fields.Po \ + ./$(DEPDIR)/test-mail-cache-purge.Po \ + ./$(DEPDIR)/test-mail-cache.Po \ + ./$(DEPDIR)/test-mail-index-map.Po \ + ./$(DEPDIR)/test-mail-index-modseq.Po \ + ./$(DEPDIR)/test-mail-index-sync-ext.Po \ + ./$(DEPDIR)/test-mail-index-transaction-finish.Po \ + ./$(DEPDIR)/test-mail-index-transaction-update.Po \ + ./$(DEPDIR)/test-mail-index-write.Po \ + ./$(DEPDIR)/test-mail-index.Po \ + ./$(DEPDIR)/test-mail-transaction-log-append.Po \ + ./$(DEPDIR)/test-mail-transaction-log-file.Po \ + ./$(DEPDIR)/test-mail-transaction-log-view.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libindex_la_SOURCES) $(test_mail_cache_SOURCES) \ + $(test_mail_cache_fields_SOURCES) \ + $(test_mail_cache_purge_SOURCES) $(test_mail_index_SOURCES) \ + $(test_mail_index_map_SOURCES) \ + $(test_mail_index_modseq_SOURCES) \ + $(test_mail_index_sync_ext_SOURCES) \ + $(test_mail_index_transaction_finish_SOURCES) \ + $(test_mail_index_transaction_update_SOURCES) \ + $(test_mail_index_write_SOURCES) \ + $(test_mail_transaction_log_append_SOURCES) \ + $(test_mail_transaction_log_file_SOURCES) \ + $(test_mail_transaction_log_view_SOURCES) +DIST_SOURCES = $(libindex_la_SOURCES) $(test_mail_cache_SOURCES) \ + $(test_mail_cache_fields_SOURCES) \ + $(test_mail_cache_purge_SOURCES) $(test_mail_index_SOURCES) \ + $(test_mail_index_map_SOURCES) \ + $(test_mail_index_modseq_SOURCES) \ + $(test_mail_index_sync_ext_SOURCES) \ + $(test_mail_index_transaction_finish_SOURCES) \ + $(test_mail_index_transaction_update_SOURCES) \ + $(test_mail_index_write_SOURCES) \ + $(test_mail_transaction_log_append_SOURCES) \ + $(test_mail_transaction_log_file_SOURCES) \ + $(test_mail_transaction_log_view_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libindex.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-mail + +libindex_la_SOURCES = \ + mail-cache.c \ + mail-cache-decisions.c \ + mail-cache-fields.c \ + mail-cache-lookup.c \ + mail-cache-purge.c \ + mail-cache-transaction.c \ + mail-cache-sync-update.c \ + mail-index.c \ + mail-index-alloc-cache.c \ + mail-index-dummy-view.c \ + mail-index-fsck.c \ + mail-index-lock.c \ + mail-index-map.c \ + mail-index-map-hdr.c \ + mail-index-map-read.c \ + mail-index-modseq.c \ + mail-index-transaction.c \ + mail-index-transaction-export.c \ + mail-index-transaction-finish.c \ + mail-index-transaction-sort-appends.c \ + mail-index-transaction-update.c \ + mail-index-transaction-view.c \ + mail-index-strmap.c \ + mail-index-sync.c \ + mail-index-sync-ext.c \ + mail-index-sync-keywords.c \ + mail-index-sync-update.c \ + mail-index-util.c \ + mail-index-view.c \ + mail-index-view-sync.c \ + mail-index-write.c \ + mail-transaction-log.c \ + mail-transaction-log-append.c \ + mail-transaction-log-file.c \ + mail-transaction-log-modseq.c \ + mail-transaction-log-view.c \ + mailbox-log.c + +headers = \ + mail-cache.h \ + mail-cache-private.h \ + mail-index.h \ + mail-index-alloc-cache.h \ + mail-index-modseq.h \ + mail-index-private.h \ + mail-index-strmap.h \ + mail-index-sync-private.h \ + mail-index-transaction-private.h \ + mail-index-util.h \ + mail-index-view-private.h \ + mail-transaction-log.h \ + mail-transaction-log-private.h \ + mail-transaction-log-view-private.h \ + mailbox-log.h + +test_programs = \ + test-mail-cache \ + test-mail-cache-fields \ + test-mail-cache-purge \ + test-mail-index \ + test-mail-index-map \ + test-mail-index-modseq \ + test-mail-index-sync-ext \ + test-mail-index-transaction-finish \ + test-mail-index-transaction-update \ + test-mail-index-write \ + test-mail-transaction-log-append \ + test-mail-transaction-log-file \ + test-mail-transaction-log-view + +test_libs = \ + mail-index-util.lo \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_deps = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_cache_SOURCES = test-mail-cache-common.c test-mail-cache.c +test_mail_cache_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_cache_DEPENDENCIES = $(test_deps) +test_mail_cache_fields_SOURCES = test-mail-cache-common.c test-mail-cache-fields.c +test_mail_cache_fields_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_cache_fields_DEPENDENCIES = $(test_deps) +test_mail_cache_purge_SOURCES = test-mail-cache-common.c test-mail-cache-purge.c +test_mail_cache_purge_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_cache_purge_DEPENDENCIES = $(test_deps) +test_mail_index_SOURCES = test-mail-index.c +test_mail_index_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_DEPENDENCIES = $(test_deps) +test_mail_index_map_SOURCES = test-mail-index-map.c +test_mail_index_map_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_map_DEPENDENCIES = $(test_deps) +test_mail_index_modseq_SOURCES = test-mail-index-modseq.c +test_mail_index_modseq_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_modseq_DEPENDENCIES = $(test_deps) +test_mail_index_sync_ext_SOURCES = test-mail-index-sync-ext.c +test_mail_index_sync_ext_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_index_sync_ext_DEPENDENCIES = $(test_deps) +test_mail_index_transaction_finish_SOURCES = test-mail-index-transaction-finish.c +test_mail_index_transaction_finish_LDADD = mail-index-transaction-finish.lo $(test_libs) +test_mail_index_transaction_finish_DEPENDENCIES = $(test_deps) +test_mail_index_transaction_update_SOURCES = test-mail-index-transaction-update.c +test_mail_index_transaction_update_LDADD = mail-index-transaction-update.lo $(test_libs) +test_mail_index_transaction_update_DEPENDENCIES = $(test_deps) +test_mail_index_write_SOURCES = test-mail-index-write.c +test_mail_index_write_LDADD = mail-index-write.lo $(test_libs) +test_mail_index_write_DEPENDENCIES = $(test_deps) +test_mail_transaction_log_append_SOURCES = test-mail-transaction-log-append.c +test_mail_transaction_log_append_LDADD = mail-transaction-log-append.lo $(test_libs) +test_mail_transaction_log_append_DEPENDENCIES = $(test_deps) +test_mail_transaction_log_file_SOURCES = test-mail-transaction-log-file.c +test_mail_transaction_log_file_LDADD = $(noinst_LTLIBRARIES) $(test_libs) +test_mail_transaction_log_file_DEPENDENCIES = $(test_deps) +test_mail_transaction_log_view_SOURCES = test-mail-transaction-log-view.c +test_mail_transaction_log_view_LDADD = mail-transaction-log-view.lo $(test_libs) +test_mail_transaction_log_view_DEPENDENCIES = $(test_deps) +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +noinst_HEADERS = \ + test-mail-cache.h \ + test-mail-index.h + +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-index/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-index/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libindex.la: $(libindex_la_OBJECTS) $(libindex_la_DEPENDENCIES) $(EXTRA_libindex_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libindex_la_OBJECTS) $(libindex_la_LIBADD) $(LIBS) + +test-mail-cache$(EXEEXT): $(test_mail_cache_OBJECTS) $(test_mail_cache_DEPENDENCIES) $(EXTRA_test_mail_cache_DEPENDENCIES) + @rm -f test-mail-cache$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_cache_OBJECTS) $(test_mail_cache_LDADD) $(LIBS) + +test-mail-cache-fields$(EXEEXT): $(test_mail_cache_fields_OBJECTS) $(test_mail_cache_fields_DEPENDENCIES) $(EXTRA_test_mail_cache_fields_DEPENDENCIES) + @rm -f test-mail-cache-fields$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_cache_fields_OBJECTS) $(test_mail_cache_fields_LDADD) $(LIBS) + +test-mail-cache-purge$(EXEEXT): $(test_mail_cache_purge_OBJECTS) $(test_mail_cache_purge_DEPENDENCIES) $(EXTRA_test_mail_cache_purge_DEPENDENCIES) + @rm -f test-mail-cache-purge$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_cache_purge_OBJECTS) $(test_mail_cache_purge_LDADD) $(LIBS) + +test-mail-index$(EXEEXT): $(test_mail_index_OBJECTS) $(test_mail_index_DEPENDENCIES) $(EXTRA_test_mail_index_DEPENDENCIES) + @rm -f test-mail-index$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_index_OBJECTS) $(test_mail_index_LDADD) $(LIBS) + +test-mail-index-map$(EXEEXT): $(test_mail_index_map_OBJECTS) $(test_mail_index_map_DEPENDENCIES) $(EXTRA_test_mail_index_map_DEPENDENCIES) + @rm -f test-mail-index-map$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_index_map_OBJECTS) $(test_mail_index_map_LDADD) $(LIBS) + +test-mail-index-modseq$(EXEEXT): $(test_mail_index_modseq_OBJECTS) $(test_mail_index_modseq_DEPENDENCIES) $(EXTRA_test_mail_index_modseq_DEPENDENCIES) + @rm -f test-mail-index-modseq$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_index_modseq_OBJECTS) $(test_mail_index_modseq_LDADD) $(LIBS) + +test-mail-index-sync-ext$(EXEEXT): $(test_mail_index_sync_ext_OBJECTS) $(test_mail_index_sync_ext_DEPENDENCIES) $(EXTRA_test_mail_index_sync_ext_DEPENDENCIES) + @rm -f test-mail-index-sync-ext$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_index_sync_ext_OBJECTS) $(test_mail_index_sync_ext_LDADD) $(LIBS) + +test-mail-index-transaction-finish$(EXEEXT): $(test_mail_index_transaction_finish_OBJECTS) $(test_mail_index_transaction_finish_DEPENDENCIES) $(EXTRA_test_mail_index_transaction_finish_DEPENDENCIES) + @rm -f test-mail-index-transaction-finish$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_index_transaction_finish_OBJECTS) $(test_mail_index_transaction_finish_LDADD) $(LIBS) + +test-mail-index-transaction-update$(EXEEXT): $(test_mail_index_transaction_update_OBJECTS) $(test_mail_index_transaction_update_DEPENDENCIES) $(EXTRA_test_mail_index_transaction_update_DEPENDENCIES) + @rm -f test-mail-index-transaction-update$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_index_transaction_update_OBJECTS) $(test_mail_index_transaction_update_LDADD) $(LIBS) + +test-mail-index-write$(EXEEXT): $(test_mail_index_write_OBJECTS) $(test_mail_index_write_DEPENDENCIES) $(EXTRA_test_mail_index_write_DEPENDENCIES) + @rm -f test-mail-index-write$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_index_write_OBJECTS) $(test_mail_index_write_LDADD) $(LIBS) + +test-mail-transaction-log-append$(EXEEXT): $(test_mail_transaction_log_append_OBJECTS) $(test_mail_transaction_log_append_DEPENDENCIES) $(EXTRA_test_mail_transaction_log_append_DEPENDENCIES) + @rm -f test-mail-transaction-log-append$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_transaction_log_append_OBJECTS) $(test_mail_transaction_log_append_LDADD) $(LIBS) + +test-mail-transaction-log-file$(EXEEXT): $(test_mail_transaction_log_file_OBJECTS) $(test_mail_transaction_log_file_DEPENDENCIES) $(EXTRA_test_mail_transaction_log_file_DEPENDENCIES) + @rm -f test-mail-transaction-log-file$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_transaction_log_file_OBJECTS) $(test_mail_transaction_log_file_LDADD) $(LIBS) + +test-mail-transaction-log-view$(EXEEXT): $(test_mail_transaction_log_view_OBJECTS) $(test_mail_transaction_log_view_DEPENDENCIES) $(EXTRA_test_mail_transaction_log_view_DEPENDENCIES) + @rm -f test-mail-transaction-log-view$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_mail_transaction_log_view_OBJECTS) $(test_mail_transaction_log_view_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-cache-decisions.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-cache-fields.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-cache-lookup.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-cache-purge.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-cache-sync-update.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-cache-transaction.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-cache.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-alloc-cache.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-dummy-view.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-fsck.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-lock.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-map-hdr.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-map-read.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-map.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-modseq.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-strmap.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-sync-ext.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-sync-keywords.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-sync-update.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-transaction-export.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-transaction-finish.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-transaction-sort-appends.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-transaction-update.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-transaction-view.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-transaction.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-util.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-view-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-view.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index-write.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-index.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-transaction-log-append.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-transaction-log-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-transaction-log-modseq.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-transaction-log-view.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mail-transaction-log.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mailbox-log.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-cache-common.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-cache-fields.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-cache-purge.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-cache.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-index-map.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-index-modseq.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-index-sync-ext.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-index-transaction-finish.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-index-transaction-update.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-index-write.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-index.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-transaction-log-append.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-transaction-log-file.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-mail-transaction-log-view.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/mail-cache-decisions.Plo + -rm -f ./$(DEPDIR)/mail-cache-fields.Plo + -rm -f ./$(DEPDIR)/mail-cache-lookup.Plo + -rm -f ./$(DEPDIR)/mail-cache-purge.Plo + -rm -f ./$(DEPDIR)/mail-cache-sync-update.Plo + -rm -f ./$(DEPDIR)/mail-cache-transaction.Plo + -rm -f ./$(DEPDIR)/mail-cache.Plo + -rm -f ./$(DEPDIR)/mail-index-alloc-cache.Plo + -rm -f ./$(DEPDIR)/mail-index-dummy-view.Plo + -rm -f ./$(DEPDIR)/mail-index-fsck.Plo + -rm -f ./$(DEPDIR)/mail-index-lock.Plo + -rm -f ./$(DEPDIR)/mail-index-map-hdr.Plo + -rm -f ./$(DEPDIR)/mail-index-map-read.Plo + -rm -f ./$(DEPDIR)/mail-index-map.Plo + -rm -f ./$(DEPDIR)/mail-index-modseq.Plo + -rm -f ./$(DEPDIR)/mail-index-strmap.Plo + -rm -f ./$(DEPDIR)/mail-index-sync-ext.Plo + -rm -f ./$(DEPDIR)/mail-index-sync-keywords.Plo + -rm -f ./$(DEPDIR)/mail-index-sync-update.Plo + -rm -f ./$(DEPDIR)/mail-index-sync.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-export.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-finish.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-sort-appends.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-update.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-view.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction.Plo + -rm -f ./$(DEPDIR)/mail-index-util.Plo + -rm -f ./$(DEPDIR)/mail-index-view-sync.Plo + -rm -f ./$(DEPDIR)/mail-index-view.Plo + -rm -f ./$(DEPDIR)/mail-index-write.Plo + -rm -f ./$(DEPDIR)/mail-index.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-append.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-file.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-modseq.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-view.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log.Plo + -rm -f ./$(DEPDIR)/mailbox-log.Plo + -rm -f ./$(DEPDIR)/test-mail-cache-common.Po + -rm -f ./$(DEPDIR)/test-mail-cache-fields.Po + -rm -f ./$(DEPDIR)/test-mail-cache-purge.Po + -rm -f ./$(DEPDIR)/test-mail-cache.Po + -rm -f ./$(DEPDIR)/test-mail-index-map.Po + -rm -f ./$(DEPDIR)/test-mail-index-modseq.Po + -rm -f ./$(DEPDIR)/test-mail-index-sync-ext.Po + -rm -f ./$(DEPDIR)/test-mail-index-transaction-finish.Po + -rm -f ./$(DEPDIR)/test-mail-index-transaction-update.Po + -rm -f ./$(DEPDIR)/test-mail-index-write.Po + -rm -f ./$(DEPDIR)/test-mail-index.Po + -rm -f ./$(DEPDIR)/test-mail-transaction-log-append.Po + -rm -f ./$(DEPDIR)/test-mail-transaction-log-file.Po + -rm -f ./$(DEPDIR)/test-mail-transaction-log-view.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/mail-cache-decisions.Plo + -rm -f ./$(DEPDIR)/mail-cache-fields.Plo + -rm -f ./$(DEPDIR)/mail-cache-lookup.Plo + -rm -f ./$(DEPDIR)/mail-cache-purge.Plo + -rm -f ./$(DEPDIR)/mail-cache-sync-update.Plo + -rm -f ./$(DEPDIR)/mail-cache-transaction.Plo + -rm -f ./$(DEPDIR)/mail-cache.Plo + -rm -f ./$(DEPDIR)/mail-index-alloc-cache.Plo + -rm -f ./$(DEPDIR)/mail-index-dummy-view.Plo + -rm -f ./$(DEPDIR)/mail-index-fsck.Plo + -rm -f ./$(DEPDIR)/mail-index-lock.Plo + -rm -f ./$(DEPDIR)/mail-index-map-hdr.Plo + -rm -f ./$(DEPDIR)/mail-index-map-read.Plo + -rm -f ./$(DEPDIR)/mail-index-map.Plo + -rm -f ./$(DEPDIR)/mail-index-modseq.Plo + -rm -f ./$(DEPDIR)/mail-index-strmap.Plo + -rm -f ./$(DEPDIR)/mail-index-sync-ext.Plo + -rm -f ./$(DEPDIR)/mail-index-sync-keywords.Plo + -rm -f ./$(DEPDIR)/mail-index-sync-update.Plo + -rm -f ./$(DEPDIR)/mail-index-sync.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-export.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-finish.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-sort-appends.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-update.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction-view.Plo + -rm -f ./$(DEPDIR)/mail-index-transaction.Plo + -rm -f ./$(DEPDIR)/mail-index-util.Plo + -rm -f ./$(DEPDIR)/mail-index-view-sync.Plo + -rm -f ./$(DEPDIR)/mail-index-view.Plo + -rm -f ./$(DEPDIR)/mail-index-write.Plo + -rm -f ./$(DEPDIR)/mail-index.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-append.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-file.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-modseq.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log-view.Plo + -rm -f ./$(DEPDIR)/mail-transaction-log.Plo + -rm -f ./$(DEPDIR)/mailbox-log.Plo + -rm -f ./$(DEPDIR)/test-mail-cache-common.Po + -rm -f ./$(DEPDIR)/test-mail-cache-fields.Po + -rm -f ./$(DEPDIR)/test-mail-cache-purge.Po + -rm -f ./$(DEPDIR)/test-mail-cache.Po + -rm -f ./$(DEPDIR)/test-mail-index-map.Po + -rm -f ./$(DEPDIR)/test-mail-index-modseq.Po + -rm -f ./$(DEPDIR)/test-mail-index-sync-ext.Po + -rm -f ./$(DEPDIR)/test-mail-index-transaction-finish.Po + -rm -f ./$(DEPDIR)/test-mail-index-transaction-update.Po + -rm -f ./$(DEPDIR)/test-mail-index-write.Po + -rm -f ./$(DEPDIR)/test-mail-index.Po + -rm -f ./$(DEPDIR)/test-mail-transaction-log-append.Po + -rm -f ./$(DEPDIR)/test-mail-transaction-log-file.Po + -rm -f ./$(DEPDIR)/test-mail-transaction-log-view.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-pkginc_libHEADERS install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-index/mail-cache-decisions.c b/src/lib-index/mail-cache-decisions.c new file mode 100644 index 0000000..93435d0 --- /dev/null +++ b/src/lib-index/mail-cache-decisions.c @@ -0,0 +1,238 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* + IMAP clients can work in many different ways. There are basically 2 + types: + + 1. Online clients that ask for the same information multiple times (e.g. + webmails, Pine) + + 2. Offline clients that usually download first some of the interesting + message headers and only after that the message bodies (possibly + automatically, or possibly only when the user opens the mail). Most + non-webmail IMAP clients behave like this. + + Cache file is extremely helpful with the type 1 clients. The first time + that client requests message headers or some other metadata they're + stored into the cache file. The second time they ask for the same + information Dovecot can now get it quickly from the cache file instead + of opening the message and parsing the headers. + + For type 2 clients the cache file is also somewhat helpful if client + fetches any initial metadata. Some of the information is helpful in any + case, for example it's required to know the message's virtual size when + downloading the message with IMAP. Without the virtual size being in cache + Dovecot first has to read the whole message first to calculate it, which + increases CPU usage. + + Only the specified fields that client(s) have asked for earlier are + stored into cache file. This allows Dovecot to be adaptive to different + clients' needs and still not waste disk space (and cause extra disk + I/O!) for fields that client never needs. + + Dovecot can cache fields either permanently or temporarily. Temporarily + cached fields are dropped from the cache file after about a week. + Dovecot uses two rules to determine when data should be cached + permanently instead of temporarily: + + 1. Client accessed messages in non-sequential order within this session. + This most likely means it doesn't have a local cache. + + 2. Client accessed a message older than one week. + + These rules might not always work optimally, so Dovecot also re-evaluates + the caching decisions once in a while: + + - When caching decision is YES (permanently cache the field), the field's + last_used is updated only when the caching decision has been verified to + be correct. + + - When caching decision is TEMP, the last_used is updated whenever the field + is accessed. + + - When last_used becomes 30 days old (or unaccessed_field_drop_secs) a + YES caching decision is changed to TEMP. + + - When last_used becomes 60 days old (or 2*unaccessed_field_drop_secs) a + TEMP caching decision is changed to NO. +*/ + +#include "lib.h" +#include "ioloop.h" +#include "mail-cache-private.h" + +const char *mail_cache_decision_to_string(enum mail_cache_decision_type dec) +{ + switch (dec & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) { + case MAIL_CACHE_DECISION_NO: + return "no"; + case MAIL_CACHE_DECISION_TEMP: + return "temp"; + case MAIL_CACHE_DECISION_YES: + return "yes"; + } + i_unreached(); +} + +struct event_passthrough * +mail_cache_decision_changed_event(struct mail_cache *cache, struct event *event, + unsigned int field) +{ + return event_create_passthrough(event)-> + set_name("mail_cache_decision_changed")-> + add_str("field", cache->fields[field].field.name)-> + add_int("last_used", cache->fields[field].field.last_used); +} + +static void +mail_cache_update_last_used(struct mail_cache *cache, unsigned int field) +{ + cache->fields[field].field.last_used = (uint32_t)ioloop_time; + if (cache->field_file_map[field] != (uint32_t)-1) + cache->field_header_write_pending = TRUE; +} + +void mail_cache_decision_state_update(struct mail_cache_view *view, + uint32_t seq, unsigned int field) +{ + struct mail_cache *cache = view->cache; + enum mail_cache_decision_type dec; + const struct mail_index_header *hdr; + uint32_t uid; + + i_assert(field < cache->fields_count); + + if (view->no_decision_updates) + return; + + dec = cache->fields[field].field.decision; + if (dec == (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED)) { + /* don't update last_used */ + return; + } + + /* update last_used about once a day */ + bool last_used_need_update = + ioloop_time - cache->fields[field].field.last_used > 3600*24; + + if (dec == MAIL_CACHE_DECISION_NO || + (dec & MAIL_CACHE_DECISION_FORCED) != 0) { + /* a) forced decision + b) not cached, mail_cache_decision_add() will handle this */ + if (last_used_need_update) + mail_cache_update_last_used(cache, field); + return; + } + if (dec == MAIL_CACHE_DECISION_YES) { + if (!last_used_need_update) + return; + /* update last_used only when we can confirm that the YES + decision is still correct. */ + } else { + /* see if we want to change decision from TEMP to YES */ + i_assert(dec == MAIL_CACHE_DECISION_TEMP); + if (last_used_need_update) + mail_cache_update_last_used(cache, field); + } + + mail_index_lookup_uid(view->view, seq, &uid); + hdr = mail_index_get_header(view->view); + + if (uid >= cache->fields[field].uid_highwater && + uid >= hdr->day_first_uid[7]) { + cache->fields[field].uid_highwater = uid; + } else if (dec == MAIL_CACHE_DECISION_YES) { + /* Confirmed that we still want to preserve YES as cache + decision. We can update last_used now. */ + i_assert(last_used_need_update); + mail_cache_update_last_used(cache, field); + } else { + /* a) nonordered access within this session. if client doesn't + request messages in growing order, we assume it doesn't + have a permanent local cache. + b) accessing message older than one week. assume it's a + client with no local cache. if it was just a new client + generating the local cache for the first time, we'll + drop back to TEMP within few months. */ + i_assert(dec == MAIL_CACHE_DECISION_TEMP); + cache->fields[field].field.decision = MAIL_CACHE_DECISION_YES; + cache->fields[field].decision_dirty = TRUE; + cache->field_header_write_pending = TRUE; + + const char *reason = uid < hdr->day_first_uid[7] ? + "old_mail" : "unordered_access"; + struct event_passthrough *e = + mail_cache_decision_changed_event( + view->cache, view->cache->event, field)-> + add_str("reason", reason)-> + add_int("uid", uid)-> + add_str("old_decision", "temp")-> + add_str("new_decision", "yes"); + e_debug(e->event(), "Changing field %s decision temp -> yes (uid=%u)", + cache->fields[field].field.name, uid); + } +} + +void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq, + unsigned int field) +{ + struct mail_cache *cache = view->cache; + struct mail_cache_field_private *priv; + uint32_t uid; + + i_assert(field < cache->fields_count); + + if (view->no_decision_updates) + return; + + priv = &cache->fields[field]; + if (priv->field.decision != MAIL_CACHE_DECISION_NO && + priv->field.last_used != 0) { + /* a) forced decision + b) we're already caching it, so it just wasn't in cache */ + return; + } + + /* field used the first time */ + if (priv->field.decision == MAIL_CACHE_DECISION_NO) + priv->field.decision = MAIL_CACHE_DECISION_TEMP; + priv->field.last_used = ioloop_time; + priv->decision_dirty = TRUE; + cache->field_header_write_pending = TRUE; + + mail_index_lookup_uid(view->view, seq, &uid); + priv->uid_highwater = uid; + + const char *new_decision = + mail_cache_decision_to_string(priv->field.decision); + struct event_passthrough *e = + mail_cache_decision_changed_event(cache, cache->event, field)-> + add_str("reason", "add")-> + add_int("uid", uid)-> + add_str("old_decision", "no")-> + add_str("new_decision", new_decision); + e_debug(e->event(), "Adding field %s to cache for the first time (uid=%u)", + priv->field.name, uid); +} + +int mail_cache_decisions_copy(struct mail_cache *src, struct mail_cache *dst) +{ + if (mail_cache_open_and_verify(src) < 0) + return -1; + if (MAIL_CACHE_IS_UNUSABLE(src)) + return 0; /* no caching decisions */ + + unsigned int count = 0; + struct mail_cache_field *fields = + mail_cache_register_get_list(src, pool_datastack_create(), &count); + i_assert(fields != NULL || count == 0); + if (count > 0) + mail_cache_register_fields(dst, fields, count); + + /* Destination cache isn't expected to exist yet, so use purging + to create it. Setting field_header_write_pending also guarantees + that the fields are updated even if the cache was already created + and no purging was done. */ + dst->field_header_write_pending = TRUE; + return mail_cache_purge(dst, 0, "copy cache decisions"); +} diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c new file mode 100644 index 0000000..429e0d2 --- /dev/null +++ b/src/lib-index/mail-cache-fields.c @@ -0,0 +1,660 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "buffer.h" +#include "hash.h" +#include "file-cache.h" +#include "read-full.h" +#include "write-full.h" +#include "mmap-util.h" +#include "mail-cache-private.h" + +#include <stddef.h> + +#define CACHE_FIELD_IS_NEWLY_WANTED(cache, field_idx) \ + ((cache)->field_file_map[field_idx] == (uint32_t)-1 && \ + (cache)->fields[field_idx].used) + +static bool field_has_fixed_size(enum mail_cache_field_type type) +{ + switch (type) { + case MAIL_CACHE_FIELD_FIXED_SIZE: + case MAIL_CACHE_FIELD_BITMASK: + return TRUE; + case MAIL_CACHE_FIELD_VARIABLE_SIZE: + case MAIL_CACHE_FIELD_STRING: + case MAIL_CACHE_FIELD_HEADER: + return FALSE; + + case MAIL_CACHE_FIELD_COUNT: + break; + } + + i_unreached(); + return FALSE; +} + +static bool field_decision_is_valid(enum mail_cache_decision_type type) +{ + switch (type & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) { + case MAIL_CACHE_DECISION_NO: + case MAIL_CACHE_DECISION_TEMP: + case MAIL_CACHE_DECISION_YES: + return TRUE; + default: + return FALSE; + } +} + +static int field_type_verify(struct mail_cache *cache, unsigned int idx, + enum mail_cache_field_type type, unsigned int size) +{ + const struct mail_cache_field *field = &cache->fields[idx].field; + + if (field->type != type) { + mail_cache_set_corrupted(cache, + "registered field %s type changed", field->name); + return -1; + } + if (field->field_size != size && field_has_fixed_size(type)) { + mail_cache_set_corrupted(cache, + "registered field %s size changed", field->name); + return -1; + } + return 0; +} + +static void +mail_cache_field_update(struct mail_cache *cache, + const struct mail_cache_field *newfield) +{ + struct mail_cache_field_private *orig; + bool initial_registering; + + i_assert(newfield->type < MAIL_CACHE_FIELD_COUNT); + + /* are we still doing the initial cache field registering for + internal fields and for mail_*cache_fields settings? */ + initial_registering = cache->file_fields_count == 0; + + orig = &cache->fields[newfield->idx]; + if ((newfield->decision & MAIL_CACHE_DECISION_FORCED) != 0 || + ((orig->field.decision & MAIL_CACHE_DECISION_FORCED) == 0 && + newfield->decision > orig->field.decision)) { + orig->field.decision = newfield->decision; + if (!initial_registering) + orig->decision_dirty = TRUE; + } + if (orig->field.last_used < newfield->last_used) { + orig->field.last_used = newfield->last_used; + if (!initial_registering) + orig->decision_dirty = TRUE; + } + if (orig->decision_dirty) + cache->field_header_write_pending = TRUE; + + (void)field_type_verify(cache, newfield->idx, + newfield->type, newfield->field_size); +} + +void mail_cache_register_fields(struct mail_cache *cache, + struct mail_cache_field *fields, + unsigned int fields_count) +{ + char *name; + void *value; + unsigned int new_idx; + unsigned int i, j, registered_count; + + new_idx = cache->fields_count; + for (i = 0; i < fields_count; i++) { + if (hash_table_lookup_full(cache->field_name_hash, + fields[i].name, &name, &value)) { + fields[i].idx = POINTER_CAST_TO(value, unsigned int); + mail_cache_field_update(cache, &fields[i]); + continue; + } + + /* check if the same header is being registered in the + same field array */ + for (j = 0; j < i; j++) { + if (strcasecmp(fields[i].name, fields[j].name) == 0) { + fields[i].idx = fields[j].idx; + break; + } + } + + if (j == i) + fields[i].idx = new_idx++; + } + + if (new_idx == cache->fields_count) + return; + + /* @UNSAFE */ + cache->fields = i_realloc_type(cache->fields, + struct mail_cache_field_private, + cache->fields_count, new_idx); + cache->field_file_map = + i_realloc_type(cache->field_file_map, uint32_t, + cache->fields_count, new_idx); + + registered_count = cache->fields_count; + for (i = 0; i < fields_count; i++) { + unsigned int idx = fields[i].idx; + + if (idx < registered_count) + continue; + + /* new index - save it */ + name = p_strdup(cache->field_pool, fields[i].name); + cache->fields[idx].field = fields[i]; + cache->fields[idx].field.name = name; + cache->fields[idx].field.last_used = fields[i].last_used; + cache->field_file_map[idx] = (uint32_t)-1; + + if (!field_has_fixed_size(cache->fields[idx].field.type)) + cache->fields[idx].field.field_size = UINT_MAX; + + hash_table_insert(cache->field_name_hash, name, + POINTER_CAST(idx)); + registered_count++; + } + i_assert(registered_count == new_idx); + cache->fields_count = new_idx; +} + +unsigned int +mail_cache_register_lookup(struct mail_cache *cache, const char *name) +{ + char *key; + void *value; + + if (hash_table_lookup_full(cache->field_name_hash, name, &key, &value)) + return POINTER_CAST_TO(value, unsigned int); + else + return UINT_MAX; +} + +const struct mail_cache_field * +mail_cache_register_get_field(struct mail_cache *cache, unsigned int field_idx) +{ + i_assert(field_idx < cache->fields_count); + + return &cache->fields[field_idx].field; +} + +struct mail_cache_field * +mail_cache_register_get_list(struct mail_cache *cache, pool_t pool, + unsigned int *count_r) +{ + struct mail_cache_field *list; + unsigned int i; + + if (!cache->opened) + (void)mail_cache_open_and_verify(cache); + + list = cache->fields_count == 0 ? NULL : + p_new(pool, struct mail_cache_field, cache->fields_count); + for (i = 0; i < cache->fields_count; i++) { + list[i] = cache->fields[i].field; + list[i].name = p_strdup(pool, list[i].name); + } + + *count_r = cache->fields_count; + return list; +} + +static int +mail_cache_header_fields_get_offset(struct mail_cache *cache, + uint32_t *offset_r, + const struct mail_cache_header_fields **field_hdr_r) +{ + const struct mail_cache_header_fields *field_hdr; + struct mail_cache_header_fields tmp_field_hdr; + const void *data; + uint32_t offset = 0, next_offset, field_hdr_size; + unsigned int next_count = 0; + int ret; + + if (MAIL_CACHE_IS_UNUSABLE(cache)) { + *offset_r = 0; + if (field_hdr_r != NULL) + *field_hdr_r = NULL; + return 0; + } + + /* find the latest header */ + offset = 0; + next_offset = cache->last_field_header_offset != 0 ? + cache->last_field_header_offset : + mail_index_offset_to_uint32(cache->hdr->field_header_offset); + while (next_offset != 0) { + if (next_offset == offset) { + mail_cache_set_corrupted(cache, + "next_offset in field header loops"); + return -1; + } + /* In Dovecot v2.2+ we don't try to use any holes, + so next_offset must always be larger than current offset. + also makes it easier to guarantee there aren't any loops + (which we don't bother doing for old files) */ + if (next_offset < offset && cache->hdr->minor_version != 0) { + mail_cache_set_corrupted(cache, + "next_offset in field header decreases"); + return -1; + } + offset = next_offset; + + if (cache->mmap_base != NULL || cache->map_with_read) { + ret = mail_cache_map(cache, offset, sizeof(*field_hdr), + &data); + if (ret <= 0) { + if (ret < 0) + return -1; + mail_cache_set_corrupted(cache, + "header field next_offset points outside file"); + return -1; + } + field_hdr = data; + } else { + /* if we need to follow multiple offsets to get to + the last one, it's faster to just pread() the file + instead of going through cache */ + ret = pread_full(cache->fd, &tmp_field_hdr, + sizeof(tmp_field_hdr), offset); + if (ret < 0) { + mail_cache_set_syscall_error(cache, "pread()"); + return -1; + } + if (ret == 0) { + mail_cache_set_corrupted(cache, + "header field next_offset points outside file"); + return -1; + } + field_hdr = &tmp_field_hdr; + } + + next_offset = + mail_index_offset_to_uint32(field_hdr->next_offset); + next_count++; + } + + if (offset == 0) { + mail_cache_set_corrupted(cache, "missing header fields"); + return -1; + } + cache->last_field_header_offset = offset; + + if (next_count > cache->index->optimization_set.cache.purge_header_continue_count) { + mail_cache_purge_later(cache, t_strdup_printf( + "Too many continued headers (%u)", next_count)); + } + + if (field_hdr_r != NULL) { + /* detect corrupted size later */ + field_hdr_size = I_MAX(field_hdr->size, sizeof(*field_hdr)); + if (cache->file_cache != NULL) { + /* invalidate the cache fields area to make sure we + get the latest cache decisions/last_used fields */ + file_cache_invalidate(cache->file_cache, offset, + field_hdr_size); + } + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + ret = mail_cache_map(cache, offset, field_hdr_size, &data); + if (ret < 0) + return -1; + if (ret == 0) { + mail_cache_set_corrupted(cache, + "header field size outside file"); + return -1; + } + *field_hdr_r = data; + } + *offset_r = offset; + return 0; +} + +int mail_cache_header_fields_read(struct mail_cache *cache) +{ + const struct mail_cache_header_fields *field_hdr; + struct mail_cache_field field; + const uint32_t *last_used, *sizes; + const uint8_t *types, *decisions; + const char *p, *names, *end; + char *orig_key; + void *orig_value; + unsigned int fidx, new_fields_count; + struct mail_cache_purge_drop_ctx drop_ctx; + uint32_t offset, i; + + if (mail_cache_header_fields_get_offset(cache, &offset, &field_hdr) < 0) + return -1; + + if (offset == 0) { + /* no fields - the file is empty */ + return 0; + } + + /* check the fixed size of the header. name[] has to be checked + separately */ + if (field_hdr->fields_count > INT_MAX / MAIL_CACHE_FIELD_NAMES(1) || + field_hdr->size < MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count)) { + mail_cache_set_corrupted(cache, "invalid field header size"); + return -1; + } + + new_fields_count = field_hdr->fields_count; + if (new_fields_count != 0) { + cache->file_field_map = + i_realloc_type(cache->file_field_map, unsigned int, + cache->file_fields_count, new_fields_count); + } else { + i_free_and_null(cache->file_field_map); + } + cache->file_fields_count = new_fields_count; + + last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED()); + sizes = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count)); + types = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count)); + decisions = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count)); + names = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count)); + end = CONST_PTR_OFFSET(field_hdr, field_hdr->size); + i_assert(names <= end); + + /* clear the old mapping */ + for (i = 0; i < cache->fields_count; i++) + cache->field_file_map[i] = (uint32_t)-1; + + mail_cache_purge_drop_init(cache, &cache->index->map->hdr, &drop_ctx); + i_zero(&field); + for (i = 0; i < field_hdr->fields_count; i++) { + for (p = names; p != end && *p != '\0'; p++) ; + if (p == end || *names == '\0') { + mail_cache_set_corrupted(cache, + "field header names corrupted"); + return -1; + } + + if (types[i] > MAIL_CACHE_FIELD_COUNT) { + mail_cache_set_corrupted(cache, "field type corrupted"); + return -1; + } + if (!field_decision_is_valid(decisions[i])) { + mail_cache_set_corrupted(cache, + "field decision type corrupted"); + return -1; + } + + /* ignore any forced-flags in the file */ + enum mail_cache_decision_type file_dec = + decisions[i] & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + + if (hash_table_lookup_full(cache->field_name_hash, names, + &orig_key, &orig_value)) { + /* already exists, see if decision can be updated */ + fidx = POINTER_CAST_TO(orig_value, unsigned int); + enum mail_cache_decision_type cur_dec = + cache->fields[fidx].field.decision; + if ((cur_dec & MAIL_CACHE_DECISION_FORCED) != 0) { + /* Forced decision. If the decision has + changed, update the fields in the file. */ + if ((cur_dec & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) != file_dec) + cache->field_header_write_pending = TRUE; + } else if (cache->fields[fidx].decision_dirty) { + /* Decisions have recently been updated + internally. Don't change them. */ + } else { + /* Use the decision from the cache file. */ + cache->fields[fidx].field.decision = file_dec; + } + if (field_type_verify(cache, fidx, + types[i], sizes[i]) < 0) + return -1; + } else { + /* field is currently unknown, so just use whatever + exists in the file. */ + field.name = names; + field.type = types[i]; + field.field_size = sizes[i]; + field.decision = file_dec; + mail_cache_register_fields(cache, &field, 1); + fidx = field.idx; + } + if (cache->field_file_map[fidx] != (uint32_t)-1) { + mail_cache_set_corrupted(cache, + "Duplicated field in header: %s", names); + return -1; + } + cache->fields[fidx].used = TRUE; + + cache->field_file_map[fidx] = i; + cache->file_field_map[i] = fidx; + + /* Update last_used if it's newer than ours. Note that the + last_used may have been overwritten while we were reading + this cache header. In theory this can mean that the + last_used field is only half-updated and contains garbage. + This practically won't matter, since the worst that can + happen is that we trigger a purge earlier than necessary. + The purging re-reads the last_used while cache is locked and + correctly figures out whether to drop the field. */ + if ((time_t)last_used[i] > cache->fields[fidx].field.last_used) + cache->fields[fidx].field.last_used = last_used[i]; + + switch (mail_cache_purge_drop_test(&drop_ctx, fidx)) { + case MAIL_CACHE_PURGE_DROP_DECISION_NONE: + break; + case MAIL_CACHE_PURGE_DROP_DECISION_DROP: + mail_cache_purge_later(cache, t_strdup_printf( + "Drop old field %s (last_used=%"PRIdTIME_T")", + cache->fields[fidx].field.name, + cache->fields[fidx].field.last_used)); + break; + case MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP: + /* This cache decision change can cause the field to be + dropped for old mails, so do it via purging. */ + mail_cache_purge_later(cache, t_strdup_printf( + "Change cache decision to temp for old field %s " + "(last_used=%"PRIdTIME_T")", + cache->fields[fidx].field.name, + cache->fields[fidx].field.last_used)); + break; + } + + names = p + 1; + } + return 0; +} + +static void copy_to_buf(struct mail_cache *cache, buffer_t *dest, bool add_new, + size_t offset, size_t size) +{ + const void *data; + unsigned int i, field; + + /* copy the existing fields */ + for (i = 0; i < cache->file_fields_count; i++) { + field = cache->file_field_map[i]; + data = CONST_PTR_OFFSET(&cache->fields[field], offset); + buffer_append(dest, data, size); + } + if (!add_new) + return; + + /* copy newly wanted fields */ + for (i = 0; i < cache->fields_count; i++) { + if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) { + data = CONST_PTR_OFFSET(&cache->fields[i], offset); + buffer_append(dest, data, size); + } + } +} + +static void copy_to_buf_byte(struct mail_cache *cache, buffer_t *dest, + bool add_new, size_t offset) +{ + const int *data; + unsigned int i, field; + uint8_t byte; + + /* copy the existing fields */ + for (i = 0; i < cache->file_fields_count; i++) { + field = cache->file_field_map[i]; + data = CONST_PTR_OFFSET(&cache->fields[field], offset); + byte = (uint8_t)*data; + buffer_append(dest, &byte, 1); + } + if (!add_new) + return; + + /* copy newly wanted fields */ + for (i = 0; i < cache->fields_count; i++) { + if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) { + data = CONST_PTR_OFFSET(&cache->fields[i], offset); + byte = (uint8_t)*data; + buffer_append(dest, &byte, 1); + } + } +} + +static void +copy_to_buf_last_used(struct mail_cache *cache, buffer_t *dest, bool add_new) +{ + size_t offset = offsetof(struct mail_cache_field, last_used); +#if defined(WORDS_BIGENDIAN) && SIZEOF_VOID_P == 8 + /* 64bit time_t with big endian CPUs: copy the last 32 bits instead of + the first 32 bits (that are always 0). The 32 bits are enough until + year 2106, so we're not in a hurry to use 64 bits on disk. */ + offset += sizeof(uint32_t); +#endif + copy_to_buf(cache, dest, add_new, offset, sizeof(uint32_t)); +} + +static int mail_cache_header_fields_update_locked(struct mail_cache *cache) +{ + buffer_t *buffer; + uint32_t i, offset, dec_offset; + int ret = 0; + + if (mail_cache_header_fields_read(cache) < 0 || + mail_cache_header_fields_get_offset(cache, &offset, NULL) < 0) + return -1; + + buffer = t_buffer_create(256); + + copy_to_buf_last_used(cache, buffer, FALSE); + ret = mail_cache_write(cache, buffer->data, buffer->used, + offset + MAIL_CACHE_FIELD_LAST_USED()); + if (ret == 0) { + buffer_set_used_size(buffer, 0); + copy_to_buf_byte(cache, buffer, FALSE, + offsetof(struct mail_cache_field, decision)); + + dec_offset = offset + + MAIL_CACHE_FIELD_DECISION(cache->file_fields_count); + ret = mail_cache_write(cache, buffer->data, buffer->used, + dec_offset); + if (ret == 0) { + for (i = 0; i < cache->file_fields_count; i++) + cache->fields[i].decision_dirty = FALSE; + } + } + + if (ret == 0) + cache->field_header_write_pending = FALSE; + return ret; +} + +int mail_cache_header_fields_update(struct mail_cache *cache) +{ + int ret; + + if (cache->locked) { + T_BEGIN { + ret = mail_cache_header_fields_update_locked(cache); + } T_END; + return ret; + } + + if (mail_cache_lock(cache) <= 0) + return -1; + + T_BEGIN { + ret = mail_cache_header_fields_update_locked(cache); + } T_END; + i_assert(!cache->hdr_modified); + mail_cache_unlock(cache); + return ret; +} + +void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest) +{ + struct mail_cache_header_fields hdr; + unsigned int field; + const char *name; + uint32_t i; + + i_zero(&hdr); + hdr.fields_count = cache->file_fields_count; + for (i = 0; i < cache->fields_count; i++) { + if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) + hdr.fields_count++; + } + buffer_append(dest, &hdr, sizeof(hdr)); + + /* we have to keep the field order for the existing fields. */ + copy_to_buf_last_used(cache, dest, TRUE); + copy_to_buf(cache, dest, TRUE, + offsetof(struct mail_cache_field, field_size), + sizeof(uint32_t)); + copy_to_buf_byte(cache, dest, TRUE, + offsetof(struct mail_cache_field, type)); + copy_to_buf_byte(cache, dest, TRUE, + offsetof(struct mail_cache_field, decision)); + + i_assert(dest->used == sizeof(hdr) + + (sizeof(uint32_t)*2 + 2) * hdr.fields_count); + + /* add existing fields' names */ + for (i = 0; i < cache->file_fields_count; i++) { + field = cache->file_field_map[i]; + name = cache->fields[field].field.name; + buffer_append(dest, name, strlen(name)+1); + } + /* add newly wanted fields' names */ + for (i = 0; i < cache->fields_count; i++) { + if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) { + name = cache->fields[i].field.name; + buffer_append(dest, name, strlen(name)+1); + } + } + + hdr.size = dest->used; + buffer_write(dest, 0, &hdr, sizeof(hdr)); + + if ((hdr.size & 3) != 0) + buffer_append_zero(dest, 4 - (hdr.size & 3)); +} + +int mail_cache_header_fields_get_next_offset(struct mail_cache *cache, + uint32_t *offset_r) +{ + if (mail_cache_header_fields_get_offset(cache, offset_r, NULL) < 0) + return -1; + + if (*offset_r == 0) { + *offset_r = offsetof(struct mail_cache_header, + field_header_offset); + } else { + *offset_r += offsetof(struct mail_cache_header_fields, + next_offset); + } + return 0; +} diff --git a/src/lib-index/mail-cache-lookup.c b/src/lib-index/mail-cache-lookup.c new file mode 100644 index 0000000..e57b40b --- /dev/null +++ b/src/lib-index/mail-cache-lookup.c @@ -0,0 +1,694 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "str.h" +#include "mail-cache-private.h" + + +#define CACHE_PREFETCH IO_BLOCK_SIZE + +int mail_cache_get_record(struct mail_cache *cache, uint32_t offset, + const struct mail_cache_record **rec_r) +{ + const struct mail_cache_record *rec; + const void *data; + int ret; + + i_assert(offset != 0); + + if (offset % sizeof(uint32_t) != 0) { + /* records are always 32-bit aligned */ + mail_cache_set_corrupted(cache, "invalid record offset"); + return -1; + } + + /* we don't know yet how large the record is, so just guess */ + if (mail_cache_map(cache, offset, sizeof(*rec) + CACHE_PREFETCH, + &data) < 0) + return -1; + + if (offset + sizeof(*rec) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return -1; + } + rec = data; + + if (rec->size < sizeof(*rec)) { + mail_cache_set_corrupted(cache, "invalid record size"); + return -1; + } + if (rec->size > CACHE_PREFETCH) { + /* larger than we guessed. map the rest of the record. */ + if ((ret = mail_cache_map(cache, offset, rec->size, &data)) < 0) + return -1; + if (ret == 0) { + mail_cache_set_corrupted(cache, "record points outside file"); + return -1; + } + rec = data; + } + + *rec_r = rec; + return 0; +} + +uint32_t mail_cache_lookup_cur_offset(struct mail_index_view *view, + uint32_t seq, uint32_t *reset_id_r) +{ + struct mail_cache *cache = mail_index_view_get_index(view)->cache; + struct mail_index_map *map; + const void *data; + uint32_t offset; + + mail_index_lookup_ext_full(view, seq, cache->ext_id, &map, &data, NULL); + if (data == NULL) { + /* no cache offsets */ + return 0; + } + offset = *((const uint32_t *)data); + if (offset == 0) + return 0; + + if (!mail_index_ext_get_reset_id(view, map, cache->ext_id, reset_id_r)) + i_unreached(); + return offset; +} + +static int +mail_cache_lookup_offset(struct mail_cache *cache, struct mail_index_view *view, + uint32_t seq, uint32_t *offset_r) +{ + uint32_t offset, reset_id, reset_id2; + int ret; + + offset = mail_cache_lookup_cur_offset(view, seq, &reset_id); + if (offset == 0) + return 0; + + while (cache->hdr->file_seq != reset_id) { + /* reset_it doesn't match - sync the index/cache */ + if ((ret = mail_cache_sync_reset_id(cache)) <= 0) + return ret; + + /* lookup again after syncing */ + offset = mail_cache_lookup_cur_offset(view, seq, &reset_id2); + if (offset == 0) + return 0; + if (cache->hdr->file_seq == reset_id2) + break; /* match - all good */ + if (reset_id == reset_id2) { + /* reset_id didn't change after sync. This means it's + pointing to an old already deleted cache file. */ + return 0; + } + /* reset_id changed - try again */ + reset_id = reset_id2; + } + + *offset_r = offset; + return 1; +} + +bool mail_cache_track_loops(struct mail_cache_loop_track *loop_track, + uoff_t offset, uoff_t size) +{ + i_assert(offset != 0); + i_assert(size != 0); + + /* looping happens only in rare error conditions, so it's enough if we + just catch it eventually. we do this by checking if we've seen + more record data than possible in the accessed file area. */ + if (loop_track->size_sum == 0) { + /* first call */ + loop_track->min_offset = offset; + loop_track->max_offset = offset + size; + } else { + if (loop_track->min_offset > offset) + loop_track->min_offset = offset; + if (loop_track->max_offset < offset + size) + loop_track->max_offset = offset + size; + } + + loop_track->size_sum += size; + return loop_track->size_sum > + (loop_track->max_offset - loop_track->min_offset); +} + +void mail_cache_lookup_iter_init(struct mail_cache_view *view, uint32_t seq, + struct mail_cache_lookup_iterate_ctx *ctx_r) +{ + struct mail_cache_lookup_iterate_ctx *ctx = ctx_r; + int ret; + + if (!view->cache->opened) + (void)mail_cache_open_and_verify(view->cache); + + i_zero(ctx); + ctx->view = view; + ctx->seq = seq; + + if (!MAIL_CACHE_IS_UNUSABLE(view->cache)) { + /* look up the first offset */ + ret = mail_cache_lookup_offset(view->cache, view->view, seq, + &ctx->offset); + if (ret <= 0) { + ctx->stop = TRUE; + ctx->failed = ret < 0; + } + } + ctx->remap_counter = view->cache->remap_counter; + + i_zero(&view->loop_track); +} + +static bool +mail_cache_lookup_iter_transaction(struct mail_cache_lookup_iterate_ctx *ctx) +{ + ctx->rec = mail_cache_transaction_lookup_rec(ctx->view->transaction, + ctx->seq, + &ctx->trans_next_idx); + if (ctx->rec == NULL) + return FALSE; + + ctx->inmemory_field_idx = TRUE; + ctx->remap_counter = ctx->view->cache->remap_counter; + ctx->pos = sizeof(*ctx->rec); + ctx->rec_size = ctx->rec->size; + return TRUE; +} + +static int +mail_cache_lookup_iter_next_record(struct mail_cache_lookup_iterate_ctx *ctx) +{ + struct mail_cache_view *view = ctx->view; + + if (ctx->failed) + return -1; + + if (ctx->rec != NULL) + ctx->offset = ctx->rec->prev_offset; + if (ctx->offset == 0) { + /* end of this record list. check newly appended data. */ + if (view->trans_seq1 > ctx->seq || + view->trans_seq2 < ctx->seq) + return 0; + /* check data still in memory. this works for recent mails + even with INDEX=MEMORY */ + if (!ctx->memory_appends_checked) { + if (mail_cache_lookup_iter_transaction(ctx)) + return 1; + ctx->memory_appends_checked = TRUE; + } + if (MAIL_CACHE_IS_UNUSABLE(view->cache) || ctx->stop) + return 0; + + /* check data already written to cache file */ + if (ctx->disk_appends_checked || + mail_cache_lookup_offset(view->cache, view->trans_view, + ctx->seq, &ctx->offset) <= 0) + return 0; + + ctx->disk_appends_checked = TRUE; + ctx->remap_counter = view->cache->remap_counter; + i_zero(&view->loop_track); + } + + if (ctx->stop) + return 0; + + /* look up the next record */ + if (mail_cache_get_record(view->cache, ctx->offset, &ctx->rec) < 0) + return -1; + if (mail_cache_track_loops(&view->loop_track, ctx->offset, + ctx->rec->size)) { + mail_cache_set_corrupted(view->cache, + "record list is circular"); + return -1; + } + ctx->inmemory_field_idx = FALSE; + ctx->remap_counter = view->cache->remap_counter; + + ctx->pos = sizeof(*ctx->rec); + ctx->rec_size = ctx->rec->size; + return 1; +} + +static int +mail_cache_lookup_rec_get_field(struct mail_cache_lookup_iterate_ctx *ctx, + unsigned int *field_idx_r) +{ + struct mail_cache *cache = ctx->view->cache; + uint32_t file_field; + + file_field = *((const uint32_t *)CONST_PTR_OFFSET(ctx->rec, ctx->pos)); + if (ctx->inmemory_field_idx) { + *field_idx_r = file_field; + return 0; + } + + if (file_field >= cache->file_fields_count) { + /* new field, have to re-read fields header to figure + out its size. don't do this if we're purging. */ + if (!cache->locked) { + if (mail_cache_header_fields_read(cache) < 0) + return -1; + } + if (file_field >= cache->file_fields_count) { + mail_cache_set_corrupted(cache, + "field index too large (%u >= %u)", + file_field, cache->file_fields_count); + return -1; + } + + /* field reading might have re-mmaped the file and + caused rec pointer to break. need to get it again. */ + if (mail_cache_get_record(cache, ctx->offset, &ctx->rec) < 0) + return -1; + ctx->remap_counter = cache->remap_counter; + } + + *field_idx_r = cache->file_field_map[file_field]; + return 0; +} + +int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx, + struct mail_cache_iterate_field *field_r) +{ + struct mail_cache *cache = ctx->view->cache; + unsigned int field_idx; + unsigned int data_size; + int ret; + + i_assert(ctx->remap_counter == cache->remap_counter); + + if (ctx->pos + sizeof(uint32_t) > ctx->rec_size) { + if (ctx->pos != ctx->rec_size) { + mail_cache_set_corrupted(cache, + "record has invalid size"); + return -1; + } + + if ((ret = mail_cache_lookup_iter_next_record(ctx)) <= 0) + return ret; + } + + /* return the next field */ + if (mail_cache_lookup_rec_get_field(ctx, &field_idx) < 0) + return -1; + ctx->pos += sizeof(uint32_t); + + data_size = cache->fields[field_idx].field.field_size; + if (data_size == UINT_MAX && + ctx->pos + sizeof(uint32_t) <= ctx->rec->size) { + /* variable size field. get its size from the file. */ + data_size = *((const uint32_t *) + CONST_PTR_OFFSET(ctx->rec, ctx->pos)); + ctx->pos += sizeof(uint32_t); + } + + if (ctx->rec->size - ctx->pos < data_size) { + mail_cache_set_corrupted(cache, + "record continues outside its allocated size"); + return -1; + } + + field_r->field_idx = field_idx; + field_r->data = CONST_PTR_OFFSET(ctx->rec, ctx->pos); + field_r->size = data_size; + field_r->offset = ctx->offset + ctx->pos; + + /* each record begins from 32bit aligned position */ + ctx->pos += (data_size + sizeof(uint32_t)-1) & ~(sizeof(uint32_t)-1); + return 1; +} + +static int mail_cache_seq(struct mail_cache_view *view, uint32_t seq) +{ + struct mail_cache_lookup_iterate_ctx iter; + struct mail_cache_iterate_field field; + int ret; + + view->cached_exists_value = (view->cached_exists_value + 1) & UINT8_MAX; + if (view->cached_exists_value == 0) { + /* wrapped, we'll have to clear the buffer */ + buffer_set_used_size(view->cached_exists_buf, 0); + view->cached_exists_value++; + } + view->cached_exists_seq = seq; + + mail_cache_lookup_iter_init(view, seq, &iter); + while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) { + buffer_write(view->cached_exists_buf, field.field_idx, + &view->cached_exists_value, 1); + } + return ret; +} + +int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq, + unsigned int field) +{ + const uint8_t *data; + + i_assert(seq > 0); + + /* NOTE: view might point to a non-committed transaction that has + fields that don't yet exist in the cache file. So don't add any + fast-paths checking whether the field exists in the file. */ + + /* FIXME: we should discard the cache if view has been synced */ + if (view->cached_exists_seq != seq) { + if (mail_cache_seq(view, seq) < 0) + return -1; + } + + data = view->cached_exists_buf->data; + return (field < view->cached_exists_buf->used && + data[field] == view->cached_exists_value) ? 1 : 0; +} + +bool mail_cache_field_exists_any(struct mail_cache_view *view, uint32_t seq) +{ + uint32_t reset_id; + + return mail_cache_lookup_cur_offset(view->view, seq, &reset_id) != 0; +} + +enum mail_cache_decision_type +mail_cache_field_get_decision(struct mail_cache *cache, unsigned int field_idx) +{ + i_assert(field_idx < cache->fields_count); + + return cache->fields[field_idx].field.decision; +} + +static int +mail_cache_lookup_bitmask(struct mail_cache_lookup_iterate_ctx *iter, + unsigned int field_idx, unsigned int field_size, + buffer_t *dest_buf) +{ + struct mail_cache_iterate_field field; + const unsigned char *src; + unsigned char *dest; + unsigned int i; + bool found = FALSE; + int ret; + + /* make sure all bits are cleared first */ + buffer_write_zero(dest_buf, 0, field_size); + + while ((ret = mail_cache_lookup_iter_next(iter, &field)) > 0) { + if (field.field_idx != field_idx) + continue; + + /* merge all bits */ + src = field.data; + dest = buffer_get_space_unsafe(dest_buf, 0, field.size); + for (i = 0; i < field.size; i++) + dest[i] |= src[i]; + found = TRUE; + } + return ret < 0 ? -1 : (found ? 1 : 0); +} + +int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf, + uint32_t seq, unsigned int field_idx) +{ + struct mail_cache_lookup_iterate_ctx iter; + struct mail_cache_iterate_field field; + int ret; + + ret = mail_cache_field_exists(view, seq, field_idx); + mail_cache_decision_state_update(view, seq, field_idx); + if (ret <= 0) + return ret; + + /* the field should exist */ + mail_cache_lookup_iter_init(view, seq, &iter); + if (view->cache->fields[field_idx].field.type == MAIL_CACHE_FIELD_BITMASK) { + ret = mail_cache_lookup_bitmask(&iter, field_idx, + view->cache->fields[field_idx].field.field_size, + dest_buf); + } else { + /* return the first one that's found. if there are multiple + they're all identical. */ + while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) { + if (field.field_idx == field_idx) { + buffer_append(dest_buf, field.data, field.size); + break; + } + } + } + /* NOTE: view->cache->fields may have been reallocated by + mail_cache_lookup_*(). */ + return ret; +} + +struct header_lookup_data { + uint32_t data_size; + const unsigned char *data; +}; + +struct header_lookup_line { + uint32_t line_num; + struct header_lookup_data *data; +}; + +struct header_lookup_context { + struct mail_cache_view *view; + pool_t pool; + ARRAY(struct header_lookup_line) lines; +}; + +enum { + HDR_FIELD_STATE_DONTWANT = 0, + HDR_FIELD_STATE_WANT, + HDR_FIELD_STATE_SEEN +}; + +static void header_lines_save(struct header_lookup_context *ctx, + const struct mail_cache_iterate_field *field) +{ + const uint32_t *lines = field->data; + uint32_t data_size = field->size; + struct header_lookup_line hdr_line; + struct header_lookup_data *hdr_data; + void *data_dup; + unsigned int i, lines_count, pos; + + /* data = { line_nums[], 0, "headers" } */ + for (i = 0; data_size >= sizeof(uint32_t); i++) { + data_size -= sizeof(uint32_t); + if (lines[i] == 0) + break; + } + lines_count = i; + pos = (lines_count+1) * sizeof(uint32_t); + + hdr_data = p_new(ctx->pool, struct header_lookup_data, 1); + hdr_data->data_size = data_size; + if (data_size > 0) { + hdr_data->data = data_dup = + p_malloc(ctx->pool, data_size); + memcpy(data_dup, CONST_PTR_OFFSET(field->data, pos), data_size); + } + + for (i = 0; i < lines_count; i++) { + hdr_line.line_num = lines[i]; + hdr_line.data = hdr_data; + array_push_back(&ctx->lines, &hdr_line); + } +} + +static int header_lookup_line_cmp(const struct header_lookup_line *l1, + const struct header_lookup_line *l2) +{ + return (int)l1->line_num - (int)l2->line_num; +} + +static int +mail_cache_lookup_headers_real(struct mail_cache_view *view, string_t *dest, + uint32_t seq, const unsigned int field_idxs[], + unsigned int fields_count, pool_t *pool_r) +{ + struct mail_cache_lookup_iterate_ctx iter; + struct mail_cache_iterate_field field; + struct header_lookup_context ctx; + struct header_lookup_line *lines; + const unsigned char *p, *start, *end; + uint8_t *field_state; + unsigned int i, count, max_field = 0; + size_t hdr_size; + uint8_t want = HDR_FIELD_STATE_WANT; + buffer_t *buf; + int ret; + + *pool_r = NULL; + + if (fields_count == 0) + return 1; + + /* update the decision state regardless of whether the fields + actually exist or not. */ + for (i = 0; i < fields_count; i++) + mail_cache_decision_state_update(view, seq, field_idxs[i]); + + /* mark all the fields we want to find. */ + buf = t_buffer_create(32); + for (i = 0; i < fields_count; i++) { + if (field_idxs[i] > max_field) + max_field = field_idxs[i]; + + buffer_write(buf, field_idxs[i], &want, 1); + } + field_state = buffer_get_modifiable_data(buf, NULL); + + /* lookup the fields */ + i_zero(&ctx); + ctx.view = view; + ctx.pool = *pool_r = pool_alloconly_create(MEMPOOL_GROWING"mail cache headers", 1024); + t_array_init(&ctx.lines, 32); + + mail_cache_lookup_iter_init(view, seq, &iter); + while ((ret = mail_cache_lookup_iter_next(&iter, &field)) > 0) { + if (field.field_idx > max_field || + field_state[field.field_idx] != HDR_FIELD_STATE_WANT) { + /* a) don't want it, b) duplicate */ + } else { + field_state[field.field_idx] = HDR_FIELD_STATE_SEEN; + header_lines_save(&ctx, &field); + } + + } + if (ret < 0) + return -1; + + /* check that all fields were found */ + for (i = 0; i <= max_field; i++) { + if (field_state[i] == HDR_FIELD_STATE_WANT) + return 0; + } + + /* we need to return headers in the order they existed originally. + we can do this by sorting the messages by their line numbers. */ + array_sort(&ctx.lines, header_lookup_line_cmp); + lines = array_get_modifiable(&ctx.lines, &count); + + /* then start filling dest buffer from the headers */ + for (i = 0; i < count; i++) { + start = lines[i].data->data; + end = start + lines[i].data->data_size; + + /* find the end of the (multiline) header */ + for (p = start; p != end; p++) { + if (*p == '\n' && + (p+1 == end || (p[1] != ' ' && p[1] != '\t'))) { + p++; + break; + } + } + hdr_size = (size_t)(p - start); + buffer_append(dest, start, hdr_size); + + /* if there are more lines for this header, the following lines + continue after this one. so skip this line. */ + lines[i].data->data += hdr_size; + lines[i].data->data_size -= hdr_size; + } + return 1; +} + +int mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest, + uint32_t seq, const unsigned int field_idxs[], + unsigned int fields_count) +{ + pool_t pool = NULL; + int ret; + + if (buffer_get_pool(dest)->datastack_pool) + ret = mail_cache_lookup_headers_real(view, dest, seq, + field_idxs, fields_count, + &pool); + else T_BEGIN { + ret = mail_cache_lookup_headers_real(view, dest, seq, + field_idxs, fields_count, + &pool); + } T_END; + pool_unref(&pool); + return ret; +} + +static uint32_t +mail_cache_get_highest_seq_with_cache(struct mail_cache_view *view, + uint32_t below_seq, uint32_t *reset_id_r) +{ + struct mail_cache_missing_reason_cache *rc = &view->reason_cache; + uint32_t seq = below_seq-1, highest_checked_seq = 0; + + /* find the newest mail that has anything in cache */ + if (rc->log_file_head_offset == view->view->log_file_head_offset && + rc->log_file_head_seq == view->view->log_file_head_seq) { + /* reason_cache matches the current view - we can use it */ + highest_checked_seq = rc->highest_checked_seq; + } else { + rc->log_file_head_offset = view->view->log_file_head_offset; + rc->log_file_head_seq = view->view->log_file_head_seq; + } + rc->highest_checked_seq = below_seq; + + /* first check anything not already in reason_cache */ + for (; seq > highest_checked_seq; seq--) { + if (mail_cache_lookup_cur_offset(view->view, seq, reset_id_r) != 0) { + rc->highest_seq_with_cache = seq; + rc->reset_id = *reset_id_r; + return seq; + } + } + if (seq == 0) + return 0; + /* then return the result from cache */ + *reset_id_r = rc->reset_id; + return rc->highest_seq_with_cache; +} + +const char * +mail_cache_get_missing_reason(struct mail_cache_view *view, uint32_t seq) +{ + uint32_t offset, reset_id; + + if (mail_index_is_expunged(view->view, seq)) + return "Mail is already expunged"; + + if (MAIL_CACHE_IS_UNUSABLE(view->cache)) + return "Cache file is unusable"; + + offset = mail_cache_lookup_cur_offset(view->view, seq, &reset_id); + if (offset != 0) { + if (view->cache->hdr->file_seq != reset_id) { + return t_strdup_printf( + "Index reset_id=%u doesn't match cache reset_id=%u", + reset_id, view->cache->hdr->file_seq); + } + return t_strdup_printf( + "Mail has other cached fields, reset_id=%u", reset_id); + } + seq = mail_cache_get_highest_seq_with_cache(view, seq, &reset_id); + if (seq == 0) { + return t_strdup_printf("Cache file is empty, reset_id=%u", + view->cache->hdr->file_seq); + } + + uint32_t uid; + mail_index_lookup_uid(view->view, seq, &uid); + + if (view->cache->hdr->file_seq != reset_id) { + return t_strdup_printf( + "Mail not cached, highest cached seq=%u uid=%u: " + "Index reset_id=%u doesn't match cache reset_id=%u", + seq, uid, reset_id, view->cache->hdr->file_seq); + } + return t_strdup_printf( + "Mail not cached, highest cached seq=%u uid=%u: reset_id=%u", + seq, uid, reset_id); +} diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h new file mode 100644 index 0000000..c2fee17 --- /dev/null +++ b/src/lib-index/mail-cache-private.h @@ -0,0 +1,421 @@ +#ifndef MAIL_CACHE_PRIVATE_H +#define MAIL_CACHE_PRIVATE_H + +#include "file-dotlock.h" +#include "mail-index-private.h" +#include "mail-cache.h" + +#define MAIL_CACHE_MAJOR_VERSION 1 +#define MAIL_CACHE_MINOR_VERSION 1 + +#define MAIL_CACHE_LOCK_TIMEOUT 10 +#define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300 + +#define MAIL_CACHE_MAX_WRITE_BUFFER (1024*256) + +#define MAIL_CACHE_IS_UNUSABLE(cache) \ + ((cache)->hdr == NULL) + +struct mail_cache_header { + /* Major version is increased only when you can't have backwards + compatibility. If the field doesn't match MAIL_CACHE_MAJOR_VERSION, + don't even try to read it. */ + uint8_t major_version; + /* If this isn't the same as sizeof(uoff_t), the cache file can't be + safely used with the current implementation. */ + uint8_t compat_sizeof_uoff_t; + /* Minor version is increased when the file format changes in a + backwards compatible way. */ + uint8_t minor_version; + uint8_t unused; + + /* Unique index file ID, which must match the main index's indexid. + See mail_index_header.indexid. */ + uint32_t indexid; + /* Cache file sequence. Increased on every purge. This must match the + main index's reset_id for "cache" extension or the cache offsets + aren't valid. When creating the first cache file, use the current + UNIX timestamp as the file_seq. */ + uint32_t file_seq; + + /* Number of cache records that are linked inside the cache file, + instead of being directly pointed from the main index. */ + uint32_t continued_record_count; + + /* Number of messages cached in this file. This does not include + the continuation records. + + NOTE: <=v2.1 used this for hole offset, so we can't fully + rely on it */ + uint32_t record_count; + /* Currently unused. */ + uint32_t backwards_compat_used_file_size; + /* Number of already expunged messages that currently have cache + content in this file. */ + uint32_t deleted_record_count; + + /* Offset to the first mail_cache_header_fields. */ + uint32_t field_header_offset; +}; + +struct mail_cache_header_fields { + /* Offset to the updated version of this header. Use + mail_index_offset_to_uint32() to decode it. */ + uint32_t next_offset; + /* Full size of this header. */ + uint32_t size; + /* Number of fields in this header. */ + uint32_t fields_count; + +#if 0 + /* Last time the field was accessed. Not updated more often than + once a day. This field may be overwritten later on, which in theory + could cause reading to see a partially updated (corrupted) value. + Don't fully trust this field unless it was read while cache is + locked. */ + uint32_t last_used[fields_count]; + /* (uint32_t)-1 for variable sized fields */ + uint32_t size[fields_count]; + /* enum mail_cache_field_type */ + uint8_t type[fields_count]; + /* enum mail_cache_decision_type. This field can be overwritten + later on to update the caching decision. */ + uint8_t decision[fields_count]; + /* NUL-separated list of field names */ + char name[fields_count][]; +#endif +}; + +/* Macros to return offsets to the fields in mail_cache_header_fields. */ +#define MAIL_CACHE_FIELD_LAST_USED() \ + (sizeof(uint32_t) * 3) +#define MAIL_CACHE_FIELD_SIZE(count) \ + (MAIL_CACHE_FIELD_LAST_USED() + sizeof(uint32_t) * (count)) +#define MAIL_CACHE_FIELD_TYPE(count) \ + (MAIL_CACHE_FIELD_SIZE(count) + sizeof(uint32_t) * (count)) +#define MAIL_CACHE_FIELD_DECISION(count) \ + (MAIL_CACHE_FIELD_TYPE(count) + sizeof(uint8_t) * (count)) +#define MAIL_CACHE_FIELD_NAMES(count) \ + (MAIL_CACHE_FIELD_DECISION(count) + sizeof(uint8_t) * (count)) + +struct mail_cache_record { + uint32_t prev_offset; + uint32_t size; /* full record size, including this header */ + /* array of { uint32_t field; [ uint32_t size; ] { .. } } */ +}; + +struct mail_cache_field_private { + struct mail_cache_field field; + + /* Highest message UID whose cache field of this type have been + accessed within this session. This is used to track whether messages + are accessed in non-ascending order, which indicates an IMAP client + that doesn't have a local cache. That will result in the caching + decision to change from TEMP to YES. */ + uint32_t uid_highwater; + + /* Unused fields aren't written to cache file */ + bool used:1; + /* field.decision is pending a write to cache file header. If the + cache header is read from disk, don't overwrite it. */ + bool decision_dirty:1; +}; + +struct mail_cache { + struct mail_index *index; + struct event *event; + /* Registered "cache" extension ID */ + uint32_t ext_id; + + char *filepath; + int fd; + + struct dotlock_settings dotlock_settings; + struct file_lock *file_lock; + + /* Cache file's inode, device and size when it was last fstat()ed. */ + ino_t st_ino; + dev_t st_dev; + uoff_t last_stat_size; + + /* Used to avoid logging mmap() errors too rapidly. */ + time_t last_mmap_error_time; + + /* a) mmaping the whole file */ + void *mmap_base; + /* b) using file cache */ + struct file_cache *file_cache; + /* c) using small read() calls with MAIL_INDEX_OPEN_FLAG_SAVEONLY */ + uoff_t read_offset; + buffer_t *read_buf; + /* Size of the cache file as currently mapped to memory. Used for all + of a), b), and c). */ + size_t mmap_length; + /* mail_cache_map() increases this always. Used only for asserts. */ + unsigned int remap_counter; + /* Linked list of all cache views. */ + struct mail_cache_view *views; + + /* mmap_disable=no: hdr points to data / NULL when cache is invalid. + mmap_disable=yes: hdr points to hdr_ro_copy. this is needed because + cache invalidation can zero the data any time */ + const struct mail_cache_header *hdr; + struct mail_cache_header hdr_ro_copy; + /* hdr_copy gets updated when cache is locked and written when + unlocking and hdr_modified=TRUE */ + struct mail_cache_header hdr_copy; + /* If non-0, the offset for the last seen mail_cache_header_fields. + Used as a cache to avoid reading through multiple next_offset + pointers. */ + uint32_t last_field_header_offset; + + /* Memory pool used for permanent field allocations. Currently this + means mail_cache_field.name and field_name_hash. */ + pool_t field_pool; + /* Size of fields[] and field_file_map[] */ + unsigned int fields_count; + /* All the registered cache fields. */ + struct mail_cache_field_private *fields; + /* mail_cache_field.idx -> file-specific header index. The reverse + of this is file_field_map[]. */ + uint32_t *field_file_map; + /* mail_cache_field.name -> mail_cache_field.idx */ + HASH_TABLE(char *, void *) field_name_hash; /* name -> idx */ + + /* file-specific header index -> mail_cache_fields.idx. The reverse + of this is field_file_map[]. */ + unsigned int *file_field_map; + /* Size of file_field_map[] */ + unsigned int file_fields_count; + + /* mail_cache_purge_later() sets these values to trigger purging on + the next index sync. need_purge_file_seq is set to the current + cache file_seq. If at sync time the file_seq differs, it means + the cache was already purged and another purge isn't necessary. */ + uint32_t need_purge_file_seq; + /* Human-readable reason for purging. Used for debugging and events. */ + char *need_purge_reason; + + /* Cache has been opened (or it doesn't exist). */ + bool opened:1; + /* Cache has been locked with mail_cache_lock(). */ + bool locked:1; + /* TRUE if the last lock attempt failed. The next locking attempt will + be non-blocking to avoid unnecessarily waiting on a cache that has + been locked for a long time. Since cache isn't strictly required, + this could avoid unnecessarily long waits with some edge cases. */ + bool last_lock_failed:1; + /* cache->hdr_copy has been modified. This must be used only while + cache is locked. */ + bool hdr_modified:1; + /* At least one of the cache fields' last_used or cache decision has + changed. mail_cache_header_fields_update() will be used to overwrite + these to the latest mail_cache_header_fields. */ + bool field_header_write_pending:1; + /* Cache is currently being purged. */ + bool purging:1; + /* Access the cache file by reading as little as possible from it + (as opposed to mmap()ing it or using file-cache.h API to cache + larger parts of it). This is used with MAIL_INDEX_OPEN_FLAG_SAVEONLY + to avoid unnecessary cache reads. */ + bool map_with_read:1; +}; + +struct mail_cache_loop_track { + /* we're looping if size_sum > (max_offset-min_offset) */ + uoff_t min_offset, max_offset; + uoff_t size_sum; +}; + +struct mail_cache_missing_reason_cache { + uint32_t highest_checked_seq; + uint32_t highest_seq_with_cache; + + uint32_t reset_id; + uint32_t log_file_head_seq; + uoff_t log_file_head_offset; +}; + +struct mail_cache_view { + struct mail_cache *cache; + struct mail_cache_view *prev, *next; + struct mail_index_view *view, *trans_view; + + struct mail_cache_transaction_ctx *transaction; + /* mail_cache_add() has been called for some of the messages between + trans_seq1..trans_seq2 in an uncommitted transaction. Check also + the transaction contents when looking up cache fields for these + mails. */ + uint32_t trans_seq1, trans_seq2; + + /* Used to avoid infinite loops in case cache records point to each + others, causing a loop. FIXME: New cache files no longer support + overwriting existing data, so this could be removed and replaced + with a simple check that prev_offset is always smaller than the + current record's offset. */ + struct mail_cache_loop_track loop_track; + /* Used for optimizing mail_cache_get_missing_reason() */ + struct mail_cache_missing_reason_cache reason_cache; + + /* if cached_exists_buf[field] == cached_exists_value, it's cached. + this allows us to avoid constantly clearing the whole buffer. + it needs to be cleared only when cached_exists_value is wrapped. */ + buffer_t *cached_exists_buf; + uint8_t cached_exists_value; + uint32_t cached_exists_seq; + + /* mail_cache_view_update_cache_decisions() has been used to disable + updating cache decisions. */ + bool no_decision_updates:1; +}; + +/* mail_cache_lookup_iter_next() returns the next found field. */ +struct mail_cache_iterate_field { + /* mail_cache_field.idx */ + unsigned int field_idx; + /* Size of data */ + unsigned int size; + /* Cache field content in the field type-specific format */ + const void *data; + /* Offset to data in cache file */ + uoff_t offset; +}; + +struct mail_cache_lookup_iterate_ctx { + struct mail_cache_view *view; + /* This must match mail_cache.remap_counter or the iterator is + invalid. */ + unsigned int remap_counter; + /* Message sequence as given to mail_cache_lookup_iter_init() */ + uint32_t seq; + + /* Pointer to current cache record being iterated. This may point + to the cache file or uncommitted transaction. */ + const struct mail_cache_record *rec; + /* Iterator's current position in the cache record. Starts from + sizeof(mail_cache_record). */ + unsigned int pos; + /* Copy of rec->size */ + unsigned int rec_size; + /* Cache file offset to the beginning of rec, or 0 if it points to + an uncommitted transaction. */ + uint32_t offset; + + /* Used to loop through all changes in the uncommited transaction, + in case there are multiple changes to the same message. */ + unsigned int trans_next_idx; + + /* Cache has become unusable. Stop the iteration. */ + bool stop:1; + /* I/O error or lock timeout occurred during iteration. Normally there + is no locking during iteration, but it may happen while cache is + being purged to wait for the purging to finish before cache can be + accessed again. */ + bool failed:1; + /* Iteration has finished returning changes from uncommitted + transaction's in-memory buffer. */ + bool memory_appends_checked:1; + /* Iteration has finished returning changes from uncommitted + transaction that were already written to cache file, but not + to main index. */ + bool disk_appends_checked:1; + /* TRUE if the field index numbers in rec as the internal + mail_cache_field.idx (instead of the file-specific indexes). + This indicates that the rec points to uncommited transaction's + in-memory buffer. */ + bool inmemory_field_idx:1; +}; + +/* Explicitly lock the cache file. Returns -1 if error / timed out, + 1 if ok, 0 if cache is broken/doesn't exist */ +int mail_cache_lock(struct mail_cache *cache); +/* Flush pending header updates and unlock. Returns -1 if cache is / just got + corrupted, 0 if ok. */ +int mail_cache_flush_and_unlock(struct mail_cache *cache); +/* Unlock the cache without any header updates. */ +void mail_cache_unlock(struct mail_cache *cache); + +int mail_cache_write(struct mail_cache *cache, const void *data, size_t size, + uoff_t offset); +int mail_cache_append(struct mail_cache *cache, const void *data, size_t size, + uint32_t *offset); + +int mail_cache_header_fields_read(struct mail_cache *cache); +int mail_cache_header_fields_update(struct mail_cache *cache); +void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest); +int mail_cache_header_fields_get_next_offset(struct mail_cache *cache, + uint32_t *offset_r); +void mail_cache_expunge_count(struct mail_cache *cache, unsigned int count); + +uint32_t mail_cache_lookup_cur_offset(struct mail_index_view *view, + uint32_t seq, uint32_t *reset_id_r); +int mail_cache_get_record(struct mail_cache *cache, uint32_t offset, + const struct mail_cache_record **rec_r); +uint32_t mail_cache_get_first_new_seq(struct mail_index_view *view); + +/* Returns TRUE if offset..size area has been tracked before. + Returns FALSE if the area may or may not have been tracked before, + but we don't know for sure yet. */ +bool mail_cache_track_loops(struct mail_cache_loop_track *loop_track, + uoff_t offset, uoff_t size); + +/* Iterate through a message's cached fields. */ +void mail_cache_lookup_iter_init(struct mail_cache_view *view, uint32_t seq, + struct mail_cache_lookup_iterate_ctx *ctx_r); +/* Returns 1 if field was returned, 0 if end of fields, or -1 if error. + Note that this may trigger re-reading and reallocating cache fields. */ +int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx, + struct mail_cache_iterate_field *field_r); +const struct mail_cache_record * +mail_cache_transaction_lookup_rec(struct mail_cache_transaction_ctx *ctx, + unsigned int seq, + unsigned int *trans_next_idx); +bool mail_cache_transactions_have_changes(struct mail_cache *cache); + +/* Return data from the specified position in the cache file. Returns 1 if + successful, 0 if offset/size points outside the cache file, -1 if I/O + error. */ +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r); +/* Map the whole cache file into memory. Returns 1 if ok, 0 if corrupted + (and deleted), -1 if I/O error. */ +int mail_cache_map_all(struct mail_cache *cache); +void mail_cache_file_close(struct mail_cache *cache); +int mail_cache_reopen(struct mail_cache *cache); +int mail_cache_sync_reset_id(struct mail_cache *cache); + +/* Notify the decision handling code that field was looked up for seq. + This should be called even for fields that aren't currently in cache file. + This is used to update caching decisions for fields that already exist + in the cache file. */ +void mail_cache_decision_state_update(struct mail_cache_view *view, + uint32_t seq, unsigned int field); +const char *mail_cache_decision_to_string(enum mail_cache_decision_type dec); +struct event_passthrough * +mail_cache_decision_changed_event(struct mail_cache *cache, struct event *event, + unsigned int field); + +struct mail_cache_purge_drop_ctx { + struct mail_cache *cache; + time_t max_yes_downgrade_time; + time_t max_temp_drop_time; +}; +enum mail_cache_purge_drop_decision { + MAIL_CACHE_PURGE_DROP_DECISION_NONE, + MAIL_CACHE_PURGE_DROP_DECISION_DROP, + MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP, +}; +void mail_cache_purge_drop_init(struct mail_cache *cache, + const struct mail_index_header *hdr, + struct mail_cache_purge_drop_ctx *ctx_r); +enum mail_cache_purge_drop_decision +mail_cache_purge_drop_test(struct mail_cache_purge_drop_ctx *ctx, + unsigned int field); + +int mail_cache_expunge_handler(struct mail_index_sync_map_ctx *sync_ctx, + const void *data, void **sync_context); + +void mail_cache_set_syscall_error(struct mail_cache *cache, + const char *function) ATTR_COLD; + +#endif diff --git a/src/lib-index/mail-cache-purge.c b/src/lib-index/mail-cache-purge.c new file mode 100644 index 0000000..bc86bf1 --- /dev/null +++ b/src/lib-index/mail-cache-purge.c @@ -0,0 +1,707 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ostream.h" +#include "nfs-workarounds.h" +#include "read-full.h" +#include "file-dotlock.h" +#include "file-cache.h" +#include "file-set-size.h" +#include "mail-cache-private.h" + +#include <stdio.h> +#include <sys/stat.h> + +struct mail_cache_copy_context { + struct mail_cache *cache; + struct event *event; + struct mail_cache_purge_drop_ctx drop_ctx; + + buffer_t *buffer, *field_seen; + ARRAY(unsigned int) bitmask_pos; + uint32_t *field_file_map; + + uint8_t field_seen_value; + bool new_msg; +}; + +static void +mail_cache_merge_bitmask(struct mail_cache_copy_context *ctx, + const struct mail_cache_iterate_field *field) +{ + unsigned char *dest; + unsigned int i, *pos; + + pos = array_idx_get_space(&ctx->bitmask_pos, field->field_idx); + if (*pos == 0) { + /* we decided to drop this field */ + return; + } + + dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size); + for (i = 0; i < field->size; i++) + dest[i] |= ((const unsigned char*)field->data)[i]; +} + +static void +mail_cache_purge_field(struct mail_cache_copy_context *ctx, + const struct mail_cache_iterate_field *field) +{ + struct mail_cache_field *cache_field; + enum mail_cache_decision_type dec; + uint32_t file_field_idx, size32; + uint8_t *field_seen; + + file_field_idx = ctx->field_file_map[field->field_idx]; + if (file_field_idx == (uint32_t)-1) + return; + + cache_field = &ctx->cache->fields[field->field_idx].field; + + field_seen = buffer_get_space_unsafe(ctx->field_seen, + field->field_idx, 1); + if (*field_seen == ctx->field_seen_value) { + /* duplicate */ + if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) + mail_cache_merge_bitmask(ctx, field); + return; + } + *field_seen = ctx->field_seen_value; + + dec = cache_field->decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + if (ctx->new_msg) { + if (dec == MAIL_CACHE_DECISION_NO) + return; + } else { + if (dec != MAIL_CACHE_DECISION_YES) + return; + } + + buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx)); + + if (cache_field->field_size == UINT_MAX) { + size32 = (uint32_t)field->size; + buffer_append(ctx->buffer, &size32, sizeof(size32)); + } + + if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) { + /* remember the position in case we need to update it */ + unsigned int pos = ctx->buffer->used; + + array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos); + } + buffer_append(ctx->buffer, field->data, field->size); + if ((field->size & 3) != 0) + buffer_append_zero(ctx->buffer, 4 - (field->size & 3)); +} + +static uint32_t get_next_file_seq(struct mail_cache *cache) +{ + const struct mail_index_ext *ext; + struct mail_index_view *view; + uint32_t file_seq; + + /* make sure we look up the latest reset_id */ + if (mail_index_refresh(cache->index) < 0) + return -1; + + view = mail_index_view_open(cache->index); + ext = mail_index_view_get_ext(view, cache->ext_id); + file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time; + + if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq) + file_seq = cache->hdr->file_seq + 1; + mail_index_view_close(&view); + + return file_seq != 0 ? file_seq : 1; +} + +static void +mail_cache_purge_get_fields(struct mail_cache_copy_context *ctx, + unsigned int used_fields_count) +{ + struct mail_cache *cache = ctx->cache; + unsigned int i, j, idx; + + /* Make mail_cache_header_fields_get() return the fields in + the same order as we saved them. */ + memcpy(cache->field_file_map, ctx->field_file_map, + sizeof(uint32_t) * cache->fields_count); + + /* reverse mapping */ + cache->file_fields_count = used_fields_count; + i_free(cache->file_field_map); + cache->file_field_map = used_fields_count == 0 ? NULL : + i_new(unsigned int, used_fields_count); + for (i = j = 0; i < cache->fields_count; i++) { + idx = cache->field_file_map[i]; + if (idx != (uint32_t)-1) { + i_assert(idx < used_fields_count && + cache->file_field_map != NULL && + cache->file_field_map[idx] == 0); + cache->file_field_map[idx] = i; + j++; + } + } + i_assert(j == used_fields_count); + + buffer_set_used_size(ctx->buffer, 0); + mail_cache_header_fields_get(cache, ctx->buffer); +} + +static bool +mail_cache_purge_check_field(struct mail_cache_copy_context *ctx, + unsigned int field) +{ + struct mail_cache_field_private *priv = &ctx->cache->fields[field]; + enum mail_cache_decision_type dec = priv->field.decision; + + switch (mail_cache_purge_drop_test(&ctx->drop_ctx, field)) { + case MAIL_CACHE_PURGE_DROP_DECISION_NONE: + break; + case MAIL_CACHE_PURGE_DROP_DECISION_DROP: { + const char *dec_str = mail_cache_decision_to_string(dec); + struct event_passthrough *e = + event_create_passthrough(ctx->event)-> + set_name("mail_cache_purge_drop_field")-> + add_str("field", priv->field.name)-> + add_str("decision", dec_str)-> + add_int("last_used", priv->field.last_used); + e_debug(e->event(), "Purge dropped field %s " + "(decision=%s, last_used=%"PRIdTIME_T")", + priv->field.name, dec_str, priv->field.last_used); + dec = MAIL_CACHE_DECISION_NO; + break; + } + case MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP: { + struct event_passthrough *e = + mail_cache_decision_changed_event( + ctx->cache, ctx->event, field)-> + add_str("old_decision", "yes")-> + add_str("new_decision", "temp"); + e_debug(e->event(), "Purge changes field %s " + "cache decision yes -> temp " + "(last_used=%"PRIdTIME_T")", + priv->field.name, priv->field.last_used); + dec = MAIL_CACHE_DECISION_TEMP; + break; + } + } + priv->field.decision = dec; + + /* drop all fields we don't want */ + if ((dec & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) == MAIL_CACHE_DECISION_NO) { + priv->used = FALSE; + priv->field.last_used = 0; + } + return priv->used; +} + +static int +mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans, + struct event *event, int fd, const char *reason, + uint32_t *file_seq_r, uoff_t *file_size_r, uint32_t *max_uid_r, + uint32_t *ext_first_seq_r, ARRAY_TYPE(uint32_t) *ext_offsets) +{ + struct mail_cache_copy_context ctx; + struct mail_cache_lookup_iterate_ctx iter; + struct mail_cache_iterate_field field; + struct mail_index_view *view; + struct mail_cache_view *cache_view; + const struct mail_index_header *idx_hdr; + struct mail_cache_header hdr; + struct mail_cache_record cache_rec; + struct ostream *output; + uint32_t message_count, seq, first_new_seq, ext_offset; + unsigned int i, used_fields_count, orig_fields_count, record_count; + + i_assert(reason != NULL); + + *max_uid_r = 0; + *ext_first_seq_r = 0; + + /* get the latest info on fields */ + if (mail_cache_header_fields_read(cache) < 0) + return -1; + + view = mail_index_transaction_open_updated_view(trans); + cache_view = mail_cache_view_open(cache, view); + output = o_stream_create_fd_file(fd, 0, FALSE); + + i_zero(&hdr); + hdr.major_version = MAIL_CACHE_MAJOR_VERSION; + hdr.minor_version = MAIL_CACHE_MINOR_VERSION; + hdr.compat_sizeof_uoff_t = sizeof(uoff_t); + hdr.indexid = cache->index->indexid; + hdr.file_seq = get_next_file_seq(cache); + o_stream_nsend(output, &hdr, sizeof(hdr)); + + event_add_str(event, "reason", reason); + event_add_int(event, "file_seq", hdr.file_seq); + event_set_name(event, "mail_cache_purge_started"); + e_debug(event, "Purging (new file_seq=%u): %s", hdr.file_seq, reason); + + i_zero(&ctx); + ctx.cache = cache; + ctx.event = event; + ctx.buffer = buffer_create_dynamic(default_pool, 4096); + ctx.field_seen = buffer_create_dynamic(default_pool, 64); + ctx.field_seen_value = 0; + ctx.field_file_map = t_new(uint32_t, cache->fields_count + 1); + t_array_init(&ctx.bitmask_pos, 32); + + /* @UNSAFE: drop unused fields and create a field mapping for + used fields */ + idx_hdr = mail_index_get_header(view); + mail_cache_purge_drop_init(cache, idx_hdr, &ctx.drop_ctx); + + orig_fields_count = cache->fields_count; + if (cache->file_fields_count == 0) { + /* creating the initial cache file. add all fields. */ + for (i = 0; i < orig_fields_count; i++) + ctx.field_file_map[i] = i; + used_fields_count = i; + } else { + for (i = used_fields_count = 0; i < orig_fields_count; i++) { + if (!mail_cache_purge_check_field(&ctx, i)) + ctx.field_file_map[i] = (uint32_t)-1; + else + ctx.field_file_map[i] = used_fields_count++; + } + } + + /* get sequence of first message which doesn't need its temp fields + removed. */ + first_new_seq = mail_cache_get_first_new_seq(view); + message_count = mail_index_view_get_messages_count(view); + if (!trans->reset) + seq = 1; + else { + /* Index is being rebuilt. Ignore old messages. */ + seq = trans->first_new_seq; + } + + *ext_first_seq_r = seq; + i_array_init(ext_offsets, message_count); record_count = 0; + for (; seq <= message_count; seq++) { + if (mail_index_transaction_is_expunged(trans, seq)) { + array_append_zero(ext_offsets); + continue; + } + + ctx.new_msg = seq >= first_new_seq; + buffer_set_used_size(ctx.buffer, 0); + + ctx.field_seen_value = (ctx.field_seen_value + 1) & UINT8_MAX; + if (ctx.field_seen_value == 0) { + memset(buffer_get_modifiable_data(ctx.field_seen, NULL), + 0, buffer_get_size(ctx.field_seen)); + ctx.field_seen_value++; + } + array_clear(&ctx.bitmask_pos); + + i_zero(&cache_rec); + buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec)); + + mail_cache_lookup_iter_init(cache_view, seq, &iter); + while (mail_cache_lookup_iter_next(&iter, &field) > 0) + mail_cache_purge_field(&ctx, &field); + + if (ctx.buffer->used == sizeof(cache_rec) || + ctx.buffer->used > cache->index->optimization_set.cache.record_max_size) { + /* nothing cached */ + ext_offset = 0; + } else { + mail_index_lookup_uid(view, seq, max_uid_r); + cache_rec.size = ctx.buffer->used; + ext_offset = output->offset; + buffer_write(ctx.buffer, 0, &cache_rec, + sizeof(cache_rec)); + o_stream_nsend(output, ctx.buffer->data, cache_rec.size); + record_count++; + } + + array_push_back(ext_offsets, &ext_offset); + } + i_assert(orig_fields_count == cache->fields_count); + + hdr.record_count = record_count; + hdr.field_header_offset = mail_index_uint32_to_offset(output->offset); + mail_cache_purge_get_fields(&ctx, used_fields_count); + o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used); + + hdr.backwards_compat_used_file_size = output->offset; + buffer_free(&ctx.buffer); + buffer_free(&ctx.field_seen); + + *file_size_r = output->offset; + (void)o_stream_seek(output, 0); + o_stream_nsend(output, &hdr, sizeof(hdr)); + + mail_cache_view_close(&cache_view); + mail_index_view_close(&view); + + if (o_stream_finish(output) < 0) { + mail_cache_set_syscall_error(cache, "write()"); + o_stream_destroy(&output); + array_free(ext_offsets); + return -1; + } + o_stream_destroy(&output); + + if (cache->index->set.fsync_mode == FSYNC_MODE_ALWAYS) { + if (fdatasync(fd) < 0) { + mail_cache_set_syscall_error(cache, "fdatasync()"); + array_free(ext_offsets); + return -1; + } + } + + *file_seq_r = hdr.file_seq; + return 0; +} + +static int +mail_cache_purge_write(struct mail_cache *cache, + struct mail_index_transaction *trans, + int fd, const char *temp_path, const char * + reason, bool *unlock) +{ + struct event *event; + struct stat st; + uint32_t prev_file_seq, file_seq, old_offset, max_uid, ext_first_seq; + ARRAY_TYPE(uint32_t) ext_offsets; + const uint32_t *offsets; + uoff_t prev_file_size, file_size; + unsigned int i, count, prev_deleted_records; + + if (cache->hdr == NULL) { + prev_file_seq = 0; + prev_file_size = 0; + prev_deleted_records = 0; + } else { + prev_file_seq = cache->hdr->file_seq; + prev_file_size = cache->last_stat_size; + prev_deleted_records = cache->hdr->deleted_record_count; + } + event = event_create(cache->event); + event_add_int(event, "prev_file_seq", prev_file_seq); + event_add_int(event, "prev_file_size", prev_file_size); + event_add_int(event, "prev_deleted_records", prev_deleted_records); + + if (mail_cache_copy(cache, trans, event, fd, reason, + &file_seq, &file_size, &max_uid, + &ext_first_seq, &ext_offsets) < 0) + return -1; + + if (fstat(fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + array_free(&ext_offsets); + return -1; + } + if (rename(temp_path, cache->filepath) < 0) { + mail_cache_set_syscall_error(cache, "rename()"); + array_free(&ext_offsets); + return -1; + } + + event_add_int(event, "file_size", file_size); + event_add_int(event, "max_uid", max_uid); + event_set_name(event, "mail_cache_purge_finished"); + e_debug(event, "Purging finished, file_seq changed %u -> %u, " + "size=%"PRIuUOFF_T" -> %"PRIuUOFF_T", max_uid=%u", + prev_file_seq, file_seq, prev_file_size, file_size, max_uid); + event_unref(&event); + + /* once we're sure that the purging was successful, + update the offsets */ + mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE); + offsets = array_get(&ext_offsets, &count); + for (i = 0; i < count; i++) { + if (offsets[i] != 0) { + mail_index_update_ext(trans, ext_first_seq + i, + cache->ext_id, + &offsets[i], &old_offset); + } + } + array_free(&ext_offsets); + + if (*unlock) { + mail_cache_unlock(cache); + *unlock = FALSE; + } + + mail_cache_file_close(cache); + cache->opened = TRUE; + cache->fd = fd; + cache->st_ino = st.st_ino; + cache->st_dev = st.st_dev; + cache->field_header_write_pending = FALSE; + return 0; +} + +static int +mail_cache_purge_has_file_changed(struct mail_cache *cache, + uint32_t purge_file_seq) +{ + struct mail_cache_header hdr; + unsigned int i; + int fd, ret; + + for (i = 0;; i++) { + fd = nfs_safe_open(cache->filepath, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) + return 0; + + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + ret = read_full(fd, &hdr, sizeof(hdr)); + i_close_fd(&fd); + + if (ret >= 0) { + if (ret == 0) + return 0; + if (purge_file_seq == 0) { + /* previously it didn't exist or it + was unusable and was just unlinked */ + return 1; + } + return hdr.file_seq != purge_file_seq ? 1 : 0; + } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) { + mail_cache_set_syscall_error(cache, "read()"); + return -1; + } + } +} + +static int mail_cache_purge_locked(struct mail_cache *cache, + uint32_t purge_file_seq, + struct mail_index_transaction *trans, + const char *reason, bool *unlock) +{ + const char *temp_path; + int fd, ret; + + /* we've locked the cache purging now. if somebody else had just + recreated the cache, reopen the cache and return success. */ + if (purge_file_seq != (uint32_t)-1 && + (ret = mail_cache_purge_has_file_changed(cache, purge_file_seq)) != 0) { + if (ret < 0) + return -1; + + /* was just purged, forget this */ + mail_cache_purge_later_reset(cache); + + if (*unlock) { + (void)mail_cache_unlock(cache); + *unlock = FALSE; + } + + return mail_cache_reopen(cache) < 0 ? -1 : 0; + } + if (cache->fd != -1) { + /* make sure we have mapped it before reading. */ + if (mail_cache_map_all(cache) <= 0) + return -1; + } + + /* we want to recreate the cache. write it first to a temporary file */ + fd = mail_index_create_tmp_file(cache->index, cache->filepath, &temp_path); + if (fd == -1) + return -1; + if (mail_cache_purge_write(cache, trans, fd, temp_path, reason, unlock) < 0) { + i_close_fd(&fd); + i_unlink(temp_path); + return -1; + } + if (cache->file_cache != NULL) + file_cache_set_fd(cache->file_cache, cache->fd); + + if (mail_cache_map_all(cache) <= 0) + return -1; + if (mail_cache_header_fields_read(cache) < 0) + return -1; + + mail_cache_purge_later_reset(cache); + return 0; +} + +static int +mail_cache_purge_full(struct mail_cache *cache, + struct mail_index_transaction *trans, + uint32_t purge_file_seq, const char *reason) +{ + bool unlock = FALSE; + int ret; + + i_assert(!cache->purging); + i_assert(cache->index->log_sync_locked); + + if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly) + return 0; + + /* purging isn't very efficient with small read()s */ + if (cache->map_with_read) { + cache->map_with_read = FALSE; + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + } + + /* .log lock already prevents other processes from purging cache at + the same time, but locking the cache file itself prevents other + processes from doing other changes to it (header changes, adding + more cached data). */ + switch (mail_cache_lock(cache)) { + case -1: + /* lock timeout or some other error */ + return -1; + case 0: + /* cache is broken or doesn't exist. + just start creating it. */ + break; + default: + /* locking succeeded. */ + unlock = TRUE; + } + cache->purging = TRUE; + ret = mail_cache_purge_locked(cache, purge_file_seq, trans, reason, &unlock); + cache->purging = FALSE; + if (unlock) + mail_cache_unlock(cache); + i_assert(!cache->hdr_modified); + if (ret < 0) { + /* the fields may have been updated in memory already. + reverse those changes by re-reading them from file. */ + (void)mail_cache_header_fields_read(cache); + } + return ret; +} + +int mail_cache_purge_with_trans(struct mail_cache *cache, + struct mail_index_transaction *trans, + uint32_t purge_file_seq, const char *reason) +{ + return mail_cache_purge_full(cache, trans, purge_file_seq, reason); +} + +int mail_cache_purge(struct mail_cache *cache, uint32_t purge_file_seq, + const char *reason) +{ + struct mail_index_view *view; + struct mail_index_transaction *trans; + bool lock_log; + int ret; + + lock_log = !cache->index->log_sync_locked; + if (lock_log) { + uint32_t file_seq; + uoff_t file_offset; + + if (mail_transaction_log_sync_lock(cache->index->log, + "mail cache purge", + &file_seq, &file_offset) < 0) + return -1; + } + /* make sure we see the latest changes in index */ + ret = mail_index_refresh(cache->index); + + view = mail_index_view_open(cache->index); + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + if (ret < 0) + ; + else if ((ret = mail_cache_purge_full(cache, trans, purge_file_seq, + reason)) < 0) + mail_index_transaction_rollback(&trans); + else { + if (mail_index_transaction_commit(&trans) < 0) + ret = -1; + } + mail_index_view_close(&view); + if (lock_log) { + mail_transaction_log_sync_unlock(cache->index->log, + "mail cache purge"); + } + return ret; +} + +bool mail_cache_need_purge(struct mail_cache *cache, const char **reason_r) +{ + if (cache->need_purge_file_seq == 0) + return FALSE; /* delayed purging not requested */ + if (cache->index->readonly) + return FALSE; /* no purging when opened as read-only */ + if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0) { + /* Mail deliveries don't really need to purge, even if there + could be some work to do. Just delay until the next regular + mail access comes before doing any extra work. */ + return FALSE; + } + + i_assert(cache->need_purge_reason != NULL); + /* t_strdup() the reason in case it gets freed (or replaced) + before it's used */ + *reason_r = t_strdup(cache->need_purge_reason); + return TRUE; +} + +void mail_cache_purge_later(struct mail_cache *cache, const char *reason) +{ + i_assert(cache->hdr != NULL); + + cache->need_purge_file_seq = cache->hdr->file_seq; + i_free(cache->need_purge_reason); + cache->need_purge_reason = i_strdup(reason); +} + +void mail_cache_purge_later_reset(struct mail_cache *cache) +{ + cache->need_purge_file_seq = 0; + i_free(cache->need_purge_reason); +} + +void mail_cache_purge_drop_init(struct mail_cache *cache, + const struct mail_index_header *hdr, + struct mail_cache_purge_drop_ctx *ctx_r) +{ + i_zero(ctx_r); + ctx_r->cache = cache; + if (hdr->day_stamp != 0) { + const struct mail_index_cache_optimization_settings *opt = + &cache->index->optimization_set.cache; + ctx_r->max_yes_downgrade_time = hdr->day_stamp - + opt->unaccessed_field_drop_secs; + ctx_r->max_temp_drop_time = hdr->day_stamp - + 2 * opt->unaccessed_field_drop_secs; + } +} + +enum mail_cache_purge_drop_decision +mail_cache_purge_drop_test(struct mail_cache_purge_drop_ctx *ctx, + unsigned int field) +{ + struct mail_cache_field_private *priv = &ctx->cache->fields[field]; + enum mail_cache_decision_type dec = priv->field.decision; + + if ((dec & MAIL_CACHE_DECISION_FORCED) != 0) + return MAIL_CACHE_PURGE_DROP_DECISION_NONE; + if (dec != MAIL_CACHE_DECISION_NO && + priv->field.last_used < ctx->max_temp_drop_time) { + /* YES or TEMP decision field hasn't been accessed for a long + time now. Drop it. */ + return MAIL_CACHE_PURGE_DROP_DECISION_DROP; + } + if (dec == MAIL_CACHE_DECISION_YES && + priv->field.last_used < ctx->max_yes_downgrade_time) { + /* YES decision field hasn't been accessed for a while + now. Change its decision to TEMP. */ + return MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP; + } + return MAIL_CACHE_PURGE_DROP_DECISION_NONE; +} diff --git a/src/lib-index/mail-cache-sync-update.c b/src/lib-index/mail-cache-sync-update.c new file mode 100644 index 0000000..9073187 --- /dev/null +++ b/src/lib-index/mail-cache-sync-update.c @@ -0,0 +1,68 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-cache-private.h" +#include "mail-index-sync-private.h" + +struct mail_cache_sync_context { + unsigned expunge_count; +}; + +void mail_cache_expunge_count(struct mail_cache *cache, unsigned int count) +{ + if (mail_cache_lock(cache) > 0) { + cache->hdr_copy.deleted_record_count += count; + if (cache->hdr_copy.record_count >= count) + cache->hdr_copy.record_count -= count; + else + cache->hdr_copy.record_count = 0; + cache->hdr_modified = TRUE; + (void)mail_cache_flush_and_unlock(cache); + } +} + +static struct mail_cache_sync_context *mail_cache_handler_init(void **context) +{ + struct mail_cache_sync_context *ctx; + + if (*context != NULL) + ctx = *context; + else { + *context = i_new(struct mail_cache_sync_context, 1); + ctx = *context; + } + return ctx; +} + +static void mail_cache_handler_deinit(struct mail_index_sync_map_ctx *sync_ctx, + struct mail_cache_sync_context *ctx) +{ + struct mail_cache *cache = sync_ctx->view->index->cache; + + if (ctx == NULL) + return; + + mail_cache_expunge_count(cache, ctx->expunge_count); + + i_free(ctx); +} + +int mail_cache_expunge_handler(struct mail_index_sync_map_ctx *sync_ctx, + const void *data, void **sync_context) +{ + struct mail_cache_sync_context *ctx = *sync_context; + const uint32_t *cache_offset = data; + + if (data == NULL) { + mail_cache_handler_deinit(sync_ctx, ctx); + *sync_context = NULL; + return 0; + } + + if (*cache_offset == 0) + return 0; + + ctx = mail_cache_handler_init(sync_context); + ctx->expunge_count++; + return 0; +} diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c new file mode 100644 index 0000000..0cdf089 --- /dev/null +++ b/src/lib-index/mail-cache-transaction.c @@ -0,0 +1,929 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "module-context.h" +#include "file-cache.h" +#include "file-set-size.h" +#include "read-full.h" +#include "write-full.h" +#include "mail-cache-private.h" +#include "mail-index-transaction-private.h" + +#include <stddef.h> +#include <sys/stat.h> + +#define MAIL_CACHE_INIT_WRITE_BUFFER (1024*16) + +#define CACHE_TRANS_CONTEXT(obj) \ + MODULE_CONTEXT(obj, cache_mail_index_transaction_module) +#define CACHE_TRANS_CONTEXT_REQUIRE(obj) \ + MODULE_CONTEXT_REQUIRE(obj, cache_mail_index_transaction_module) + +struct mail_cache_transaction_rec { + uint32_t seq; + uint32_t cache_data_pos; +}; + +struct mail_cache_transaction_ctx { + union mail_index_transaction_module_context module_ctx; + struct mail_index_transaction_vfuncs super; + + struct mail_cache *cache; + struct mail_cache_view *view; + struct mail_index_transaction *trans; + + uint32_t cache_file_seq; + uint32_t first_new_seq; + + buffer_t *cache_data; + ARRAY(uint8_t) cache_field_idx_used; + ARRAY(struct mail_cache_transaction_rec) cache_data_seq; + ARRAY_TYPE(seq_range) cache_data_wanted_seqs; + uint32_t prev_seq, min_seq; + size_t last_rec_pos; + + unsigned int records_written; + + bool tried_purging:1; + bool decisions_refreshed:1; + bool have_noncommited_mails:1; + bool changes:1; +}; + +static MODULE_CONTEXT_DEFINE_INIT(cache_mail_index_transaction_module, + &mail_index_module_register); + +static int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx); +static bool +mail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx, + size_t *size_r); +static int mail_cache_header_rewrite_fields(struct mail_cache *cache); + +static void mail_index_transaction_cache_reset(struct mail_index_transaction *t) +{ + struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT_REQUIRE(t); + struct mail_index_transaction_vfuncs super = ctx->super; + + mail_cache_transaction_reset(ctx); + super.reset(t); +} + +static int +mail_index_transaction_cache_commit(struct mail_index_transaction *t, + struct mail_index_transaction_commit_result *result_r) +{ + struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT_REQUIRE(t); + struct mail_index_transaction_vfuncs super = ctx->super; + + /* a failed cache commit isn't important enough to fail the entire + index transaction, so we'll just ignore it */ + (void)mail_cache_transaction_commit(&ctx); + return super.commit(t, result_r); +} + +static void +mail_index_transaction_cache_rollback(struct mail_index_transaction *t) +{ + struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT_REQUIRE(t); + struct mail_index_transaction_vfuncs super = ctx->super; + + mail_cache_transaction_rollback(&ctx); + super.rollback(t); +} + +struct mail_cache_transaction_ctx * +mail_cache_get_transaction(struct mail_cache_view *view, + struct mail_index_transaction *t) +{ + struct mail_cache_transaction_ctx *ctx; + + ctx = !cache_mail_index_transaction_module.id.module_id_set ? NULL : + CACHE_TRANS_CONTEXT(t); + + if (ctx != NULL) + return ctx; + + ctx = i_new(struct mail_cache_transaction_ctx, 1); + ctx->cache = view->cache; + ctx->view = view; + ctx->trans = t; + + i_assert(view->transaction == NULL); + view->transaction = ctx; + view->trans_view = mail_index_transaction_open_updated_view(t); + + ctx->super = t->v; + t->v.reset = mail_index_transaction_cache_reset; + t->v.commit = mail_index_transaction_cache_commit; + t->v.rollback = mail_index_transaction_cache_rollback; + + MODULE_CONTEXT_SET(t, cache_mail_index_transaction_module, ctx); + return ctx; +} + +static void +mail_cache_transaction_forget_flushed(struct mail_cache_transaction_ctx *ctx, + bool reset_id_changed) +{ + uint32_t new_cache_file_seq = MAIL_CACHE_IS_UNUSABLE(ctx->cache) ? 0 : + ctx->cache->hdr->file_seq; + if (reset_id_changed && ctx->records_written > 0) { + e_warning(ctx->cache->event, + "Purging lost %u written cache records " + "(reset_id changed %u -> %u)", ctx->records_written, + ctx->cache_file_seq, new_cache_file_seq); + /* don't increase deleted_record_count in the new file */ + ctx->records_written = 0; + } + ctx->cache_file_seq = new_cache_file_seq; + /* forget all cache extension updates even if reset_id doesn't change */ + mail_index_ext_set_reset_id(ctx->trans, ctx->cache->ext_id, + ctx->cache_file_seq); +} + +void mail_cache_transaction_reset(struct mail_cache_transaction_ctx *ctx) +{ + mail_cache_transaction_forget_flushed(ctx, FALSE); + if (ctx->cache_data != NULL) + buffer_set_used_size(ctx->cache_data, 0); + if (array_is_created(&ctx->cache_data_seq)) + array_clear(&ctx->cache_data_seq); + ctx->prev_seq = 0; + ctx->last_rec_pos = 0; + + ctx->changes = FALSE; +} + +void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx **_ctx) +{ + struct mail_cache_transaction_ctx *ctx = *_ctx; + + *_ctx = NULL; + + if (ctx->records_written > 0) { + /* we already wrote to the cache file. we can't (or don't want + to) delete that data, so just mark it as deleted space */ + if (mail_cache_transaction_lock(ctx) > 0) { + ctx->cache->hdr_copy.deleted_record_count += + ctx->records_written; + ctx->cache->hdr_modified = TRUE; + (void)mail_cache_flush_and_unlock(ctx->cache); + } + } + + MODULE_CONTEXT_UNSET(ctx->trans, cache_mail_index_transaction_module); + + ctx->view->transaction = NULL; + ctx->view->trans_seq1 = ctx->view->trans_seq2 = 0; + + mail_index_view_close(&ctx->view->trans_view); + buffer_free(&ctx->cache_data); + if (array_is_created(&ctx->cache_data_seq)) + array_free(&ctx->cache_data_seq); + if (array_is_created(&ctx->cache_data_wanted_seqs)) + array_free(&ctx->cache_data_wanted_seqs); + array_free(&ctx->cache_field_idx_used); + i_free(ctx); +} + +bool mail_cache_transactions_have_changes(struct mail_cache *cache) +{ + struct mail_cache_view *view; + + for (view = cache->views; view != NULL; view = view->next) { + if (view->transaction != NULL && + view->transaction->changes) + return TRUE; + } + return FALSE; +} + +static int +mail_cache_transaction_purge(struct mail_cache_transaction_ctx *ctx, + const char *reason) +{ + struct mail_cache *cache = ctx->cache; + + ctx->tried_purging = TRUE; + + uint32_t purge_file_seq = + MAIL_CACHE_IS_UNUSABLE(cache) ? 0 : cache->hdr->file_seq; + + int ret = mail_cache_purge(cache, purge_file_seq, reason); + /* already written cache records must be forgotten, but records in + memory can still be written to the new cache file */ + mail_cache_transaction_forget_flushed(ctx, TRUE); + return ret; +} + +static int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache *cache = ctx->cache; + const uoff_t cache_max_size = + cache->index->optimization_set.cache.max_size; + int ret; + + if ((ret = mail_cache_lock(cache)) <= 0) { + if (ret < 0) + return -1; + + if (!ctx->tried_purging) { + if (mail_cache_transaction_purge(ctx, "creating cache") < 0) + return -1; + return mail_cache_transaction_lock(ctx); + } else { + return 0; + } + } + i_assert(!MAIL_CACHE_IS_UNUSABLE(cache)); + + if (!ctx->tried_purging && ctx->cache_data != NULL && + cache->last_stat_size + ctx->cache_data->used > cache_max_size) { + /* Looks like cache file is becoming too large. Try to purge + it to free up some space. */ + if (cache->hdr->continued_record_count > 0 || + cache->hdr->deleted_record_count > 0) { + mail_cache_unlock(cache); + (void)mail_cache_transaction_purge(ctx, "cache is too large"); + return mail_cache_transaction_lock(ctx); + } + } + + if (ctx->cache_file_seq == 0) + ctx->cache_file_seq = cache->hdr->file_seq; + else if (ctx->cache_file_seq != cache->hdr->file_seq) { + /* already written cache records must be forgotten, but records + in memory can still be written to the new cache file */ + mail_cache_transaction_forget_flushed(ctx, TRUE); + i_assert(ctx->cache_file_seq == cache->hdr->file_seq); + } + return 1; +} + +const struct mail_cache_record * +mail_cache_transaction_lookup_rec(struct mail_cache_transaction_ctx *ctx, + unsigned int seq, + unsigned int *trans_next_idx) +{ + const struct mail_cache_transaction_rec *recs; + unsigned int i, count; + + recs = array_get(&ctx->cache_data_seq, &count); + for (i = *trans_next_idx; i < count; i++) { + if (recs[i].seq == seq) { + *trans_next_idx = i + 1; + return CONST_PTR_OFFSET(ctx->cache_data->data, + recs[i].cache_data_pos); + } + } + *trans_next_idx = i + 1; + if (seq == ctx->prev_seq && i == count) { + /* update the unfinished record's (temporary) size and + return it */ + size_t size; + if (!mail_cache_transaction_update_last_rec_size(ctx, &size)) + return NULL; + return CONST_PTR_OFFSET(ctx->cache_data->data, + ctx->last_rec_pos); + } + return NULL; +} + +static void +mail_cache_transaction_update_index(struct mail_cache_transaction_ctx *ctx, + uint32_t write_offset, bool committing) +{ + struct mail_cache *cache = ctx->cache; + struct mail_index_transaction *trans; + const struct mail_cache_record *rec = ctx->cache_data->data; + const struct mail_cache_transaction_rec *recs; + uint32_t i, seq_count; + + if (committing) { + /* The transaction is being committed now. Use it. */ + trans = ctx->trans; + } else if (ctx->have_noncommited_mails) { + /* Some of the mails haven't been committed yet. We must use + the provided transaction to update the cache records. */ + trans = ctx->trans; + } else { + /* We can commit these changes immediately. This way even if + the provided transaction runs for a very long time, we + still once in a while commit the cache changes so they + become visible to other processes as well. */ + trans = mail_index_transaction_begin(ctx->view->trans_view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + } + + mail_index_ext_using_reset_id(trans, ctx->cache->ext_id, + ctx->cache_file_seq); + + /* write the cache_offsets to index file. records' prev_offset + is updated to point to old cache record when index is being + synced. */ + recs = array_get(&ctx->cache_data_seq, &seq_count); + for (i = 0; i < seq_count; i++) { + mail_index_update_ext(trans, recs[i].seq, cache->ext_id, + &write_offset, NULL); + + write_offset += rec->size; + rec = CONST_PTR_OFFSET(rec, rec->size); + ctx->records_written++; + } + if (trans != ctx->trans) { + i_assert(cache->index->log_sync_locked); + if (mail_index_transaction_commit(&trans) < 0) { + /* failed, but can't really do anything */ + } else { + ctx->records_written = 0; + } + } +} + +static int +mail_cache_link_records(struct mail_cache_transaction_ctx *ctx, + uint32_t write_offset) +{ + struct mail_index_map *map; + struct mail_cache_record *rec; + const struct mail_cache_transaction_rec *recs; + const uint32_t *prev_offsetp; + ARRAY_TYPE(uint32_t) seq_offsets; + uint32_t i, seq_count, reset_id, prev_offset, *offsetp; + const void *data; + + i_assert(ctx->min_seq != 0); + + i_array_init(&seq_offsets, 64); + recs = array_get(&ctx->cache_data_seq, &seq_count); + rec = buffer_get_modifiable_data(ctx->cache_data, NULL); + for (i = 0; i < seq_count; i++) { + offsetp = array_idx_get_space(&seq_offsets, + recs[i].seq - ctx->min_seq); + if (*offsetp != 0) + prev_offset = *offsetp; + else { + mail_index_lookup_ext_full(ctx->view->trans_view, recs[i].seq, + ctx->cache->ext_id, &map, + &data, NULL); + prev_offsetp = data; + + if (prev_offsetp == NULL || *prev_offsetp == 0) + prev_offset = 0; + else if (mail_index_ext_get_reset_id(ctx->view->trans_view, map, + ctx->cache->ext_id, + &reset_id) && + reset_id == ctx->cache_file_seq) + prev_offset = *prev_offsetp; + else + prev_offset = 0; + if (prev_offset >= write_offset) { + mail_cache_set_corrupted(ctx->cache, + "Cache record offset points outside existing file"); + array_free(&seq_offsets); + return -1; + } + } + + if (prev_offset != 0) { + /* link this record to previous one */ + rec->prev_offset = prev_offset; + ctx->cache->hdr_copy.continued_record_count++; + } else { + ctx->cache->hdr_copy.record_count++; + } + *offsetp = write_offset; + + write_offset += rec->size; + rec = PTR_OFFSET(rec, rec->size); + } + array_free(&seq_offsets); + ctx->cache->hdr_modified = TRUE; + return 0; +} + +static bool +mail_cache_transaction_set_used(struct mail_cache_transaction_ctx *ctx) +{ + const uint8_t *cache_fields_used; + unsigned int field_idx, count; + bool missing_file_fields = FALSE; + + cache_fields_used = array_get(&ctx->cache_field_idx_used, &count); + i_assert(count <= ctx->cache->fields_count); + for (field_idx = 0; field_idx < count; field_idx++) { + if (cache_fields_used[field_idx] != 0) { + ctx->cache->fields[field_idx].used = TRUE; + if (ctx->cache->field_file_map[field_idx] == (uint32_t)-1) + missing_file_fields = TRUE; + } + } + return missing_file_fields; +} + +static int +mail_cache_transaction_update_fields(struct mail_cache_transaction_ctx *ctx) +{ + unsigned char *p; + const unsigned char *end, *rec_end; + uint32_t field_idx, data_size; + + if (mail_cache_transaction_set_used(ctx)) { + /* add missing fields to cache */ + if (mail_cache_header_rewrite_fields(ctx->cache) < 0) + return -1; + /* make sure they were actually added */ + if (mail_cache_transaction_set_used(ctx)) { + mail_index_set_error(ctx->cache->index, + "Cache file %s: Unexpectedly lost newly added field", + ctx->cache->filepath); + return -1; + } + } + + /* Go through all the added cache records and replace the in-memory + field_idx with the cache file-specific field index. Update only + up to last_rec_pos, because that's how far flushing is done. The + fields after that keep the in-memory field_idx until the next + flush. */ + p = buffer_get_modifiable_data(ctx->cache_data, NULL); + end = CONST_PTR_OFFSET(ctx->cache_data->data, ctx->last_rec_pos); + rec_end = p; + while (p < end) { + if (p >= rec_end) { + /* next cache record */ + i_assert(p == rec_end); + const struct mail_cache_record *rec = + (const struct mail_cache_record *)p; + /* note that the last rec->size==0 */ + rec_end = CONST_PTR_OFFSET(p, rec->size); + p += sizeof(*rec); + } + /* replace field_idx */ + uint32_t *file_fieldp = (uint32_t *)p; + field_idx = *file_fieldp; + *file_fieldp = ctx->cache->field_file_map[field_idx]; + i_assert(*file_fieldp != (uint32_t)-1); + p += sizeof(field_idx); + + /* Skip to next cache field. Next is <data size> if the field + is not fixed size. */ + data_size = ctx->cache->fields[field_idx].field.field_size; + if (data_size == UINT_MAX) { + memcpy(&data_size, p, sizeof(data_size)); + p += sizeof(data_size); + } + /* data & 32bit padding */ + p += data_size; + if ((data_size & 3) != 0) + p += 4 - (data_size & 3); + } + i_assert(p == end); + return 0; +} + +static void +mail_cache_transaction_drop_last_flush(struct mail_cache_transaction_ctx *ctx) +{ + buffer_copy(ctx->cache_data, 0, + ctx->cache_data, ctx->last_rec_pos, SIZE_MAX); + buffer_set_used_size(ctx->cache_data, + ctx->cache_data->used - ctx->last_rec_pos); + ctx->last_rec_pos = 0; + ctx->min_seq = 0; + + array_clear(&ctx->cache_data_seq); + array_clear(&ctx->cache_data_wanted_seqs); +} + +static int +mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx, + bool committing) +{ + struct stat st; + uint32_t write_offset = 0; + int ret = 0; + + i_assert(!ctx->cache->locked); + + if (array_count(&ctx->cache_data_seq) == 0) { + /* we had done some changes, but they were aborted. */ + i_assert(ctx->last_rec_pos == 0); + ctx->min_seq = 0; + return 0; + } + + /* If we're going to be committing a transaction, the log must be + locked before we lock cache or we can deadlock. */ + bool lock_log = !ctx->cache->index->log_sync_locked && + !committing && !ctx->have_noncommited_mails; + if (lock_log) { + uint32_t file_seq; + uoff_t file_offset; + + if (mail_transaction_log_sync_lock(ctx->cache->index->log, + "mail cache transaction flush", + &file_seq, &file_offset) < 0) + return -1; + } + + if (mail_cache_transaction_lock(ctx) <= 0) { + if (lock_log) { + mail_transaction_log_sync_unlock(ctx->cache->index->log, + "mail cache transaction flush: cache lock failed"); + } + return -1; + } + + i_assert(ctx->cache_data != NULL); + i_assert(ctx->last_rec_pos <= ctx->cache_data->used); + + if (mail_cache_transaction_update_fields(ctx) < 0) { + if (lock_log) { + mail_transaction_log_sync_unlock(ctx->cache->index->log, + "mail cache transaction flush: field update failed"); + } + mail_cache_unlock(ctx->cache); + return -1; + } + + /* we need to get the final write offset for linking records */ + if (fstat(ctx->cache->fd, &st) < 0) { + if (!ESTALE_FSTAT(errno)) + mail_cache_set_syscall_error(ctx->cache, "fstat()"); + ret = -1; + } else if ((uoff_t)st.st_size + ctx->last_rec_pos > ctx->cache->index->optimization_set.cache.max_size) { + mail_cache_set_corrupted(ctx->cache, "Cache file too large"); + ret = -1; + } else { + write_offset = st.st_size; + if (mail_cache_link_records(ctx, write_offset) < 0) + ret = -1; + } + + /* write to cache file */ + if (ret < 0 || + mail_cache_append(ctx->cache, ctx->cache_data->data, + ctx->last_rec_pos, &write_offset) < 0) + ret = -1; + else { + /* update records' cache offsets to index */ + mail_cache_transaction_update_index(ctx, write_offset, + committing); + } + if (mail_cache_flush_and_unlock(ctx->cache) < 0) + ret = -1; + + if (lock_log) { + mail_transaction_log_sync_unlock(ctx->cache->index->log, + "mail cache transaction flush"); + } + return ret; +} + +static void +mail_cache_transaction_drop_unwanted(struct mail_cache_transaction_ctx *ctx, + size_t space_needed) +{ + struct mail_cache_transaction_rec *recs; + unsigned int i, count; + + recs = array_get_modifiable(&ctx->cache_data_seq, &count); + /* find out how many records to delete. delete all unwanted sequences, + and if that's not enough delete some more. */ + for (i = 0; i < count; i++) { + if (seq_range_exists(&ctx->cache_data_wanted_seqs, recs[i].seq)) { + if (recs[i].cache_data_pos >= space_needed) + break; + /* we're going to forcibly delete it - remove it also + from the array since it's no longer useful there */ + seq_range_array_remove(&ctx->cache_data_wanted_seqs, + recs[i].seq); + } + } + unsigned int deleted_count = i; + size_t deleted_space = i < count ? + recs[i].cache_data_pos : ctx->last_rec_pos; + for (; i < count; i++) + recs[i].cache_data_pos -= deleted_space; + ctx->last_rec_pos -= deleted_space; + array_delete(&ctx->cache_data_seq, 0, deleted_count); + buffer_delete(ctx->cache_data, 0, deleted_space); +} + +static bool +mail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx, + size_t *size_r) +{ + struct mail_cache_record *rec; + void *data; + size_t size; + + data = buffer_get_modifiable_data(ctx->cache_data, &size); + rec = PTR_OFFSET(data, ctx->last_rec_pos); + rec->size = size - ctx->last_rec_pos; + if (rec->size == sizeof(*rec)) + return FALSE; + i_assert(rec->size > sizeof(*rec)); + *size_r = rec->size; + return TRUE; +} + +static void +mail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache_transaction_rec *trans_rec; + size_t size; + + if (!mail_cache_transaction_update_last_rec_size(ctx, &size) || + size > ctx->cache->index->optimization_set.cache.record_max_size) { + buffer_set_used_size(ctx->cache_data, ctx->last_rec_pos); + return; + } + + if (ctx->min_seq > ctx->prev_seq || ctx->min_seq == 0) + ctx->min_seq = ctx->prev_seq; + trans_rec = array_append_space(&ctx->cache_data_seq); + trans_rec->seq = ctx->prev_seq; + trans_rec->cache_data_pos = ctx->last_rec_pos; + ctx->last_rec_pos = ctx->cache_data->used; +} + +static void +mail_cache_transaction_switch_seq(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache_record new_rec; + + if (ctx->prev_seq != 0) { + /* update previously added cache record's size */ + mail_cache_transaction_update_last_rec(ctx); + } else if (ctx->cache_data == NULL) { + ctx->cache_data = + buffer_create_dynamic(default_pool, + MAIL_CACHE_INIT_WRITE_BUFFER); + i_array_init(&ctx->cache_data_seq, 64); + i_array_init(&ctx->cache_data_wanted_seqs, 32); + i_array_init(&ctx->cache_field_idx_used, 64); + } + + i_zero(&new_rec); + buffer_append(ctx->cache_data, &new_rec, sizeof(new_rec)); + + ctx->prev_seq = 0; + ctx->changes = TRUE; +} + +int mail_cache_transaction_commit(struct mail_cache_transaction_ctx **_ctx) +{ + struct mail_cache_transaction_ctx *ctx = *_ctx; + int ret = 0; + + if (ctx->changes) { + if (ctx->prev_seq != 0) + mail_cache_transaction_update_last_rec(ctx); + if (mail_cache_transaction_flush(ctx, TRUE) < 0) + ret = -1; + else { + /* successfully wrote everything */ + ctx->records_written = 0; + } + /* Here would be a good place to do fdatasync() to make sure + everything is written before offsets are updated to index. + However it slows down I/O needlessly and we're pretty good + at catching and fixing cache corruption, so we no longer do + it. */ + } + mail_cache_transaction_rollback(_ctx); + return ret; +} + +static int +mail_cache_header_fields_write(struct mail_cache *cache, const buffer_t *buffer) +{ + uint32_t offset, hdr_offset; + + i_assert(cache->locked); + + offset = 0; + if (mail_cache_append(cache, buffer->data, buffer->used, &offset) < 0) + return -1; + + if (cache->index->set.fsync_mode == FSYNC_MODE_ALWAYS) { + if (fdatasync(cache->fd) < 0) { + mail_cache_set_syscall_error(cache, "fdatasync()"); + return -1; + } + } + /* find offset to the previous header's "next_offset" field */ + if (mail_cache_header_fields_get_next_offset(cache, &hdr_offset) < 0) + return -1; + + /* update the next_offset offset, so our new header will be found */ + offset = mail_index_uint32_to_offset(offset); + if (mail_cache_write(cache, &offset, sizeof(offset), hdr_offset) < 0) + return -1; + + if (hdr_offset == offsetof(struct mail_cache_header, + field_header_offset)) { + /* we're adding the first field. hdr_copy needs to be kept + in sync so unlocking won't overwrite it. */ + cache->hdr_copy.field_header_offset = hdr_offset; + cache->hdr_ro_copy.field_header_offset = hdr_offset; + } + return 0; +} + +static int mail_cache_header_rewrite_fields(struct mail_cache *cache) +{ + int ret; + + /* re-read header to make sure we don't lose any fields. */ + if (mail_cache_header_fields_read(cache) < 0) + return -1; + + T_BEGIN { + buffer_t *buffer; + + buffer = t_buffer_create(256); + mail_cache_header_fields_get(cache, buffer); + ret = mail_cache_header_fields_write(cache, buffer); + } T_END; + + if (ret == 0) { + /* we wrote all the headers, so there are no pending changes */ + cache->field_header_write_pending = FALSE; + ret = mail_cache_header_fields_read(cache); + } + return ret; +} + +static void +mail_cache_transaction_refresh_decisions(struct mail_cache_transaction_ctx *ctx) +{ + if (ctx->decisions_refreshed) + return; + + /* Read latest caching decisions from the cache file's header once + per transaction. */ + if (!ctx->cache->opened) + (void)mail_cache_open_and_verify(ctx->cache); + else + (void)mail_cache_header_fields_read(ctx->cache); + ctx->decisions_refreshed = TRUE; +} + +void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, + unsigned int field_idx, const void *data, size_t data_size) +{ + uint32_t data_size32; + unsigned int fixed_size; + size_t full_size, record_size; + + i_assert(field_idx < ctx->cache->fields_count); + i_assert(data_size < (uint32_t)-1); + + if (ctx->cache->fields[field_idx].field.decision == + (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED)) + return; + + if (seq >= ctx->trans->first_new_seq) + ctx->have_noncommited_mails = TRUE; + + /* If the cache file exists, make sure the caching decisions have been + read. */ + mail_cache_transaction_refresh_decisions(ctx); + + mail_cache_decision_add(ctx->view, seq, field_idx); + + fixed_size = ctx->cache->fields[field_idx].field.field_size; + i_assert(fixed_size == UINT_MAX || fixed_size == data_size); + + data_size32 = (uint32_t)data_size; + full_size = sizeof(field_idx) + ((data_size + 3) & ~3U); + if (fixed_size == UINT_MAX) + full_size += sizeof(data_size32); + + if (ctx->prev_seq != seq) { + mail_cache_transaction_switch_seq(ctx); + ctx->prev_seq = seq; + seq_range_array_add(&ctx->cache_data_wanted_seqs, seq); + + /* remember roughly what we have modified, so cache lookups can + look into transactions to see changes. */ + if (seq < ctx->view->trans_seq1 || ctx->view->trans_seq1 == 0) + ctx->view->trans_seq1 = seq; + if (seq > ctx->view->trans_seq2) + ctx->view->trans_seq2 = seq; + } + + if (mail_cache_transaction_update_last_rec_size(ctx, &record_size) && + record_size + full_size > + ctx->cache->index->optimization_set.cache.record_max_size) { + /* Adding this field would exceed the cache record's maximum + size. If we don't add this, it's possible that other fields + could still be added. */ + return; + } + + /* Remember that this field has been used within the transaction. Later + on we fill mail_cache_field_private.used with it. We can't rely on + setting it here, because cache purging may run and clear it. */ + uint8_t field_idx_set = 1; + array_idx_set(&ctx->cache_field_idx_used, field_idx, &field_idx_set); + + /* Remember that this value exists for the mail, in case we try to look + it up. Note that this gets forgotten whenever changing the mail. */ + buffer_write(ctx->view->cached_exists_buf, field_idx, + &ctx->view->cached_exists_value, 1); + + if (ctx->cache_data->used + full_size > MAIL_CACHE_MAX_WRITE_BUFFER && + ctx->last_rec_pos > 0) { + /* time to flush our buffer. */ + if (MAIL_INDEX_IS_IN_MEMORY(ctx->cache->index)) { + /* just drop the old data to free up memory */ + size_t space_needed = ctx->cache_data->used + + full_size - MAIL_CACHE_MAX_WRITE_BUFFER; + mail_cache_transaction_drop_unwanted(ctx, space_needed); + } else { + if (mail_cache_transaction_flush(ctx, FALSE) < 0) { + /* If this is a syscall failure, the already + flushed changes could still be finished by + writing the offsets to .log file. If this is + a corruption/lost cache, the offsets will + point to a nonexistent file or be ignored. + Either way, we don't really need to handle + this failure in any special way. */ + } + /* Regardless of whether the flush succeeded, drop all + data that it would have written. This way the flush + is attempted only once, but it could still be + possible to write new data later. Also don't reset + the transaction entirely so that the last partially + cached mail can still be accessed from memory. */ + mail_cache_transaction_drop_last_flush(ctx); + } + } + + buffer_append(ctx->cache_data, &field_idx, sizeof(field_idx)); + if (fixed_size == UINT_MAX) { + buffer_append(ctx->cache_data, &data_size32, + sizeof(data_size32)); + } + + buffer_append(ctx->cache_data, data, data_size); + if ((data_size & 3) != 0) + buffer_append_zero(ctx->cache_data, 4 - (data_size & 3)); +} + +bool mail_cache_field_want_add(struct mail_cache_transaction_ctx *ctx, + uint32_t seq, unsigned int field_idx) +{ + enum mail_cache_decision_type decision; + + mail_cache_transaction_refresh_decisions(ctx); + + decision = mail_cache_field_get_decision(ctx->view->cache, field_idx); + decision &= ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + switch (decision) { + case MAIL_CACHE_DECISION_NO: + return FALSE; + case MAIL_CACHE_DECISION_TEMP: + /* add it only if it's newer than what we would drop when + purging */ + if (ctx->first_new_seq == 0) { + ctx->first_new_seq = + mail_cache_get_first_new_seq(ctx->view->view); + } + if (seq < ctx->first_new_seq) + return FALSE; + break; + default: + break; + } + + return mail_cache_field_exists(ctx->view, seq, field_idx) == 0; +} + +bool mail_cache_field_can_add(struct mail_cache_transaction_ctx *ctx, + uint32_t seq, unsigned int field_idx) +{ + enum mail_cache_decision_type decision; + + mail_cache_transaction_refresh_decisions(ctx); + + decision = mail_cache_field_get_decision(ctx->view->cache, field_idx); + if (decision == (MAIL_CACHE_DECISION_FORCED | MAIL_CACHE_DECISION_NO)) + return FALSE; + + return mail_cache_field_exists(ctx->view, seq, field_idx) == 0; +} + +void mail_cache_close_mail(struct mail_cache_transaction_ctx *ctx, + uint32_t seq) +{ + if (array_is_created(&ctx->cache_data_wanted_seqs)) + seq_range_array_remove(&ctx->cache_data_wanted_seqs, seq); +} diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c new file mode 100644 index 0000000..bd3c939 --- /dev/null +++ b/src/lib-index/mail-cache.c @@ -0,0 +1,1005 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "hash.h" +#include "llist.h" +#include "nfs-workarounds.h" +#include "file-cache.h" +#include "mmap-util.h" +#include "read-full.h" +#include "write-full.h" +#include "mail-cache-private.h" +#include "ioloop.h" + +#include <unistd.h> + +#define MAIL_CACHE_MIN_HEADER_READ_SIZE 4096 + +static struct event_category event_category_mail_cache = { + .name = "mail-cache", +}; + +void mail_cache_set_syscall_error(struct mail_cache *cache, + const char *function) +{ + mail_index_file_set_syscall_error(cache->index, cache->filepath, + function); +} + +static void mail_cache_unlink(struct mail_cache *cache) +{ + if (!cache->index->readonly && !MAIL_INDEX_IS_IN_MEMORY(cache->index)) + i_unlink_if_exists(cache->filepath); + /* mark the cache as unusable */ + cache->hdr = NULL; +} + +void mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...) +{ + va_list va; + + mail_cache_unlink(cache); + + va_start(va, fmt); + T_BEGIN { + const char *reason = t_strdup_vprintf(fmt, va); + const char *errstr = t_strdup_printf( + "Deleting corrupted cache: %s", reason); + e_error(event_create_passthrough(cache->event)-> + set_name("mail_cache_corrupted")-> + add_str("reason", reason)->event(), "%s", errstr); + mail_index_set_error_nolog(cache->index, errstr); + } T_END; + va_end(va); +} + +void mail_cache_set_seq_corrupted_reason(struct mail_cache_view *cache_view, + uint32_t seq, const char *reason) +{ + uint32_t uid, empty = 0; + struct mail_cache *cache = cache_view->cache; + struct mail_index_view *view = cache_view->view; + + /* drop cache pointer */ + struct mail_index_transaction *t = + mail_index_transaction_begin(view, MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_ext(t, seq, cache->ext_id, &empty, NULL); + + if (mail_index_transaction_commit(&t) < 0) { + /* I/O error (e.g. out of disk space). Ignore this for now, + maybe it works again later. */ + return; + } + + mail_index_lookup_uid(cache_view->view, seq, &uid); + const char *errstr = t_strdup_printf( + "Deleting corrupted cache record uid=%u: %s", uid, reason); + e_error(event_create_passthrough(cache->event)-> + set_name("mail_cache_record_corrupted")-> + add_int("uid", uid)-> + add_str("reason", reason)->event(), "%s", errstr); + mail_cache_expunge_count(cache, 1); +} + +void mail_cache_file_close(struct mail_cache *cache) +{ + if (cache->mmap_base != NULL) { + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } + + if (cache->file_cache != NULL) + file_cache_set_fd(cache->file_cache, -1); + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + + cache->mmap_base = NULL; + cache->hdr = NULL; + cache->mmap_length = 0; + cache->last_field_header_offset = 0; + + file_lock_free(&cache->file_lock); + cache->locked = FALSE; + + if (cache->fd != -1) { + if (close(cache->fd) < 0) + mail_cache_set_syscall_error(cache, "close()"); + cache->fd = -1; + } + cache->opened = FALSE; +} + +static void mail_cache_init_file_cache(struct mail_cache *cache) +{ + struct stat st; + + if (cache->file_cache != NULL) + file_cache_set_fd(cache->file_cache, cache->fd); + + if (fstat(cache->fd, &st) == 0) { + if (cache->file_cache != NULL) + (void)file_cache_set_size(cache->file_cache, st.st_size); + } else if (!ESTALE_FSTAT(errno)) { + mail_cache_set_syscall_error(cache, "fstat()"); + } + + cache->last_stat_size = st.st_size; + cache->st_ino = st.st_ino; + cache->st_dev = st.st_dev; +} + +static int mail_cache_try_open(struct mail_cache *cache) +{ + int ret; + + i_assert(!cache->opened); + cache->opened = TRUE; + + if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) + return 0; + + i_assert(cache->fd == -1); + cache->fd = nfs_safe_open(cache->filepath, + cache->index->readonly ? O_RDONLY : O_RDWR); + if (cache->fd == -1) { + mail_cache_file_close(cache); + if (errno == ENOENT) { + mail_cache_purge_later_reset(cache); + return 0; + } + + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + mail_cache_init_file_cache(cache); + + if ((ret = mail_cache_map_all(cache)) <= 0) { + mail_cache_file_close(cache); + return ret; + } + return 1; +} + +static bool mail_cache_need_reopen(struct mail_cache *cache) +{ + struct stat st; + + if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) { + /* disabled */ + return FALSE; + } + + if (cache->fd == -1) + return TRUE; + + /* see if the file has changed */ + if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) { + i_assert(!cache->locked); + nfs_flush_file_handle_cache(cache->filepath); + } + if (nfs_safe_stat(cache->filepath, &st) < 0) { + /* if cache was already marked as corrupted, don't log errors + about nonexistent cache file */ + if (cache->hdr != NULL || errno != ENOENT) + mail_cache_set_syscall_error(cache, "stat()"); + return TRUE; + } + cache->last_stat_size = st.st_size; + + if (st.st_ino != cache->st_ino || + !CMP_DEV_T(st.st_dev, cache->st_dev)) { + /* file changed */ + return TRUE; + } + + if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) { + /* if the old file has been deleted, the new file may have + the same inode as the old one. we'll catch this here by + checking if fstat() fails with ESTALE */ + if (fstat(cache->fd, &st) < 0) { + if (ESTALE_FSTAT(errno)) + return TRUE; + mail_cache_set_syscall_error(cache, "fstat()"); + return FALSE; + } + } + return FALSE; +} + +int mail_cache_reopen(struct mail_cache *cache) +{ + mail_cache_file_close(cache); + return mail_cache_open_and_verify(cache); +} + +static void mail_cache_update_need_purge(struct mail_cache *cache) +{ + const struct mail_index_cache_optimization_settings *set = + &cache->index->optimization_set.cache; + const struct mail_cache_header *hdr = cache->hdr; + struct stat st; + unsigned int msg_count; + unsigned int records_count, cont_percentage, delete_percentage; + const char *want_purge_reason = NULL; + + if (hdr->minor_version == 0) { + /* purge to get ourself into the new header version */ + mail_cache_purge_later(cache, "Minor version too old"); + return; + } + + msg_count = cache->index->map->rec_map->records_count; + if (msg_count == 0) + records_count = 1; + else if (hdr->record_count == 0 || hdr->record_count > msg_count*2) { + /* probably not the real record_count, but hole offset that + Dovecot <=v2.1 versions used to use in this position. + we already checked that minor_version>0, but this could + happen if old Dovecot was used to access mailbox after + it had been updated. */ + records_count = I_MAX(msg_count, 1); + } else { + records_count = hdr->record_count; + } + + cont_percentage = hdr->continued_record_count * 100 / records_count; + if (cont_percentage >= set->purge_continued_percentage) { + /* too many continued rows, purge */ + want_purge_reason = t_strdup_printf( + "Too many continued records (%u/%u)", + hdr->continued_record_count, records_count); + } + + delete_percentage = hdr->deleted_record_count * 100 / + (records_count + hdr->deleted_record_count); + if (delete_percentage >= set->purge_delete_percentage) { + /* too many deleted records, purge */ + want_purge_reason = t_strdup_printf( + "Too many deleted records (%u/%u)", + hdr->deleted_record_count, records_count); + } + + if (want_purge_reason != NULL) { + if (fstat(cache->fd, &st) < 0) { + if (!ESTALE_FSTAT(errno)) + mail_cache_set_syscall_error(cache, "fstat()"); + return; + } + if ((uoff_t)st.st_size >= set->purge_min_size) + mail_cache_purge_later(cache, want_purge_reason); + } + +} + +static bool mail_cache_verify_header(struct mail_cache *cache, + const struct mail_cache_header *hdr) +{ + /* check that the header is still ok */ + if (cache->mmap_length < sizeof(struct mail_cache_header)) { + mail_cache_set_corrupted(cache, "File too small"); + return FALSE; + } + + if (hdr->major_version != MAIL_CACHE_MAJOR_VERSION) { + /* version changed - upgrade silently */ + mail_cache_set_corrupted(cache, "Unsupported major version (%u)", + hdr->major_version); + return FALSE; + } + if (hdr->compat_sizeof_uoff_t != sizeof(uoff_t)) { + /* architecture change - handle silently(?) */ + mail_cache_set_corrupted(cache, "Unsupported uoff_t size (%u)", + hdr->compat_sizeof_uoff_t); + return FALSE; + } + + if (hdr->indexid != cache->index->indexid) { + /* index id changed - handle silently */ + mail_cache_unlink(cache); + return FALSE; + } + if (hdr->file_seq == 0) { + mail_cache_set_corrupted(cache, "file_seq is 0"); + return FALSE; + } + return TRUE; +} + +static int +mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size, + const void *hdr_data, bool copy_hdr, bool *corrupted_r) +{ + const struct mail_cache_header *hdr = hdr_data; + + *corrupted_r = FALSE; + + if (offset == 0) { + /* verify the header validity only with offset=0. this way + we won't waste time re-verifying it all the time */ + if (!mail_cache_verify_header(cache, hdr)) { + if (!MAIL_CACHE_IS_UNUSABLE(cache) && + cache->hdr->file_seq != 0) + mail_cache_purge_later(cache, "Invalid header"); + *corrupted_r = TRUE; + return -1; + } + } + if (hdr_data != NULL) { + if (!copy_hdr) + cache->hdr = hdr; + else { + memcpy(&cache->hdr_ro_copy, hdr, + sizeof(cache->hdr_ro_copy)); + cache->hdr = &cache->hdr_ro_copy; + } + mail_cache_update_need_purge(cache); + } else { + i_assert(cache->hdr != NULL); + } + i_assert(cache->hdr->file_seq != 0); + + if (offset + size > cache->mmap_length) + return 0; + return 1; +} + +static int +mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r, bool *corrupted_r) +{ + const void *hdr_data; + void *data; + ssize_t ret; + + if (cache->read_buf == NULL) { + cache->read_buf = + buffer_create_dynamic(default_pool, size); + } else if (cache->read_offset <= offset && + cache->read_offset + cache->read_buf->used >= offset+size) { + /* already mapped */ + *data_r = CONST_PTR_OFFSET(cache->read_buf->data, + offset - cache->read_offset); + hdr_data = offset == 0 ? *data_r : NULL; + return mail_cache_map_finish(cache, offset, size, hdr_data, + TRUE, corrupted_r); + } else { + buffer_set_used_size(cache->read_buf, 0); + } + if (offset == 0 && size < MAIL_CACHE_MIN_HEADER_READ_SIZE) { + /* we can usually read the fields header after the cache + header. we need them both, so try to read them all with one + pread() call. */ + size = MAIL_CACHE_MIN_HEADER_READ_SIZE; + } + + data = buffer_append_space_unsafe(cache->read_buf, size); + ret = pread(cache->fd, data, size, offset); + if (ret < 0) { + if (errno != ESTALE) + mail_cache_set_syscall_error(cache, "read()"); + + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + return -1; + } + buffer_set_used_size(cache->read_buf, ret); + + cache->read_offset = offset; + cache->mmap_length = offset + cache->read_buf->used; + + *data_r = data; + hdr_data = offset == 0 ? *data_r : NULL; + return mail_cache_map_finish(cache, offset, + cache->read_buf->used, hdr_data, + TRUE, corrupted_r); +} + +static int +mail_cache_map_full(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r, bool *corrupted_r) +{ + struct stat st; + const void *data; + ssize_t ret; + size_t orig_size = size; + + *corrupted_r = FALSE; + + if (size == 0) + size = sizeof(struct mail_cache_header); + + /* verify offset + size before trying to allocate a huge amount of + memory due to them. note that we may be prefetching more than we + actually need, so don't fail too early. */ + if ((size > cache->mmap_length || offset + size > cache->mmap_length) && + (offset > 0 || size > sizeof(struct mail_cache_header))) { + if (fstat(cache->fd, &st) < 0) { + e_error(cache->index->event, + "fstat(%s) failed: %m", cache->filepath); + return -1; + } + cache->last_stat_size = st.st_size; + if (offset >= (uoff_t)st.st_size) { + *data_r = NULL; + return 0; + } + if (size > (uoff_t)st.st_size - offset) + size = st.st_size - offset; + } + + cache->remap_counter++; + if (cache->map_with_read) + return mail_cache_map_with_read(cache, offset, size, data_r, + corrupted_r); + + if (cache->file_cache != NULL) { + ret = file_cache_read(cache->file_cache, offset, size); + if (ret < 0) { + /* In case of ESTALE we'll simply fail without error + messages. The caller will then just have to + fallback to generating the value itself. + + We can't simply reopen the cache file, because + using it requires also having updated file + offsets. */ + if (errno != ESTALE) + mail_cache_set_syscall_error(cache, "read()"); + cache->hdr = NULL; + return -1; + } + + data = file_cache_get_map(cache->file_cache, + &cache->mmap_length); + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(data, offset); + return mail_cache_map_finish(cache, offset, size, + offset == 0 ? data : NULL, TRUE, + corrupted_r); + } + + if (offset < cache->mmap_length && + size <= cache->mmap_length - offset) { + /* already mapped */ + i_assert(cache->mmap_base != NULL); + *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); + if (orig_size > cache->mmap_length - offset) { + /* requested offset/size points outside file */ + return 0; + } + return 1; + } + + if (cache->mmap_base != NULL) { + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } else { + if (cache->fd == -1) { + /* unusable, waiting for purging or + index is in memory */ + i_assert(cache->need_purge_file_seq != 0 || + MAIL_INDEX_IS_IN_MEMORY(cache->index)); + return -1; + } + } + + /* map the whole file */ + cache->hdr = NULL; + cache->mmap_length = 0; + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + + cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length); + if (cache->mmap_base == MAP_FAILED) { + cache->mmap_base = NULL; + if (ioloop_time != cache->last_mmap_error_time) { + cache->last_mmap_error_time = ioloop_time; + mail_cache_set_syscall_error(cache, t_strdup_printf( + "mmap(size=%zu)", cache->mmap_length)); + } + cache->mmap_length = 0; + return -1; + } + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(cache->mmap_base, offset); + return mail_cache_map_finish(cache, offset, orig_size, + cache->mmap_base, FALSE, corrupted_r); +} + +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r) +{ + i_assert(offset != 0); + + bool corrupted; + int ret = mail_cache_map_full(cache, offset, size, data_r, &corrupted); + i_assert(!corrupted); + return ret; +} + +int mail_cache_map_all(struct mail_cache *cache) +{ + const void *data; + bool corrupted; + + int ret = mail_cache_map_full(cache, 0, 0, &data, &corrupted); + i_assert(ret != 0); + if (corrupted) { + i_assert(ret == -1); + return 0; + } + return ret < 0 ? -1 : 1; +} + +int mail_cache_open_and_verify(struct mail_cache *cache) +{ + int ret; + + if (cache->opened) { + if (!MAIL_CACHE_IS_UNUSABLE(cache)) + return 1; + mail_cache_file_close(cache); + } + if ((ret = mail_cache_try_open(cache)) < 0) { + /* I/O error */ + mail_cache_file_close(cache); + return -1; + } + + if (ret > 0) { + if (mail_cache_header_fields_read(cache) < 0) { + /* corrupted */ + ret = 0; + } + } + if (ret == 0) { + /* cache was corrupted and should have been deleted already. */ + mail_cache_file_close(cache); + } + return ret; +} + +struct mail_cache * +mail_cache_open_or_create_path(struct mail_index *index, const char *path) +{ + struct mail_cache *cache; + + cache = i_new(struct mail_cache, 1); + cache->index = index; + cache->fd = -1; + cache->filepath = i_strdup(path); + cache->field_pool = pool_alloconly_create("Cache fields", 2048); + hash_table_create(&cache->field_name_hash, cache->field_pool, 0, + strcase_hash, strcasecmp); + + cache->event = event_create(index->event); + event_add_category(cache->event, &event_category_mail_cache); + + cache->dotlock_settings.use_excl_lock = + (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0; + cache->dotlock_settings.nfs_flush = + (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0; + cache->dotlock_settings.timeout = + I_MIN(MAIL_CACHE_LOCK_TIMEOUT, index->set.max_lock_timeout_secs); + cache->dotlock_settings.stale_timeout = MAIL_CACHE_LOCK_CHANGE_TIMEOUT; + + if (!MAIL_INDEX_IS_IN_MEMORY(index) && + (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0) + cache->file_cache = file_cache_new_path(-1, cache->filepath); + cache->map_with_read = + (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0; + + cache->ext_id = + mail_index_ext_register(index, "cache", 0, + sizeof(uint32_t), sizeof(uint32_t)); + mail_index_register_expunge_handler(index, cache->ext_id, + mail_cache_expunge_handler); + return cache; +} + +struct mail_cache *mail_cache_open_or_create(struct mail_index *index) +{ + const char *path = t_strconcat(index->filepath, + MAIL_CACHE_FILE_SUFFIX, NULL); + return mail_cache_open_or_create_path(index, path); +} + +void mail_cache_free(struct mail_cache **_cache) +{ + struct mail_cache *cache = *_cache; + + *_cache = NULL; + + i_assert(cache->views == NULL); + + if (cache->file_cache != NULL) + file_cache_free(&cache->file_cache); + + mail_index_unregister_expunge_handler(cache->index, cache->ext_id); + mail_cache_file_close(cache); + + buffer_free(&cache->read_buf); + hash_table_destroy(&cache->field_name_hash); + pool_unref(&cache->field_pool); + event_unref(&cache->event); + i_free(cache->need_purge_reason); + i_free(cache->field_file_map); + i_free(cache->file_field_map); + i_free(cache->fields); + i_free(cache->filepath); + i_free(cache); +} + +static int mail_cache_lock_file(struct mail_cache *cache) +{ + unsigned int timeout_secs; + bool nonblock = FALSE; + int ret; + + if (cache->last_lock_failed) { + /* previous locking failed. don't waste time waiting on it + again, just try once to see if it's available now. */ + nonblock = TRUE; + } + + i_assert(cache->file_lock == NULL); + if (cache->index->set.lock_method != FILE_LOCK_METHOD_DOTLOCK) { + timeout_secs = I_MIN(MAIL_CACHE_LOCK_TIMEOUT, + cache->index->set.max_lock_timeout_secs); + + ret = mail_index_lock_fd(cache->index, cache->filepath, + cache->fd, F_WRLCK, + nonblock ? 0 : timeout_secs, + &cache->file_lock); + } else { + struct dotlock *dotlock; + enum dotlock_create_flags flags = + nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0; + + ret = file_dotlock_create(&cache->dotlock_settings, + cache->filepath, flags, &dotlock); + if (ret > 0) + cache->file_lock = file_lock_from_dotlock(&dotlock); + else if (ret < 0) { + mail_cache_set_syscall_error(cache, + "file_dotlock_create()"); + } + } + cache->last_lock_failed = ret <= 0; + + /* don't bother warning if locking failed due to a timeout. since cache + updating isn't all that important we're using a very short timeout + so it can be triggered sometimes on heavy load */ + if (ret <= 0) + return ret; + + mail_index_flush_read_cache(cache->index, cache->filepath, cache->fd, + TRUE); + return 1; +} + +static void mail_cache_unlock_file(struct mail_cache *cache) +{ + if (cache->file_lock != NULL) + file_unlock(&cache->file_lock); +} + +static bool +mail_cache_verify_reset_id(struct mail_cache *cache, uint32_t *reset_id_r) +{ + const struct mail_index_ext *ext; + struct mail_index_view *iview; + uint32_t reset_id; + + iview = mail_index_view_open(cache->index); + ext = mail_index_view_get_ext(iview, cache->ext_id); + reset_id = ext == NULL ? 0 : ext->reset_id; + mail_index_view_close(&iview); + + *reset_id_r = reset_id; + return cache->hdr->file_seq == reset_id; +} + +static int +mail_cache_sync_wait_index(struct mail_cache *cache, uint32_t *reset_id_r) +{ + const char *lock_reason = "cache reset_id sync"; + uint32_t file_seq; + uoff_t file_offset; + bool cache_locked = cache->file_lock != NULL; + int ret; + + if (cache->index->log_sync_locked) + return 0; + + /* Wait for .log file lock, so we can be sure that there is no cache + purging going on. (Because it first recreates the cache file, + unlocks it and only then writes the changes to the index and + releases the .log lock.) To prevent deadlocks, cache file must be + locked after the .log, not before. */ + if (cache_locked) + mail_cache_unlock_file(cache); + if (mail_transaction_log_sync_lock(cache->index->log, lock_reason, + &file_seq, &file_offset) < 0) + return -1; + /* Lock the cache file as well so we'll get a guaranteed result on + whether the reset_id can be synced or if it's already desynced and + the cache just needs to be recreated. */ + ret = -1; + while (mail_cache_lock_file(cache) > 0) { + /* Locked the current fd, but it may have already been + recreated. Reopen and retry if needed. */ + if (!mail_cache_need_reopen(cache)) { + ret = 1; + break; + } + if ((ret = mail_cache_reopen(cache)) <= 0) + break; + } + + if (ret <= 0) + ; + else if (mail_index_refresh(cache->index) < 0) + ret = -1; + else + ret = mail_cache_verify_reset_id(cache, reset_id_r) ? 1 : 0; + mail_transaction_log_sync_unlock(cache->index->log, lock_reason); + if (ret <= 0 || !cache_locked) + mail_cache_unlock_file(cache); + return ret; +} + +int mail_cache_sync_reset_id(struct mail_cache *cache) +{ + uint32_t reset_id; + int ret; + + /* verify that the index reset_id matches the cache's file_seq */ + if (mail_cache_verify_reset_id(cache, &reset_id)) + return 1; + + /* Mismatch. See if we can get it synced. */ + if (cache->index->mapping) { + /* Syncing is already locked, and we're in the middle of + mapping the index. The cache is unusable. */ + i_assert(cache->index->log_sync_locked); + mail_cache_set_corrupted(cache, "reset_id mismatch during sync"); + return 0; + } + + /* See if reset_id changes after refreshing the index. */ + if (mail_index_refresh(cache->index) < 0) + return -1; + if (mail_cache_verify_reset_id(cache, &reset_id)) + return 1; + + /* Use locking to wait for a potential cache purging to finish. + If that didn't work either, the cache is corrupted or lost. */ + ret = mail_cache_sync_wait_index(cache, &reset_id); + if (ret == 0 && cache->fd != -1 && reset_id != 0) { + mail_cache_set_corrupted(cache, + "reset_id mismatch even after locking " + "(file_seq=%u != reset_id=%u)", + cache->hdr == NULL ? 0 : cache->hdr->file_seq, + reset_id); + } + return ret; +} + +int mail_cache_lock(struct mail_cache *cache) +{ + int ret; + + i_assert(!cache->locked); + /* the only reason why we might be in here while mapping the index is + if we're coming from mail_cache_expunge_count() while syncing the + index. */ + i_assert(!cache->index->mapping || cache->index->log_sync_locked); + + if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || + cache->index->readonly) + return 0; + + /* Make sure at least some cache file is opened. Usually it's the + latest one, so delay until it's locked to check whether a newer + cache file exists. */ + if ((ret = mail_cache_open_and_verify(cache)) < 0) + return -1; + if (ret == 0) { + /* Cache doesn't exist or it was just found to be corrupted and + was unlinked. Cache purging will create it back. */ + return 0; + } + + for (;;) { + if (mail_cache_lock_file(cache) <= 0) + return -1; + if (!mail_cache_need_reopen(cache)) { + /* locked the latest file */ + break; + } + if ((ret = mail_cache_reopen(cache)) <= 0) { + i_assert(cache->file_lock == NULL); + return ret; + } + i_assert(cache->file_lock == NULL); + /* okay, so it was just purged. try again. */ + } + + if ((ret = mail_cache_sync_reset_id(cache)) <= 0) { + mail_cache_unlock_file(cache); + return ret; + } + i_assert(cache->file_lock != NULL); + + /* successfully locked - make sure our header is up to date */ + cache->locked = TRUE; + cache->hdr_modified = FALSE; + + if (cache->file_cache != NULL) { + file_cache_invalidate(cache->file_cache, 0, + sizeof(struct mail_cache_header)); + } + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + if ((ret = mail_cache_map_all(cache)) <= 0) { + mail_cache_unlock(cache); + return ret; + } + cache->hdr_copy = *cache->hdr; + return 1; +} + +int mail_cache_flush_and_unlock(struct mail_cache *cache) +{ + int ret = 0; + + i_assert(cache->locked); + + if (cache->field_header_write_pending) + ret = mail_cache_header_fields_update(cache); + + /* Cache may become unusable during for various reasons, e.g. + mail_cache_map(). Also the above mail_cache_header_fields_update() + call can make it unusable, so check this after it. */ + if (MAIL_CACHE_IS_UNUSABLE(cache)) { + mail_cache_unlock(cache); + return -1; + } + + if (cache->hdr_modified) { + cache->hdr_modified = FALSE; + if (mail_cache_write(cache, &cache->hdr_copy, + sizeof(cache->hdr_copy), 0) < 0) + ret = -1; + cache->hdr_ro_copy = cache->hdr_copy; + mail_cache_update_need_purge(cache); + } + + mail_cache_unlock(cache); + return ret; +} + +void mail_cache_unlock(struct mail_cache *cache) +{ + i_assert(cache->locked); + + if (MAIL_CACHE_IS_UNUSABLE(cache)) { + /* we found it to be broken during the lock. just clean up. */ + cache->hdr_modified = FALSE; + } else if (cache->index->set.fsync_mode == FSYNC_MODE_ALWAYS) { + if (fdatasync(cache->fd) < 0) + mail_cache_set_syscall_error(cache, "fdatasync()"); + } + + cache->locked = FALSE; + mail_cache_unlock_file(cache); +} + +int mail_cache_write(struct mail_cache *cache, const void *data, size_t size, + uoff_t offset) +{ + i_assert(cache->locked); + + if (pwrite_full(cache->fd, data, size, offset) < 0) { + mail_cache_set_syscall_error(cache, "pwrite_full()"); + return -1; + } + + if (cache->file_cache != NULL) + file_cache_write(cache->file_cache, data, size, offset); + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + return 0; +} + +int mail_cache_append(struct mail_cache *cache, const void *data, size_t size, + uint32_t *offset) +{ + struct stat st; + + if (*offset == 0) { + if (fstat(cache->fd, &st) < 0) { + if (!ESTALE_FSTAT(errno)) + mail_cache_set_syscall_error(cache, "fstat()"); + return -1; + } + cache->last_stat_size = st.st_size; + if ((uoff_t)st.st_size > cache->index->optimization_set.cache.max_size) { + mail_cache_set_corrupted(cache, "Cache file too large"); + return -1; + } + *offset = st.st_size; + } + if (*offset >= cache->index->optimization_set.cache.max_size || + cache->index->optimization_set.cache.max_size - *offset < size) { + mail_cache_set_corrupted(cache, "Cache file too large"); + return -1; + } + if (mail_cache_write(cache, data, size, *offset) < 0) + return -1; + return 0; +} + +bool mail_cache_exists(struct mail_cache *cache) +{ + return !MAIL_CACHE_IS_UNUSABLE(cache); +} + +struct mail_cache_view * +mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview) +{ + struct mail_cache_view *view; + + view = i_new(struct mail_cache_view, 1); + view->cache = cache; + view->view = iview; + view->cached_exists_buf = + buffer_create_dynamic(default_pool, + cache->file_fields_count + 10); + DLLIST_PREPEND(&cache->views, view); + return view; +} + +void mail_cache_view_close(struct mail_cache_view **_view) +{ + struct mail_cache_view *view = *_view; + + i_assert(view->trans_view == NULL); + + *_view = NULL; + if (view->cache->field_header_write_pending && + !view->cache->purging) + (void)mail_cache_header_fields_update(view->cache); + + DLLIST_REMOVE(&view->cache->views, view); + buffer_free(&view->cached_exists_buf); + i_free(view); +} + +void mail_cache_view_update_cache_decisions(struct mail_cache_view *view, + bool update) +{ + view->no_decision_updates = !update; +} + +uint32_t mail_cache_get_first_new_seq(struct mail_index_view *view) +{ + const struct mail_index_header *idx_hdr; + uint32_t first_new_seq, message_count; + + idx_hdr = mail_index_get_header(view); + if (idx_hdr->day_first_uid[7] == 0) + return 1; + + if (!mail_index_lookup_seq_range(view, idx_hdr->day_first_uid[7], + (uint32_t)-1, &first_new_seq, + &message_count)) { + /* all messages are too old */ + return idx_hdr->messages_count+1; + } + return first_new_seq; +} diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h new file mode 100644 index 0000000..09fde29 --- /dev/null +++ b/src/lib-index/mail-cache.h @@ -0,0 +1,193 @@ +#ifndef MAIL_CACHE_H +#define MAIL_CACHE_H + +#include "mail-index.h" + +#define MAIL_CACHE_FILE_SUFFIX ".cache" + +struct mail_cache; +struct mail_cache_view; +struct mail_cache_transaction_ctx; + +enum mail_cache_decision_type { + /* Not needed currently */ + MAIL_CACHE_DECISION_NO = 0x00, + /* Needed only for new mails. Drop when purging. */ + MAIL_CACHE_DECISION_TEMP = 0x01, + /* Needed. */ + MAIL_CACHE_DECISION_YES = 0x02, + + /* This decision has been forced manually, don't change it. */ + MAIL_CACHE_DECISION_FORCED = 0x80 +}; + +enum mail_cache_field_type { + /* Fixed size cache field. The size is specified only in the cache + field header, not separately for each record. */ + MAIL_CACHE_FIELD_FIXED_SIZE, + /* Variable sized binary data. */ + MAIL_CACHE_FIELD_VARIABLE_SIZE, + /* Variable sized string. There is no difference internally to how + MAIL_CACHE_FIELD_VARIABLE_SIZE is handled, but it helps at least + "doveadm dump" to know whether to hex-encode the output. */ + MAIL_CACHE_FIELD_STRING, + /* A fixed size bitmask field. It's possible to add new bits by + updating this field. All the added fields are ORed together. */ + MAIL_CACHE_FIELD_BITMASK, + /* Variable sized message header. The data begins with a 0-terminated + uint32_t line_numbers[]. The line number exists only for each + header, header continuation lines in multiline headers don't get + listed. After the line numbers comes the list of headers, including + the "header-name: " prefix for each line, LFs and the TABs or spaces + for continued lines. */ + MAIL_CACHE_FIELD_HEADER, + + MAIL_CACHE_FIELD_COUNT +}; + +struct mail_cache_field { + /* Unique name for the cache field. The field name doesn't matter + internally. */ + const char *name; + /* Field index name. Used to optimize accessing the cache field. */ + unsigned int idx; + + /* Type of the field */ + enum mail_cache_field_type type; + /* Size of the field, if it's a fixed size type. */ + unsigned int field_size; + /* Current caching decision */ + enum mail_cache_decision_type decision; + /* Timestamp when the cache field was last intentionally read (e.g. + by an IMAP client). Saving new mails doesn't update this field. + This is used to track when an unaccessed field should be dropped. */ + time_t last_used; +}; + +struct mail_cache *mail_cache_open_or_create(struct mail_index *index); +struct mail_cache * +mail_cache_open_or_create_path(struct mail_index *index, const char *path); +void mail_cache_free(struct mail_cache **cache); + +/* Register fields. fields[].idx is updated to contain field index. + If field already exists and its caching decision is NO, the decision is + updated to the input field's decision. */ +void mail_cache_register_fields(struct mail_cache *cache, + struct mail_cache_field *fields, + unsigned int fields_count); +/* Returns registered field index, or UINT_MAX if not found. */ +unsigned int +mail_cache_register_lookup(struct mail_cache *cache, const char *name); +/* Returns specified field */ +const struct mail_cache_field * +mail_cache_register_get_field(struct mail_cache *cache, unsigned int field_idx); +/* Returns a list of all registered fields */ +struct mail_cache_field * +mail_cache_register_get_list(struct mail_cache *cache, pool_t pool, + unsigned int *count_r); + +/* Returns TRUE if cache should be purged. */ +bool mail_cache_need_purge(struct mail_cache *cache, const char **reason_r); +/* Set cache file to be purged later. */ +void mail_cache_purge_later(struct mail_cache *cache, const char *reason); +/* Don't try to purge the cache file later after all. */ +void mail_cache_purge_later_reset(struct mail_cache *cache); +/* Purge cache file. Offsets are updated to given transaction. + The transaction log must already be exclusively locked. + + The cache purging is done only if the current cache file's file_seq + matches purge_file_seq. The idea is that purging isn't done if + another process had just purged it. 0 means the cache file is created + only if it didn't already exist. (uint32_t)-1 means that purging is + done always regardless of file_seq. */ +int mail_cache_purge_with_trans(struct mail_cache *cache, + struct mail_index_transaction *trans, + uint32_t purge_file_seq, const char *reason); +int mail_cache_purge(struct mail_cache *cache, uint32_t purge_file_seq, + const char *reason); +/* Returns TRUE if there is at least something in the cache. */ +bool mail_cache_exists(struct mail_cache *cache); +/* Open and read cache header. Returns 1 if ok, 0 if cache doesn't exist or it + was corrupted and just got deleted, -1 if I/O error. */ +int mail_cache_open_and_verify(struct mail_cache *cache); + +struct mail_cache_view * +mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview); +void mail_cache_view_close(struct mail_cache_view **view); + +/* Normally cache decisions are updated on lookup/add. Use this function to + enable/disable this (useful for precaching data). */ +void mail_cache_view_update_cache_decisions(struct mail_cache_view *view, + bool update); + +/* Copy caching decisions. This is expected to be called only for a newly + created empty mailbox. */ +int mail_cache_decisions_copy(struct mail_cache *src, struct mail_cache *dst); + +/* Get index transaction specific cache transaction. */ +struct mail_cache_transaction_ctx * +mail_cache_get_transaction(struct mail_cache_view *view, + struct mail_index_transaction *t); + +void mail_cache_transaction_reset(struct mail_cache_transaction_ctx *ctx); +int mail_cache_transaction_commit(struct mail_cache_transaction_ctx **ctx); +void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx **ctx); + +/* Add new field to given record. Updates are not allowed. Fixed size fields + must be exactly the expected size. */ +void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, + unsigned int field_idx, const void *data, size_t data_size); +/* Returns TRUE if field is wanted to be added and it doesn't already exist. + If current caching decisions say not to cache this field, FALSE is returned. + If seq is 0, the existence isn't checked. */ +bool mail_cache_field_want_add(struct mail_cache_transaction_ctx *ctx, + uint32_t seq, unsigned int field_idx); +/* Like mail_cache_field_want_add(), but in caching decisions FALSE is + returned only if the decision is a forced no. */ +bool mail_cache_field_can_add(struct mail_cache_transaction_ctx *ctx, + uint32_t seq, unsigned int field_idx); +/* Notify cache that the mail is now closed. Any records added with + mail_cache_add() are unlikely to be required again. This mainly tells + INDEX=MEMORY that it can free up the memory used by the mail. */ +void mail_cache_close_mail(struct mail_cache_transaction_ctx *ctx, + uint32_t seq); + +/* Returns 1 if field exists, 0 if not, -1 if error. */ +int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq, + unsigned int field_idx); +/* Returns TRUE if something is cached for the message, FALSE if not. */ +bool mail_cache_field_exists_any(struct mail_cache_view *view, uint32_t seq); +/* Returns current caching decision for given field. */ +enum mail_cache_decision_type +mail_cache_field_get_decision(struct mail_cache *cache, unsigned int field_idx); +/* Notify the decision handling code when field is committed to cache. + If this is the first time the field is added to cache, its caching decision + is updated to TEMP. */ +void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq, + unsigned int field); + +/* Set data_r and size_r to point to wanted field in cache file. + Returns 1 if field was found, 0 if not, -1 if error. */ +int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf, + uint32_t seq, unsigned int field_idx); + +/* Return specified cached headers. Returns 1 if all fields were found, + 0 if not, -1 if error. dest is updated only if all fields were found. */ +int mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest, + uint32_t seq, const unsigned int field_idxs[], + unsigned int fields_count); + +/* "Error in index cache file %s: ...". */ +void mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...) + ATTR_FORMAT(2, 3) ATTR_COLD; +void mail_cache_set_seq_corrupted_reason(struct mail_cache_view *cache_view, + uint32_t seq, const char *reason) + ATTR_COLD; + +/* Returns human-readable reason for why a cached field is missing for + the specified mail. This is mainly for debugging purposes, so the exact + field doesn't matter here. */ +const char * +mail_cache_get_missing_reason(struct mail_cache_view *view, uint32_t seq); + +#endif diff --git a/src/lib-index/mail-index-alloc-cache.c b/src/lib-index/mail-index-alloc-cache.c new file mode 100644 index 0000000..fe52754 --- /dev/null +++ b/src/lib-index/mail-index-alloc-cache.c @@ -0,0 +1,315 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "module-context.h" +#include "eacces-error.h" +#include "mail-index-private.h" +#include "mail-index-alloc-cache.h" + +#define MAIL_INDEX_ALLOC_CACHE_CONTEXT(obj) \ + MODULE_CONTEXT(obj, mail_index_alloc_cache_index_module) + +/* How many seconds to keep index opened for reuse after it's been closed */ +#define INDEX_CACHE_TIMEOUT 10 +/* How many closed indexes to keep */ +#define INDEX_CACHE_MAX 3 + +struct mail_index_alloc_cache_list { + union mail_index_module_context module_ctx; + struct mail_index_alloc_cache_list *next; + + struct mail_index *index; + char *mailbox_path; + int refcount; + bool referenced; + + dev_t index_dir_dev; + ino_t index_dir_ino; + + time_t destroy_time; +}; + +static MODULE_CONTEXT_DEFINE_INIT(mail_index_alloc_cache_index_module, + &mail_index_module_register); +static struct mail_index_alloc_cache_list *indexes = NULL; +static unsigned int indexes_cache_references_count = 0; +static struct timeout *to_index = NULL; + +static struct mail_index_alloc_cache_list * +mail_index_alloc_cache_add(struct mail_index *index, + const char *mailbox_path, struct stat *st) +{ + struct mail_index_alloc_cache_list *list; + + list = i_new(struct mail_index_alloc_cache_list, 1); + list->refcount = 1; + list->index = index; + + list->mailbox_path = i_strdup(mailbox_path); + list->index_dir_dev = st->st_dev; + list->index_dir_ino = st->st_ino; + + list->next = indexes; + indexes = list; + + MODULE_CONTEXT_SET(index, mail_index_alloc_cache_index_module, list); + return list; +} + +static void +mail_index_alloc_cache_list_unref(struct mail_index_alloc_cache_list *list) +{ + i_assert(list->referenced); + i_assert(indexes_cache_references_count > 0); + + indexes_cache_references_count--; + mail_index_close(list->index); + list->referenced = FALSE; +} + +static void +mail_index_alloc_cache_list_free(struct mail_index_alloc_cache_list *list) +{ + i_assert(list->refcount == 0); + + if (list->referenced) + mail_index_alloc_cache_list_unref(list); + mail_index_free(&list->index); + i_free(list->mailbox_path); + i_free(list); +} + +static struct mail_index_alloc_cache_list * +mail_index_alloc_cache_find_and_expire(const char *mailbox_path, + const char *index_dir, + const struct stat *index_st) +{ + struct mail_index_alloc_cache_list **indexp, *rec, *match; + unsigned int destroy_count; + struct stat st; + + destroy_count = 0; match = NULL; + for (indexp = &indexes; *indexp != NULL;) { + rec = *indexp; + + if (match != NULL) { + /* already found the index. we're just going through + the rest of them to drop 0 refcounts */ + } else if (rec->refcount == 0 && rec->index->open_count == 0) { + /* index is already closed. don't even try to + reuse it. */ + } else if (index_dir != NULL && rec->index_dir_ino != 0) { + if (index_st->st_ino == rec->index_dir_ino && + CMP_DEV_T(index_st->st_dev, rec->index_dir_dev)) { + /* make sure the directory still exists. + it might have been renamed and we're trying + to access it via its new path now. */ + if (stat(rec->index->dir, &st) < 0 || + st.st_ino != index_st->st_ino || + !CMP_DEV_T(st.st_dev, index_st->st_dev)) + rec->destroy_time = 0; + else + match = rec; + } + } else if (mailbox_path != NULL && rec->mailbox_path != NULL && + index_dir == NULL && rec->index_dir_ino == 0) { + if (strcmp(mailbox_path, rec->mailbox_path) == 0) + match = rec; + } + + if (rec->refcount == 0 && rec != match) { + if (rec->destroy_time <= ioloop_time || + destroy_count >= INDEX_CACHE_MAX) { + *indexp = rec->next; + mail_index_alloc_cache_list_free(rec); + continue; + } else { + destroy_count++; + } + } + + indexp = &(*indexp)->next; + } + return match; +} + +struct mail_index * +mail_index_alloc_cache_get(struct event *parent_event, const char *mailbox_path, + const char *index_dir, const char *prefix) +{ + struct mail_index_alloc_cache_list *match; + struct stat st; + + /* compare index_dir inodes so we don't break even with symlinks. + if index_dir doesn't exist yet or if using in-memory indexes, just + compare mailbox paths */ + i_zero(&st); + if (index_dir == NULL) { + /* in-memory indexes */ + } else if (stat(index_dir, &st) < 0) { + if (errno == ENOENT) { + /* it'll be created later */ + } else if (errno == EACCES) { + e_error(parent_event, "%s", + eacces_error_get("stat", index_dir)); + } else { + e_error(parent_event, "stat(%s) failed: %m", index_dir); + } + } + + match = mail_index_alloc_cache_find_and_expire(mailbox_path, + index_dir, &st); + if (match == NULL) { + struct mail_index *index = + mail_index_alloc(parent_event, index_dir, prefix); + match = mail_index_alloc_cache_add(index, mailbox_path, &st); + } else { + match->refcount++; + } + i_assert(match->index != NULL); + return match->index; +} + +struct mail_index * +mail_index_alloc_cache_find(const char *index_dir) +{ + struct mail_index_alloc_cache_list *rec; + struct stat st; + + if (stat(index_dir, &st) < 0) { + if (errno != ENOENT) + i_error("stat(%s) failed: %m", index_dir); + return NULL; + } + + for (rec = indexes; rec != NULL; rec = rec->next) { + if (st.st_ino == rec->index_dir_ino && + CMP_DEV_T(st.st_dev, rec->index_dir_dev)) + return rec->index; + } + return NULL; +} + +static bool destroy_unrefed(unsigned int min_destroy_count) +{ + struct mail_index_alloc_cache_list **list, *rec; + bool destroyed = FALSE; + bool seen_ref0 = FALSE; + + for (list = &indexes; *list != NULL;) { + rec = *list; + + if (rec->refcount == 0 && + (min_destroy_count > 0 || rec->destroy_time <= ioloop_time)) { + *list = rec->next; + destroyed = TRUE; + mail_index_alloc_cache_list_free(rec); + if (min_destroy_count > 0) + min_destroy_count--; + } else { + if (rec->refcount == 0) + seen_ref0 = TRUE; + if (min_destroy_count > 0 && + rec->index->open_count == 1 && + rec->referenced) { + /* we're the only one keeping this index open. + we might be here, because the caller is + deleting this mailbox and wants its indexes + to be closed. so close it. */ + destroyed = TRUE; + mail_index_alloc_cache_list_unref(rec); + } + list = &(*list)->next; + } + } + + if (!seen_ref0 && to_index != NULL) + timeout_remove(&to_index); + return destroyed; +} + +static void ATTR_NULL(1) +index_removal_timeout(void *context ATTR_UNUSED) +{ + destroy_unrefed(0); +} + +void mail_index_alloc_cache_unref(struct mail_index **_index) +{ + struct mail_index *index = *_index; + struct mail_index_alloc_cache_list *list, **listp; + + *_index = NULL; + list = NULL; + for (listp = &indexes; *listp != NULL; listp = &(*listp)->next) { + if ((*listp)->index == index) { + list = *listp; + break; + } + } + + i_assert(list != NULL); + i_assert(list->refcount > 0); + + list->refcount--; + list->destroy_time = ioloop_time + INDEX_CACHE_TIMEOUT; + + if (list->refcount == 0 && index->open_count == 0) { + /* index was already closed. don't even try to cache it. */ + *listp = list->next; + mail_index_alloc_cache_list_free(list); + } else if (to_index == NULL) { + /* Add to root ioloop in case we got here from an inner + ioloop which gets destroyed too early. */ + to_index = timeout_add_to(io_loop_get_root(), + INDEX_CACHE_TIMEOUT*1000/2, + index_removal_timeout, NULL); + } +} + +void mail_index_alloc_cache_destroy_unrefed(void) +{ + destroy_unrefed(UINT_MAX); +} + +void mail_index_alloc_cache_index_opened(struct mail_index *index) +{ + struct mail_index_alloc_cache_list *list = + MAIL_INDEX_ALLOC_CACHE_CONTEXT(index); + struct stat st; + + if (list != NULL && list->index_dir_ino == 0 && + !MAIL_INDEX_IS_IN_MEMORY(index)) { + /* newly created index directory. update its stat. */ + if (stat(index->dir, &st) == 0) { + list->index_dir_ino = st.st_ino; + list->index_dir_dev = st.st_dev; + } + } +} + +void mail_index_alloc_cache_index_closing(struct mail_index *index) +{ + struct mail_index_alloc_cache_list *list = + MAIL_INDEX_ALLOC_CACHE_CONTEXT(index); + + i_assert(index->open_count > 0); + if (index->open_count > 1 || list == NULL) + return; + + if (list->referenced) { + /* we're closing our referenced index */ + return; + } + while (indexes_cache_references_count > INDEX_CACHE_MAX) { + if (!destroy_unrefed(1)) { + /* our cache is full already, don't keep more */ + return; + } + } + /* keep the index referenced for caching */ + indexes_cache_references_count++; + list->referenced = TRUE; + index->open_count++; +} diff --git a/src/lib-index/mail-index-alloc-cache.h b/src/lib-index/mail-index-alloc-cache.h new file mode 100644 index 0000000..5f28563 --- /dev/null +++ b/src/lib-index/mail-index-alloc-cache.h @@ -0,0 +1,20 @@ +#ifndef MAIL_INDEX_ALLOC_CACHE_H +#define MAIL_INDEX_ALLOC_CACHE_H + +/* If using in-memory indexes, give index_dir=NULL. */ +struct mail_index * ATTR_NULL(1, 2) +mail_index_alloc_cache_get(struct event *parent_event, const char *mailbox_path, + const char *index_dir, const char *prefix); +void mail_index_alloc_cache_unref(struct mail_index **index); + +/* Find an existing already opened index from a given index directory. */ +struct mail_index * +mail_index_alloc_cache_find(const char *index_dir); + +void mail_index_alloc_cache_destroy_unrefed(void); + +/* internal: */ +void mail_index_alloc_cache_index_opened(struct mail_index *index); +void mail_index_alloc_cache_index_closing(struct mail_index *index); + +#endif diff --git a/src/lib-index/mail-index-dummy-view.c b/src/lib-index/mail-index-dummy-view.c new file mode 100644 index 0000000..ea69377 --- /dev/null +++ b/src/lib-index/mail-index-dummy-view.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index-private.h" +#include "mail-index-view-private.h" + +static void dummy_view_close(struct mail_index_view *view ATTR_UNUSED) +{ + i_assert(view->refcount == 0); + + array_free(&view->module_contexts); + i_free(view); +} + +static uint32_t +dummy_view_get_message_count(struct mail_index_view *view ATTR_UNUSED) +{ + return (uint32_t)-3; +} + +static struct mail_index_view_vfuncs dummy_view_vfuncs = { + dummy_view_close, + dummy_view_get_message_count, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL +}; + +struct mail_index_view *mail_index_dummy_view_open(struct mail_index *index) +{ + struct mail_index_view *view; + + view = i_new(struct mail_index_view, 1); + view->refcount = 1; + view->v = dummy_view_vfuncs; + view->index = index; + i_array_init(&view->module_contexts, + I_MIN(5, mail_index_module_register.id)); + return view; +} diff --git a/src/lib-index/mail-index-fsck.c b/src/lib-index/mail-index-fsck.c new file mode 100644 index 0000000..6636edf --- /dev/null +++ b/src/lib-index/mail-index-fsck.c @@ -0,0 +1,495 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" + +static void mail_index_fsck_error(struct mail_index *index, + const char *fmt, ...) ATTR_FORMAT(2, 3); +static void mail_index_fsck_error(struct mail_index *index, + const char *fmt, ...) +{ + va_list va; + + va_start(va, fmt); + mail_index_set_error(index, "Fixed index file %s: %s", + index->filepath, t_strdup_vprintf(fmt, va)); + va_end(va); +} + +#define CHECK(field, oper) \ + if (hdr->field oper map->hdr.field) { \ + mail_index_fsck_error(index, #field" %u -> %u", \ + map->hdr.field, hdr->field); \ + } + +static void +mail_index_fsck_log_pos(struct mail_index *index, struct mail_index_map *map, + struct mail_index_header *hdr) +{ + unsigned int hdr_size = index->log->head->hdr.hdr_size; + uint32_t file_seq; + uoff_t file_offset; + + mail_transaction_log_get_head(index->log, &file_seq, &file_offset); + if (hdr->log_file_seq < file_seq) { + /* index's log_file_seq is too old. move it to log head. */ + hdr->log_file_head_offset = hdr->log_file_tail_offset = + sizeof(struct mail_transaction_log_header); + } else if (hdr->log_file_seq == file_seq) { + /* index's log_file_seq matches the current log. make sure the + offsets are valid. */ + if (hdr->log_file_head_offset > file_offset) + hdr->log_file_head_offset = file_offset; + else if (hdr->log_file_head_offset < hdr_size) + hdr->log_file_head_offset = hdr_size; + + if (hdr->log_file_tail_offset > hdr->log_file_head_offset) + hdr->log_file_tail_offset = hdr->log_file_head_offset; + else if (hdr->log_file_tail_offset != 0 && + hdr->log_file_tail_offset < hdr_size) + hdr->log_file_tail_offset = hdr_size; + } else { + /* index's log_file_seq is newer than exists. move it to + end of the current log head. */ + hdr->log_file_head_offset = hdr->log_file_tail_offset = + file_offset; + } + hdr->log_file_seq = file_seq; + + CHECK(log_file_seq, !=); + if (hdr->log_file_seq == map->hdr.log_file_seq) { + /* don't bother complaining about these if file changed too */ + CHECK(log_file_head_offset, !=); + CHECK(log_file_tail_offset, !=); + } +} + +static void +mail_index_fsck_header(struct mail_index *index, struct mail_index_map *map, + struct mail_index_header *hdr) +{ + /* mail_index_map_check_header() has already checked that the index + isn't completely broken. */ + if (hdr->uid_validity == 0 && hdr->next_uid != 1) + hdr->uid_validity = ioloop_time; + + if (index->log->head != NULL) + mail_index_fsck_log_pos(index, map, hdr); +} + +static bool +array_has_name(const ARRAY_TYPE(const_string) *names, const char *name) +{ + const char *arr_name; + + array_foreach_elem(names, arr_name) { + if (strcmp(arr_name, name) == 0) + return TRUE; + } + return FALSE; +} + +static unsigned int +mail_index_fsck_find_keyword_count(struct mail_index_map *map, + const struct mail_index_ext_header *ext_hdr) +{ + const struct mail_index_record *rec; + const uint8_t *kw; + unsigned int r, i, j, cur, max = 0, kw_pos, kw_size; + + kw_pos = ext_hdr->record_offset; + kw_size = ext_hdr->record_size; + + rec = map->rec_map->records; + for (r = 0; r < map->rec_map->records_count; r++) { + kw = CONST_PTR_OFFSET(rec, kw_pos); + for (i = cur = 0; i < kw_size; i++) { + if (kw[i] != 0) { + for (j = 0; j < 8; j++) { + if ((kw[i] & (1 << j)) != 0) + cur = i * 8 + j + 1; + } + } + } + if (cur > max) { + max = cur; + if (max == kw_size*8) + return max; + } + rec = CONST_PTR_OFFSET(rec, map->hdr.record_size); + } + return max; +} + +static bool +keyword_name_is_valid(const char *buffer, unsigned int pos, unsigned int size) +{ + for (; pos < size; pos++) { + if (buffer[pos] == '\0') + return TRUE; + if (((unsigned char)buffer[pos] & 0x7f) < 32) { + /* control characters aren't valid */ + return FALSE; + } + } + return FALSE; +} + +static void +mail_index_fsck_keywords(struct mail_index *index, struct mail_index_map *map, + struct mail_index_header *hdr, + const struct mail_index_ext_header *ext_hdr, + unsigned int ext_offset, unsigned int *offset_p) +{ + const struct mail_index_keyword_header *kw_hdr; + struct mail_index_keyword_header *new_kw_hdr; + const struct mail_index_keyword_header_rec *kw_rec; + struct mail_index_keyword_header_rec new_kw_rec; + const char *name, *name_buffer, **name_array; + unsigned int i, j, name_pos, name_size, rec_pos, hdr_offset, diff; + unsigned int changed_count, keywords_count, name_base_pos; + ARRAY_TYPE(const_string) names; + buffer_t *dest; + bool changed = FALSE; + + hdr_offset = ext_offset + + mail_index_map_ext_hdr_offset(sizeof(MAIL_INDEX_EXT_KEYWORDS)-1); + kw_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, hdr_offset); + keywords_count = kw_hdr->keywords_count; + + kw_rec = (const void *)(kw_hdr + 1); + name_buffer = (const char *)(kw_rec + keywords_count); + + name_pos = (size_t)(name_buffer - (const char *)kw_hdr); + if (name_pos > ext_hdr->hdr_size) { + /* the header is completely broken */ + keywords_count = + mail_index_fsck_find_keyword_count(map, ext_hdr); + mail_index_fsck_error(index, "Assuming keywords_count = %u", + keywords_count); + kw_rec = NULL; + name_size = 0; + changed = TRUE; + } else { + name_size = ext_hdr->hdr_size - name_pos; + } + + /* create keyword name array. invalid keywords are added as + empty strings */ + t_array_init(&names, keywords_count); + for (i = 0; i < keywords_count; i++) { + if (name_size == 0 || + !keyword_name_is_valid(name_buffer, kw_rec[i].name_offset, + name_size)) + name = ""; + else + name = name_buffer + kw_rec[i].name_offset; + + if (*name != '\0' && array_has_name(&names, name)) { + /* duplicate */ + name = ""; + } + array_push_back(&names, &name); + } + + /* give new names to invalid keywords */ + changed_count = 0; + name_array = array_front_modifiable(&names); + for (i = j = 0; i < keywords_count; i++) { + while (name_array[i][0] == '\0') { + name = t_strdup_printf("unknown-%d", j++); + if (!array_has_name(&names, name)) { + name_array[i] = name; + changed = TRUE; + changed_count++; + } + } + } + + if (!changed) { + /* nothing was broken */ + return; + } + + mail_index_fsck_error(index, "Renamed %u keywords to unknown-*", + changed_count); + + dest = buffer_create_dynamic(default_pool, + I_MAX(ext_hdr->hdr_size, 128)); + new_kw_hdr = buffer_append_space_unsafe(dest, sizeof(*new_kw_hdr)); + new_kw_hdr->keywords_count = keywords_count; + + /* add keyword records so we can start appending names directly */ + rec_pos = dest->used; + i_zero(&new_kw_rec); + (void)buffer_append_space_unsafe(dest, keywords_count * sizeof(*kw_rec)); + + /* write the actual records and names */ + name_base_pos = dest->used; + for (i = 0; i < keywords_count; i++) { + new_kw_rec.name_offset = dest->used - name_base_pos; + buffer_write(dest, rec_pos, &new_kw_rec, sizeof(new_kw_rec)); + rec_pos += sizeof(*kw_rec); + + buffer_append(dest, name_array[i], strlen(name_array[i]) + 1); + } + + /* keep the header size at least the same size as before */ + if (dest->used < ext_hdr->hdr_size) + buffer_append_zero(dest, ext_hdr->hdr_size - dest->used); + + if (dest->used > ext_hdr->hdr_size) { + /* need to resize the header */ + struct mail_index_ext_header new_ext_hdr; + + diff = dest->used - ext_hdr->hdr_size; + buffer_copy(map->hdr_copy_buf, hdr_offset + diff, + map->hdr_copy_buf, hdr_offset, SIZE_MAX); + hdr->header_size += diff; + *offset_p += diff; + + new_ext_hdr = *ext_hdr; + new_ext_hdr.hdr_size += diff; + buffer_write(map->hdr_copy_buf, ext_offset, + &new_ext_hdr, sizeof(new_ext_hdr)); + } + + i_assert(hdr_offset + dest->used <= map->hdr_copy_buf->used); + buffer_write(map->hdr_copy_buf, hdr_offset, dest->data, dest->used); + + /* keywords changed unexpectedly, so all views are broken now */ + index->inconsistency_id++; + + buffer_free(&dest); +} + +static void +mail_index_fsck_extensions(struct mail_index *index, struct mail_index_map *map, + struct mail_index_header *hdr) +{ + const struct mail_index_ext_header *ext_hdr; + ARRAY_TYPE(const_string) names; + const char *name, *error; + unsigned int offset, next_offset, i; + + t_array_init(&names, 64); + offset = MAIL_INDEX_HEADER_SIZE_ALIGN(hdr->base_header_size); + for (i = 0; offset < hdr->header_size; i++) { + /* mail_index_map_ext_get_next() uses map->hdr, so make sure + it's up-to-date */ + map->hdr = *hdr; + + next_offset = offset; + if (mail_index_map_ext_get_next(map, &next_offset, + &ext_hdr, &name) < 0) { + /* the extension continued outside header, drop it */ + mail_index_fsck_error(index, + "Dropped extension #%d (%s) " + "with invalid header size", + i, name); + hdr->header_size = offset; + buffer_set_used_size(map->hdr_copy_buf, hdr->header_size); + break; + } + if (mail_index_map_ext_hdr_check(hdr, ext_hdr, name, + &error) < 0) { + mail_index_fsck_error(index, + "Dropped broken extension #%d (%s)", i, name); + } else if (array_has_name(&names, name)) { + mail_index_fsck_error(index, + "Dropped duplicate extension %s", name); + } else { + /* name may change if header buffer is changed */ + name = t_strdup(name); + + if (strcmp(name, MAIL_INDEX_EXT_KEYWORDS) == 0) { + mail_index_fsck_keywords(index, map, hdr, + ext_hdr, offset, + &next_offset); + } + array_push_back(&names, &name); + offset = next_offset; + continue; + } + + /* drop the field */ + hdr->header_size -= next_offset - offset; + buffer_copy(map->hdr_copy_buf, offset, + map->hdr_copy_buf, next_offset, SIZE_MAX); + buffer_set_used_size(map->hdr_copy_buf, hdr->header_size); + } +} + +static void +mail_index_fsck_records(struct mail_index *index, struct mail_index_map *map, + struct mail_index_header *hdr) +{ + struct mail_index_record *rec, *next_rec; + uint32_t i, last_uid; + bool logged_unordered_uids = FALSE, logged_zero_uids = FALSE; + bool records_dropped = FALSE; + + hdr->messages_count = 0; + hdr->seen_messages_count = 0; + hdr->deleted_messages_count = 0; + + hdr->first_unseen_uid_lowwater = 0; + hdr->first_deleted_uid_lowwater = 0; + + rec = map->rec_map->records; last_uid = 0; + for (i = 0; i < map->rec_map->records_count; ) { + next_rec = PTR_OFFSET(rec, hdr->record_size); + if (rec->uid <= last_uid) { + /* log an error once, and skip this record */ + if (rec->uid == 0) { + if (!logged_zero_uids) { + mail_index_fsck_error(index, + "Record UIDs have zeroes"); + logged_zero_uids = TRUE; + } + } else { + if (!logged_unordered_uids) { + mail_index_fsck_error(index, + "Record UIDs unordered"); + logged_unordered_uids = TRUE; + } + } + /* not the fastest way when we're skipping lots of + records, but this should happen rarely so don't + bother optimizing. */ + memmove(rec, next_rec, hdr->record_size * + (map->rec_map->records_count - i - 1)); + map->rec_map->records_count--; + records_dropped = TRUE; + continue; + } + + hdr->messages_count++; + if ((rec->flags & MAIL_SEEN) != 0) + hdr->seen_messages_count++; + if ((rec->flags & MAIL_DELETED) != 0) + hdr->deleted_messages_count++; + + if ((rec->flags & MAIL_SEEN) == 0 && + hdr->first_unseen_uid_lowwater == 0) + hdr->first_unseen_uid_lowwater = rec->uid; + if ((rec->flags & MAIL_DELETED) != 0 && + hdr->first_deleted_uid_lowwater == 0) + hdr->first_deleted_uid_lowwater = rec->uid; + + last_uid = rec->uid; + rec = next_rec; + i++; + } + + if (records_dropped) { + /* all existing views are broken now */ + index->inconsistency_id++; + } + + if (hdr->next_uid <= last_uid) { + mail_index_fsck_error(index, "next_uid %u -> %u", + hdr->next_uid, last_uid+1); + hdr->next_uid = last_uid+1; + } + + if (hdr->first_unseen_uid_lowwater == 0) + hdr->first_unseen_uid_lowwater = hdr->next_uid; + if (hdr->first_deleted_uid_lowwater == 0) + hdr->first_deleted_uid_lowwater = hdr->next_uid; + if (hdr->first_recent_uid > hdr->next_uid) + hdr->first_recent_uid = hdr->next_uid; + if (hdr->first_recent_uid == 0) + hdr->first_recent_uid = 1; + + CHECK(uid_validity, !=); + CHECK(messages_count, !=); + CHECK(seen_messages_count, !=); + CHECK(deleted_messages_count, !=); + + CHECK(first_unseen_uid_lowwater, <); + CHECK(first_deleted_uid_lowwater, <); + CHECK(first_recent_uid, !=); +} + +static void +mail_index_fsck_map(struct mail_index *index, struct mail_index_map *map) +{ + struct mail_index_header hdr; + + if (index->log->head != NULL) { + /* Remember the log head position. If we go back in the index's + head offset, ignore errors in the log up to this offset. */ + mail_transaction_log_get_head(index->log, + &index->fsck_log_head_file_seq, + &index->fsck_log_head_file_offset); + } + hdr = map->hdr; + + mail_index_fsck_header(index, map, &hdr); + mail_index_fsck_extensions(index, map, &hdr); + mail_index_fsck_records(index, map, &hdr); + + hdr.flags |= MAIL_INDEX_HDR_FLAG_FSCKD; + map->hdr = hdr; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); +} + +int mail_index_fsck(struct mail_index *index) +{ + bool orig_locked = index->log_sync_locked; + struct mail_index_map *map; + uint32_t file_seq; + uoff_t file_offset; + + i_warning("fscking index file %s", index->filepath); + + index->fscked = TRUE; + + if (index->log->head == NULL) { + /* we're trying to open the index files, but there wasn't + any .log file. */ + if (mail_transaction_log_create(index->log, FALSE) < 0) + return -1; + } + + if (!orig_locked) { + if (mail_transaction_log_sync_lock(index->log, "fscking", + &file_seq, &file_offset) < 0) + return -1; + } + + map = mail_index_map_clone(index->map); + mail_index_unmap(&index->map); + index->map = map; + + T_BEGIN { + mail_index_fsck_map(index, map); + } T_END; + + mail_index_write(index, FALSE, "fscking"); + + if (!orig_locked) + mail_transaction_log_sync_unlock(index->log, "fscking"); + return 0; +} + +void mail_index_fsck_locked(struct mail_index *index) +{ + int ret; + + i_assert(index->log_sync_locked); + ret = mail_index_fsck(index); + i_assert(ret == 0); +} + +bool mail_index_reset_fscked(struct mail_index *index) +{ + bool ret = index->fscked; + + index->fscked = FALSE; + return ret; +} diff --git a/src/lib-index/mail-index-lock.c b/src/lib-index/mail-index-lock.c new file mode 100644 index 0000000..cdf62e4 --- /dev/null +++ b/src/lib-index/mail-index-lock.c @@ -0,0 +1,63 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +/* + Locking should never fail or timeout. Exclusive locks must be kept as short + time as possible. Shared locks can be long living, so if we can't get + exclusive lock directly, we'll recreate the index. That means the shared + lock holders can keep using the old file. + + lock_id is used to figure out if acquired lock is still valid. When index + file is reopened, the lock_id can become invalid. It doesn't matter however, + as no-one's going to modify the old file anymore. + + lock_id also tells us if we're referring to a shared or an exclusive lock. + This allows us to drop back to shared locking once all exclusive locks + are dropped. Shared locks have even numbers, exclusive locks have odd numbers. + The number is increased by two every time the lock is dropped or index file + is reopened. +*/ + +#include "lib.h" +#include "nfs-workarounds.h" +#include "mail-index-private.h" + +#define MAIL_INDEX_SHARED_LOCK_TIMEOUT 120 + +int mail_index_lock_fd(struct mail_index *index, const char *path, int fd, + int lock_type, unsigned int timeout_secs, + struct file_lock **lock_r) +{ + const char *error; + int ret; + + if (fd == -1) { + i_assert(MAIL_INDEX_IS_IN_MEMORY(index)); + return 1; + } + + struct file_lock_settings lock_set = { + .lock_method = index->set.lock_method, + }; + ret = file_wait_lock(fd, path, lock_type, &lock_set, timeout_secs, + lock_r, &error); + if (ret < 0) + e_error(index->event, "%s", error); + return ret; +} + +void mail_index_flush_read_cache(struct mail_index *index, const char *path, + int fd, bool locked) +{ + if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) == 0) + return; + + /* Assume flock() is emulated with fcntl(), because that's how most + OSes work nowadays. */ + if (locked && + (index->set.lock_method == FILE_LOCK_METHOD_FCNTL || + index->set.lock_method == FILE_LOCK_METHOD_FLOCK)) { + nfs_flush_read_cache_locked(path, fd); + } else { + nfs_flush_read_cache_unlocked(path, fd); + } +} diff --git a/src/lib-index/mail-index-map-hdr.c b/src/lib-index/mail-index-map-hdr.c new file mode 100644 index 0000000..3287a28 --- /dev/null +++ b/src/lib-index/mail-index-map-hdr.c @@ -0,0 +1,359 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index-private.h" + +int mail_index_map_parse_extensions(struct mail_index_map *map) +{ + struct mail_index *index = map->index; + const struct mail_index_ext_header *ext_hdr; + unsigned int i, old_count, offset; + const char *name, *error; + uint32_t ext_id, ext_map_idx, ext_offset; + + /* extension headers always start from 64bit offsets, so if base header + doesn't happen to be 64bit aligned we'll skip some bytes */ + offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size); + if (offset >= map->hdr.header_size && map->extension_pool == NULL) { + /* nothing to do, skip allocations and all */ + return 0; + } + + old_count = array_count(&index->extensions); + mail_index_map_init_extbufs(map, old_count + 5); + + ext_id = (uint32_t)-1; + for (i = 0; i < old_count; i++) + array_push_back(&map->ext_id_map, &ext_id); + + for (i = 0; offset < map->hdr.header_size; i++) { + ext_offset = offset; + + if (mail_index_map_ext_get_next(map, &offset, + &ext_hdr, &name) < 0) { + mail_index_set_error(index, "Corrupted index file %s: " + "Header extension #%d (%s) goes outside header", + index->filepath, i, name); + return -1; + } + + if (mail_index_map_ext_hdr_check(&map->hdr, ext_hdr, + name, &error) < 0) { + mail_index_set_error(index, "Corrupted index file %s: " + "Broken extension #%d (%s): %s", + index->filepath, i, name, error); + return -1; + } + if (mail_index_map_lookup_ext(map, name, &ext_map_idx)) { + mail_index_set_error(index, "Corrupted index file %s: " + "Duplicate header extension %s", + index->filepath, name); + return -1; + } + + (void)mail_index_map_register_ext(map, name, ext_offset, ext_hdr); + } + return 0; +} + +int mail_index_map_parse_keywords(struct mail_index_map *map) +{ + struct mail_index *index = map->index; + const struct mail_index_ext *ext; + const struct mail_index_keyword_header *kw_hdr; + const struct mail_index_keyword_header_rec *kw_rec; + const char *name; + unsigned int i, name_area_end_offset, old_count; + uint32_t idx; + + if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, &idx)) { + if (array_is_created(&map->keyword_idx_map)) + array_clear(&map->keyword_idx_map); + return 0; + } + ext = array_idx(&map->extensions, idx); + + /* Extension header contains: + - struct mail_index_keyword_header + - struct mail_index_keyword_header_rec * keywords_count + - const char names[] * keywords_count + + The mail_index_keyword_header_rec are rather unnecessary nowadays. + They were originally an optimization when dovecot.index header kept + changing constantly, but nowadays the changes are usually read from + the .log changes, so re-reading dovecot.index header isn't common. + In a later version we could even remove it. + */ + i_assert(ext->hdr_offset < map->hdr.header_size); + kw_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset); + kw_rec = (const void *)(kw_hdr + 1); + name = (const char *)(kw_rec + kw_hdr->keywords_count); + + old_count = !array_is_created(&map->keyword_idx_map) ? 0 : + array_count(&map->keyword_idx_map); + + /* make sure the header is valid */ + if (kw_hdr->keywords_count < old_count) { + mail_index_set_error(index, "Corrupted index file %s: " + "Keywords removed unexpectedly", + index->filepath); + return -1; + } + + if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) { + mail_index_set_error(index, "Corrupted index file %s: " + "keywords_count larger than header size", + index->filepath); + return -1; + } + + name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name; + for (i = 0; i < kw_hdr->keywords_count; i++) { + if (kw_rec[i].name_offset > name_area_end_offset) { + mail_index_set_error(index, "Corrupted index file %s: " + "name_offset points outside allocated header", + index->filepath); + return -1; + } + } + if (name[name_area_end_offset-1] != '\0') { + mail_index_set_error(index, "Corrupted index file %s: " + "Keyword header doesn't end with NUL", + index->filepath); + return -1; + } + + /* create file -> index mapping */ + if (!array_is_created(&map->keyword_idx_map)) + i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count); + + size_t name_offset = 0; + /* Check that existing headers are still the same. */ + for (i = 0; i < array_count(&map->keyword_idx_map); i++) { + const char *keyword = name + kw_rec[i].name_offset; + const unsigned int *old_idx; + unsigned int kw_idx; + + if (kw_rec[i].name_offset != name_offset) { + /* this shouldn't happen, but the old code didn't check + for this so for safety keep this as a warning. */ + e_warning(index->event, + "Corrupted index file %s: " + "Mismatching keyword name_offset", + index->filepath); + } + name_offset += strlen(keyword) + 1; + + old_idx = array_idx(&map->keyword_idx_map, i); + if (!mail_index_keyword_lookup(index, keyword, &kw_idx) || + kw_idx != *old_idx) { + mail_index_set_error(index, "Corrupted index file %s: " + "Keywords changed unexpectedly", + index->filepath); + return -1; + } + } + + /* Register the newly seen keywords */ + i = array_count(&map->keyword_idx_map); + for (; i < kw_hdr->keywords_count; i++) { + const char *keyword = name + kw_rec[i].name_offset; + unsigned int kw_idx; + + if (kw_rec[i].name_offset != name_offset) { + /* this shouldn't happen, but the old code didn't check + for this so for safety keep this as a warning. */ + e_warning(index->event, + "Corrupted index file %s: " + "Mismatching keyword name_offset", + index->filepath); + } + name_offset += strlen(keyword) + 1; + + if (*keyword == '\0') { + mail_index_set_error(index, "Corrupted index file %s: " + "Empty keyword name in header", + index->filepath); + return -1; + } + mail_index_keyword_lookup_or_create(index, keyword, &kw_idx); + array_push_back(&map->keyword_idx_map, &kw_idx); + } + return 0; +} + +bool mail_index_check_header_compat(struct mail_index *index, + const struct mail_index_header *hdr, + uoff_t file_size, const char **error_r) +{ + enum mail_index_header_compat_flags compat_flags = 0; + +#ifndef WORDS_BIGENDIAN + compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#endif + + if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) { + /* major version change */ + *error_r = t_strdup_printf("Major version changed (%u != %u)", + hdr->major_version, MAIL_INDEX_MAJOR_VERSION); + return FALSE; + } + if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { + /* we've already complained about it */ + *error_r = "Header's corrupted flag is set"; + return FALSE; + } + + if (hdr->compat_flags != compat_flags) { + /* architecture change */ + *error_r = "CPU architecture changed"; + return FALSE; + } + + if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE || + hdr->header_size < hdr->base_header_size) { + *error_r = t_strdup_printf( + "Corrupted header sizes (base %u, full %u)", + hdr->base_header_size, hdr->header_size); + return FALSE; + } + if (hdr->header_size > file_size) { + *error_r = t_strdup_printf( + "Header size is larger than file (%u > %"PRIuUOFF_T")", + hdr->header_size, file_size); + return FALSE; + } + + if (hdr->indexid != index->indexid) { + if (index->indexid != 0) { + mail_index_set_error(index, "Index file %s: " + "indexid changed: %u -> %u", + index->filepath, index->indexid, + hdr->indexid); + } + index->indexid = hdr->indexid; + mail_transaction_log_indexid_changed(index->log); + } + return TRUE; +} + +static void mail_index_map_clear_recent_flags(struct mail_index_map *map) +{ + struct mail_index_record *rec; + uint32_t seq; + + for (seq = 1; seq <= map->hdr.messages_count; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(map, seq); + rec->flags &= ENUM_NEGATE(MAIL_RECENT); + } +} + +int mail_index_map_check_header(struct mail_index_map *map, + const char **error_r) +{ + struct mail_index *index = map->index; + const struct mail_index_header *hdr = &map->hdr; + + if (!mail_index_check_header_compat(index, hdr, UOFF_T_MAX, error_r)) + return 0; + + /* following some extra checks that only take a bit of CPU */ + if (hdr->record_size < sizeof(struct mail_index_record)) { + *error_r = t_strdup_printf( + "record_size too small (%u < %zu)", + hdr->record_size, sizeof(struct mail_index_record)); + return -1; + } + + if (hdr->uid_validity == 0 && hdr->next_uid != 1) { + *error_r = t_strdup_printf( + "uidvalidity=0, but next_uid=%u", hdr->next_uid); + return 0; + } + if (hdr->next_uid == 0) { + *error_r = "next_uid=0"; + return 0; + } + if (hdr->messages_count > map->rec_map->records_count) { + *error_r = t_strdup_printf( + "messages_count is higher in header than record map (%u > %u)", + hdr->messages_count, map->rec_map->records_count); + return 0; + } + + if (hdr->seen_messages_count > hdr->messages_count) { + *error_r = t_strdup_printf( + "seen_messages_count %u > messages_count %u", + hdr->seen_messages_count, hdr->messages_count); + return 0; + } + if (hdr->deleted_messages_count > hdr->messages_count) { + *error_r = t_strdup_printf( + "deleted_messages_count %u > messages_count %u", + hdr->deleted_messages_count, hdr->messages_count); + return 0; + } + switch (hdr->minor_version) { + case 0: + /* upgrade silently from v1.0 */ + map->hdr.unused_old_recent_messages_count = 0; + if (hdr->first_recent_uid == 0) + map->hdr.first_recent_uid = 1; + if (index->need_recreate == NULL) + index->need_recreate = i_strdup("Upgrading from index version 1.0"); + /* fall through */ + case 1: + /* pre-v1.1.rc6: make sure the \Recent flags are gone */ + mail_index_map_clear_recent_flags(map); + map->hdr.minor_version = MAIL_INDEX_MINOR_VERSION; + /* fall through */ + case 2: + /* pre-v2.2 (although should have been done in v2.1 already): + make sure the old unused fields are cleared */ + map->hdr.unused_old_sync_size_part1 = 0; + map->hdr.log2_rotate_time = 0; + map->hdr.last_temp_file_scan = 0; + } + if (hdr->first_recent_uid == 0) { + *error_r = "first_recent_uid=0"; + return 0; + } + if (hdr->first_recent_uid > hdr->next_uid) { + *error_r = t_strdup_printf( + "first_recent_uid %u > next_uid %u", + hdr->first_recent_uid, hdr->next_uid); + return 0; + } + if (hdr->first_unseen_uid_lowwater > hdr->next_uid) { + *error_r = t_strdup_printf( + "first_unseen_uid_lowwater %u > next_uid %u", + hdr->first_unseen_uid_lowwater, hdr->next_uid); + return 0; + } + if (hdr->first_deleted_uid_lowwater > hdr->next_uid) { + *error_r = t_strdup_printf( + "first_deleted_uid_lowwater %u > next_uid %u", + hdr->first_deleted_uid_lowwater, hdr->next_uid); + return 0; + } + + if (hdr->messages_count > 0) { + /* last message's UID must be smaller than next_uid. + also make sure it's not zero. */ + const struct mail_index_record *rec; + + rec = MAIL_INDEX_REC_AT_SEQ(map, hdr->messages_count); + if (rec->uid == 0) { + *error_r = "last message has uid=0"; + return -1; + } + if (rec->uid >= hdr->next_uid) { + *error_r = t_strdup_printf( + "last message uid %u >= next_uid %u", + rec->uid, hdr->next_uid); + return 0; + } + } + return 1; +} diff --git a/src/lib-index/mail-index-map-read.c b/src/lib-index/mail-index-map-read.c new file mode 100644 index 0000000..6999bf1 --- /dev/null +++ b/src/lib-index/mail-index-map-read.c @@ -0,0 +1,519 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "nfs-workarounds.h" +#include "mmap-util.h" +#include "read-full.h" +#include "mail-index-private.h" +#include "mail-index-sync-private.h" +#include "mail-transaction-log-private.h" +#include "mail-index-modseq.h" +#include "ioloop.h" + +static void mail_index_map_copy_hdr(struct mail_index_map *map, + const struct mail_index_header *hdr) +{ + if (hdr->base_header_size < sizeof(map->hdr)) { + /* header smaller than ours, make a copy so our newer headers + won't have garbage in them */ + i_zero(&map->hdr); + memcpy(&map->hdr, hdr, hdr->base_header_size); + } else { + map->hdr = *hdr; + } + + /* FIXME: backwards compatibility, remove later. In case this index is + accessed with Dovecot v1.0, avoid recent message counter errors. */ + map->hdr.unused_old_recent_messages_count = 0; +} + +static int mail_index_mmap(struct mail_index_map *map, uoff_t file_size) +{ + struct mail_index *index = map->index; + struct mail_index_record_map *rec_map = map->rec_map; + const struct mail_index_header *hdr; + const char *error; + + i_assert(rec_map->mmap_base == NULL); + + buffer_free(&rec_map->buffer); + if (file_size > SSIZE_T_MAX) { + /* too large file to map into memory */ + mail_index_set_error(index, "Index file too large: %s", + index->filepath); + return -1; + } + + rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE, index->fd, 0); + if (rec_map->mmap_base == MAP_FAILED) { + rec_map->mmap_base = NULL; + if (ioloop_time != index->last_mmap_error_time) { + index->last_mmap_error_time = ioloop_time; + mail_index_set_syscall_error(index, t_strdup_printf( + "mmap(size=%"PRIuUOFF_T")", file_size)); + } + return -1; + } + rec_map->mmap_size = file_size; + + hdr = rec_map->mmap_base; + if (rec_map->mmap_size > + offsetof(struct mail_index_header, major_version) && + hdr->major_version != MAIL_INDEX_MAJOR_VERSION) { + /* major version change - handle silently */ + return 0; + } + + if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) { + mail_index_set_error(index, "Corrupted index file %s: " + "File too small (%zu)", + index->filepath, rec_map->mmap_size); + return 0; + } + + if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size, &error)) { + /* Can't use this file */ + mail_index_set_error(index, "Corrupted index file %s: %s", + index->filepath, error); + return 0; + } + + rec_map->mmap_used_size = hdr->header_size + + hdr->messages_count * hdr->record_size; + + if (rec_map->mmap_used_size <= rec_map->mmap_size) + rec_map->records_count = hdr->messages_count; + else { + rec_map->records_count = + (rec_map->mmap_size - hdr->header_size) / + hdr->record_size; + rec_map->mmap_used_size = hdr->header_size + + rec_map->records_count * hdr->record_size; + mail_index_set_error(index, "Corrupted index file %s: " + "messages_count too large (%u > %u)", + index->filepath, hdr->messages_count, + rec_map->records_count); + } + + mail_index_map_copy_hdr(map, hdr); + buffer_set_used_size(map->hdr_copy_buf, 0); + buffer_append(map->hdr_copy_buf, rec_map->mmap_base, hdr->header_size); + + rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size); + return 1; +} + +static int mail_index_read_header(struct mail_index *index, + void *buf, size_t buf_size, size_t *pos_r) +{ + size_t pos; + int ret; + + memset(buf, 0, sizeof(struct mail_index_header)); + + /* try to read the whole header, but it's not necessarily an error to + read less since the older versions of the index format could be + smaller. Request reading up to buf_size, but accept if we only got + the header. */ + pos = 0; + do { + ret = pread(index->fd, PTR_OFFSET(buf, pos), + buf_size - pos, pos); + if (ret > 0) + pos += ret; + } while (ret > 0 && pos < sizeof(struct mail_index_header)); + + *pos_r = pos; + return ret; +} + +static int +mail_index_try_read_map(struct mail_index_map *map, + uoff_t file_size, bool *retry_r, bool try_retry) +{ + struct mail_index *index = map->index; + const struct mail_index_header *hdr; + unsigned char read_buf[IO_BLOCK_SIZE]; + const char *error; + const void *buf; + void *data = NULL; + ssize_t ret; + size_t pos, records_size, initial_buf_pos = 0; + unsigned int records_count = 0, extra; + + i_assert(map->rec_map->mmap_base == NULL); + + *retry_r = FALSE; + ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos); + buf = read_buf; hdr = buf; + + if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) && + hdr->major_version != MAIL_INDEX_MAJOR_VERSION) { + /* major version change - handle silently */ + return 0; + } + + if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE && + (ret > 0 || pos >= hdr->base_header_size)) { + if (!mail_index_check_header_compat(index, hdr, file_size, &error)) { + /* Can't use this file */ + mail_index_set_error(index, "Corrupted index file %s: %s", + index->filepath, error); + return 0; + } + + initial_buf_pos = pos; + if (pos > hdr->header_size) + pos = hdr->header_size; + + /* place the base header into memory. */ + buffer_set_used_size(map->hdr_copy_buf, 0); + buffer_append(map->hdr_copy_buf, buf, pos); + + if (pos != hdr->header_size) { + /* @UNSAFE: read the rest of the header into memory */ + data = buffer_append_space_unsafe(map->hdr_copy_buf, + hdr->header_size - + pos); + ret = pread_full(index->fd, data, + hdr->header_size - pos, pos); + } + } + + if (ret > 0) { + /* header read, read the records now. */ + records_size = (size_t)hdr->messages_count * hdr->record_size; + records_count = hdr->messages_count; + + if (file_size - hdr->header_size < records_size || + (hdr->record_size != 0 && + records_size / hdr->record_size != hdr->messages_count)) { + records_count = (file_size - hdr->header_size) / + hdr->record_size; + records_size = (size_t)records_count * hdr->record_size; + mail_index_set_error(index, "Corrupted index file %s: " + "messages_count too large (%u > %u)", + index->filepath, hdr->messages_count, + records_count); + } + + if (map->rec_map->buffer == NULL) { + map->rec_map->buffer = + buffer_create_dynamic(default_pool, + records_size); + } + + /* @UNSAFE */ + buffer_set_used_size(map->rec_map->buffer, 0); + if (initial_buf_pos <= hdr->header_size) + extra = 0; + else { + extra = initial_buf_pos - hdr->header_size; + buffer_append(map->rec_map->buffer, + CONST_PTR_OFFSET(buf, hdr->header_size), + extra); + } + if (records_size > extra) { + data = buffer_append_space_unsafe(map->rec_map->buffer, + records_size - extra); + ret = pread_full(index->fd, data, records_size - extra, + hdr->header_size + extra); + } + } + + if (ret < 0) { + if (errno == ESTALE && try_retry) { + /* a new index file was renamed over this one. */ + *retry_r = TRUE; + return 0; + } + mail_index_set_syscall_error(index, "pread_full()"); + return -1; + } + if (ret == 0) { + mail_index_set_error(index, + "Corrupted index file %s: File too small", + index->filepath); + return 0; + } + + map->rec_map->records = + buffer_get_modifiable_data(map->rec_map->buffer, NULL); + map->rec_map->records_count = records_count; + + mail_index_map_copy_hdr(map, hdr); + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); + return 1; +} + +static int mail_index_read_map(struct mail_index_map *map, uoff_t file_size) +{ + struct mail_index *index = map->index; + struct stat st; + unsigned int i; + int ret; + bool try_retry, retry; + + for (i = 0;; i++) { + try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT; + if (file_size == UOFF_T_MAX) { + /* fstat() below failed */ + ret = 0; + retry = try_retry; + } else { + ret = mail_index_try_read_map(map, file_size, + &retry, try_retry); + } + if (ret != 0 || !retry) + break; + + /* ESTALE - reopen index file */ + mail_index_close_file(index); + + ret = mail_index_try_open_only(index); + if (ret <= 0) { + if (ret == 0) { + /* the file was lost */ + errno = ENOENT; + mail_index_set_syscall_error(index, "open()"); + } + return -1; + } + + if (fstat(index->fd, &st) == 0) + file_size = st.st_size; + else { + if (!ESTALE_FSTAT(errno)) { + mail_index_set_syscall_error(index, "fstat()"); + return -1; + } + file_size = UOFF_T_MAX; + } + } + return ret; +} + +/* returns -1 = error, 0 = index files are unusable, + 1 = index files are usable or at least repairable */ +static int +mail_index_map_latest_file(struct mail_index *index, const char **reason_r) +{ + struct mail_index_map *old_map, *new_map; + struct stat st; + uoff_t file_size; + bool use_mmap, reopened, unusable = FALSE; + const char *error; + int ret, try; + + *reason_r = NULL; + + index->reopen_main_index = FALSE; + ret = mail_index_reopen_if_changed(index, &reopened, reason_r); + if (ret <= 0) { + if (ret < 0) + return -1; + + /* the index file is lost/broken. let's hope that we can + build it from the transaction log. */ + return 1; + } + i_assert(index->fd != -1); + + if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) + nfs_flush_attr_cache_fd_locked(index->filepath, index->fd); + + if (fstat(index->fd, &st) == 0) + file_size = st.st_size; + else { + if (!ESTALE_FSTAT(errno)) { + mail_index_set_syscall_error(index, "fstat()"); + return -1; + } + file_size = UOFF_T_MAX; + } + + /* mmaping seems to be slower than just reading the file, so even if + mmap isn't disabled don't use it unless the file is large enough */ + use_mmap = (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0 && + file_size != UOFF_T_MAX && file_size > MAIL_INDEX_MMAP_MIN_SIZE; + + new_map = mail_index_map_alloc(index); + if (use_mmap) { + ret = mail_index_mmap(new_map, file_size); + } else { + ret = mail_index_read_map(new_map, file_size); + } + if (ret == 0) { + /* the index files are unusable */ + unusable = TRUE; + } + + for (try = 0; ret > 0; try++) { + /* make sure the header is ok before using this mapping */ + ret = mail_index_map_check_header(new_map, &error); + if (ret < 0) { + mail_index_set_error(index, + "Corrupted index file %s: %s", + index->filepath, error); + } + if (ret > 0) T_BEGIN { + if (mail_index_map_parse_extensions(new_map) < 0) + ret = 0; + else if (mail_index_map_parse_keywords(new_map) < 0) + ret = 0; + } T_END; + if (ret != 0 || try == 2) { + if (ret < 0) { + *reason_r = "Corrupted index file"; + unusable = TRUE; + ret = 0; + } + break; + } + + /* fsck and try again */ + old_map = index->map; + index->map = new_map; + if (mail_index_fsck(index) < 0) { + ret = -1; + break; + } + + /* fsck replaced the map */ + new_map = index->map; + index->map = old_map; + } + if (ret <= 0) { + mail_index_unmap(&new_map); + return ret < 0 ? -1 : (unusable ? 0 : 1); + } + i_assert(new_map->rec_map->records != NULL); + + index->main_index_hdr_log_file_seq = new_map->hdr.log_file_seq; + index->main_index_hdr_log_file_tail_offset = + new_map->hdr.log_file_tail_offset; + mail_index_modseq_hdr_snapshot_update(new_map); + + mail_index_unmap(&index->map); + index->map = new_map; + *reason_r = t_strdup_printf("Index mapped (file_seq=%u)", + index->map->hdr.log_file_seq); + return 1; +} + +static int +mail_index_map_latest_sync(struct mail_index *index, + enum mail_index_sync_handler_type type, + const char *reason) +{ + const char *map_reason, *reopen_reason; + bool reopened; + int ret; + + if (index->log->head == NULL || index->indexid == 0) { + /* we're creating the index file, we don't have any + logs yet */ + return 1; + } + + /* and update the map with the latest changes from transaction log */ + ret = mail_index_sync_map(&index->map, type, &map_reason); + if (ret != 0) + return ret; + + if (index->fd == -1) { + reopen_reason = "Index not open"; + reopened = FALSE; + } else { + /* Check if the index was recreated while we were opening it. + This is unlikely, but could happen if + mail_index_log_optimization_settings.max_size is tiny. */ + ret = mail_index_reopen_if_changed(index, &reopened, &reopen_reason); + if (ret < 0) + return -1; + if (ret == 0) { + /* Index was unexpectedly lost. The mailbox was + probably deleted while we were opening it. Handle + this as an error. */ + index->index_deleted = TRUE; + return -1; + } + } + if (!reopened) { + /* fsck the index and try to reopen */ + mail_index_set_error(index, "Index %s: %s: %s - fscking " + "(reopen_reason: %s)", + index->filepath, reason, map_reason, + reopen_reason); + if (!index->readonly) { + if (mail_index_fsck(index) < 0) + return -1; + } + } + + ret = mail_index_map_latest_file(index, &reason); + if (ret > 0 && index->indexid != 0) { + ret = mail_index_sync_map(&index->map, type, &map_reason); + if (ret == 0) { + mail_index_set_error(index, "Index %s: %s: %s", + index->filepath, reason, map_reason); + } + } + return ret; +} + +int mail_index_map(struct mail_index *index, + enum mail_index_sync_handler_type type) +{ + const char *reason; + int ret; + + i_assert(!index->mapping); + + index->mapping = TRUE; + + if (index->map == NULL) + index->map = mail_index_map_alloc(index); + + /* first try updating the existing mapping from transaction log. */ + if (!index->initial_mapped || index->reopen_main_index) { + /* index is being created/opened for the first time */ + ret = 0; + } else if (mail_index_sync_map_want_index_reopen(index->map, type)) { + /* it's likely more efficient to reopen the index file than + sync from the transaction log. */ + ret = 0; + } else { + /* sync the map from the transaction log. */ + ret = mail_index_sync_map(&index->map, type, &reason); + if (ret == 0) { + e_debug(index->event, + "Couldn't sync map from transaction log: %s - " + "reopening index instead", + reason); + } + } + + if (ret == 0) { + /* try to open and read the latest index. if it fails, we'll + fallback to updating the existing mapping from transaction + logs (which we'll also do even if the reopening succeeds). + if index files are unusable (e.g. major version change) + don't even try to use the transaction log. */ + ret = mail_index_map_latest_file(index, &reason); + if (ret > 0) { + ret = mail_index_map_latest_sync(index, type, reason); + } else if (ret == 0 && !index->readonly) { + /* make sure we don't try to open the file again */ + if (unlink(index->filepath) < 0 && errno != ENOENT) + mail_index_set_syscall_error(index, "unlink()"); + } + } + + if (ret >= 0) + index->initial_mapped = TRUE; + index->mapping = FALSE; + return ret; +} diff --git a/src/lib-index/mail-index-map.c b/src/lib-index/mail-index-map.c new file mode 100644 index 0000000..6ac2b93 --- /dev/null +++ b/src/lib-index/mail-index-map.c @@ -0,0 +1,595 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str-sanitize.h" +#include "mmap-util.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" + +void mail_index_map_init_extbufs(struct mail_index_map *map, + unsigned int initial_count) +{ +#define EXTENSION_NAME_APPROX_LEN 20 +#define EXT_GLOBAL_ALLOC_SIZE \ + ((sizeof(map->extensions) + sizeof(buffer_t)) * 2) +#define EXT_PER_ALLOC_SIZE \ + (EXTENSION_NAME_APPROX_LEN + \ + sizeof(struct mail_index_ext) + sizeof(uint32_t)) + size_t size; + + if (map->extension_pool == NULL) { + size = EXT_GLOBAL_ALLOC_SIZE + + initial_count * EXT_PER_ALLOC_SIZE; + map->extension_pool = + pool_alloconly_create(MEMPOOL_GROWING"map extensions", + nearest_power(size)); + } else { + p_clear(map->extension_pool); + + /* try to use the existing pool's size for initial_count so + we don't grow it needlessly */ + size = p_get_max_easy_alloc_size(map->extension_pool); + if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) { + initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) / + EXT_PER_ALLOC_SIZE; + } + } + + p_array_init(&map->extensions, map->extension_pool, initial_count); + p_array_init(&map->ext_id_map, map->extension_pool, initial_count); +} + +bool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name, + uint32_t *idx_r) +{ + const struct mail_index_ext *ext; + + if (!array_is_created(&map->extensions)) + return FALSE; + + array_foreach(&map->extensions, ext) { + if (strcmp(ext->name, name) == 0) { + *idx_r = array_foreach_idx(&map->extensions, ext); + return TRUE; + } + } + return FALSE; +} + +unsigned int mail_index_map_ext_hdr_offset(unsigned int name_len) +{ + size_t size = sizeof(struct mail_index_ext_header) + name_len; + return MAIL_INDEX_HEADER_SIZE_ALIGN(size); +} + +uint32_t +mail_index_map_register_ext(struct mail_index_map *map, + const char *name, uint32_t ext_offset, + const struct mail_index_ext_header *ext_hdr) +{ + struct mail_index_ext *ext; + uint32_t idx, ext_map_idx, empty_idx = (uint32_t)-1; + + i_assert(mail_index_ext_name_is_valid(name)); + + if (!array_is_created(&map->extensions)) { + mail_index_map_init_extbufs(map, 5); + idx = 0; + } else { + idx = array_count(&map->extensions); + } + i_assert(!mail_index_map_lookup_ext(map, name, &ext_map_idx)); + + ext = array_append_space(&map->extensions); + ext->name = p_strdup(map->extension_pool, name); + ext->ext_offset = ext_offset; + ext->hdr_offset = ext_offset == (uint32_t)-1 ? (uint32_t)-1 : + ext_offset + mail_index_map_ext_hdr_offset(strlen(name)); + ext->hdr_size = ext_hdr->hdr_size; + ext->record_offset = ext_hdr->record_offset; + ext->record_size = ext_hdr->record_size; + ext->record_align = ext_hdr->record_align; + ext->reset_id = ext_hdr->reset_id; + + ext->index_idx = mail_index_ext_register(map->index, name, + ext_hdr->hdr_size, + ext_hdr->record_size, + ext_hdr->record_align); + + /* Update index ext_id -> map ext_id mapping. Fill non-used + ext_ids with (uint32_t)-1 */ + while (array_count(&map->ext_id_map) < ext->index_idx) + array_push_back(&map->ext_id_map, &empty_idx); + array_idx_set(&map->ext_id_map, ext->index_idx, &idx); + return idx; +} + +int mail_index_map_ext_get_next(struct mail_index_map *map, + unsigned int *offset_p, + const struct mail_index_ext_header **ext_hdr_r, + const char **name_r) +{ + const struct mail_index_ext_header *ext_hdr; + unsigned int offset, name_offset; + + offset = *offset_p; + *name_r = ""; + + /* Extension header contains: + - struct mail_index_ext_header + - name (not 0-terminated) + - 64bit alignment padding + - extension header contents + - 64bit alignment padding + */ + name_offset = offset + sizeof(*ext_hdr); + ext_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, offset); + if (offset + sizeof(*ext_hdr) >= map->hdr.header_size) + return -1; + + offset += mail_index_map_ext_hdr_offset(ext_hdr->name_size); + if (offset > map->hdr.header_size) + return -1; + + *name_r = t_strndup(MAIL_INDEX_MAP_HDR_OFFSET(map, name_offset), + ext_hdr->name_size); + if (strcmp(*name_r, str_sanitize(*name_r, SIZE_MAX)) != 0) { + /* we allow only plain ASCII names, so this extension + is most likely broken */ + *name_r = ""; + } + + /* finally make sure that the hdr_size is small enough. + do this last so that we could return a usable name. */ + offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size); + if (offset > map->hdr.header_size) + return -1; + + *offset_p = offset; + *ext_hdr_r = ext_hdr; + return 0; +} + +static int +mail_index_map_ext_hdr_check_record(const struct mail_index_header *hdr, + const struct mail_index_ext_header *ext_hdr, + const char **error_r) +{ + if (ext_hdr->record_align == 0) { + *error_r = "Record field alignment is zero"; + return -1; + } + + /* until we get 128 bit CPUs having a larger alignment is pointless */ + if (ext_hdr->record_align > sizeof(uint64_t)) { + *error_r = "Record alignment is too large"; + return -1; + } + /* a large record size is most likely a bug somewhere. the maximum + record size is limited to 64k anyway, so try to fail earlier. */ + if (ext_hdr->record_size >= 32768) { + *error_r = "Record size is too large"; + return -1; + } + + if (ext_hdr->record_offset == 0) { + /* if we get here from extension introduction, record_offset=0 + and hdr->record_size hasn't been updated yet */ + return 0; + } + + if (ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) { + *error_r = t_strdup_printf("Record field points " + "outside record size (%u+%u > %u)", + ext_hdr->record_offset, + ext_hdr->record_size, + hdr->record_size); + return -1; + } + + if ((ext_hdr->record_offset % ext_hdr->record_align) != 0) { + *error_r = t_strdup_printf("Record field alignment %u " + "not used", ext_hdr->record_align); + return -1; + } + if ((hdr->record_size % ext_hdr->record_align) != 0) { + *error_r = t_strdup_printf("Record size not aligned by %u " + "as required by extension", + ext_hdr->record_align); + return -1; + } + return 0; +} + +int mail_index_map_ext_hdr_check(const struct mail_index_header *hdr, + const struct mail_index_ext_header *ext_hdr, + const char *name, const char **error_r) +{ + if (ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) { + *error_r = "Invalid field values"; + return -1; + } + if (!mail_index_ext_name_is_valid(name)) { + *error_r = "Invalid name"; + return -1; + } + + if (ext_hdr->record_size != 0) { + if (mail_index_map_ext_hdr_check_record(hdr, ext_hdr, + error_r) < 0) + return -1; + } + if (ext_hdr->hdr_size > MAIL_INDEX_EXT_HEADER_MAX_SIZE) { + *error_r = t_strdup_printf("Headersize too large (%u)", + ext_hdr->hdr_size); + return -1; + } + return 0; +} + +static void mail_index_header_init(struct mail_index *index, + struct mail_index_header *hdr) +{ + i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0); + + i_zero(hdr); + + hdr->major_version = MAIL_INDEX_MAJOR_VERSION; + hdr->minor_version = MAIL_INDEX_MINOR_VERSION; + hdr->base_header_size = sizeof(*hdr); + hdr->header_size = sizeof(*hdr); + hdr->record_size = sizeof(struct mail_index_record); + +#ifndef WORDS_BIGENDIAN + hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#endif + + hdr->indexid = index->indexid; + hdr->log_file_seq = 1; + hdr->next_uid = 1; + hdr->first_recent_uid = 1; +} + +struct mail_index_map *mail_index_map_alloc(struct mail_index *index) +{ + struct mail_index_map tmp_map; + + i_zero(&tmp_map); + mail_index_header_init(index, &tmp_map.hdr); + tmp_map.hdr_copy_buf = t_buffer_create(sizeof(tmp_map.hdr)); + buffer_append(tmp_map.hdr_copy_buf, &tmp_map.hdr, sizeof(tmp_map.hdr)); + tmp_map.index = index; + + /* a bit kludgy way to do this, but it initializes everything + nicely and correctly */ + return mail_index_map_clone(&tmp_map); +} + +static void mail_index_record_map_free(struct mail_index_map *map, + struct mail_index_record_map *rec_map) +{ + if (rec_map->buffer != NULL) { + i_assert(rec_map->mmap_base == NULL); + buffer_free(&rec_map->buffer); + } else if (rec_map->mmap_base != NULL) { + i_assert(rec_map->buffer == NULL); + if (munmap(rec_map->mmap_base, rec_map->mmap_size) < 0) + mail_index_set_syscall_error(map->index, "munmap()"); + rec_map->mmap_base = NULL; + } + array_free(&rec_map->maps); + if (rec_map->modseq != NULL) + mail_index_map_modseq_free(&rec_map->modseq); + i_free(rec_map); +} + +static void mail_index_record_map_unlink(struct mail_index_map *map) +{ + struct mail_index_map *const *maps; + unsigned int idx = UINT_MAX; + + array_foreach(&map->rec_map->maps, maps) { + if (*maps == map) { + idx = array_foreach_idx(&map->rec_map->maps, maps); + break; + } + } + i_assert(idx != UINT_MAX); + + array_delete(&map->rec_map->maps, idx, 1); + if (array_count(&map->rec_map->maps) == 0) { + mail_index_record_map_free(map, map->rec_map); + map->rec_map = NULL; + } +} + +void mail_index_unmap(struct mail_index_map **_map) +{ + struct mail_index_map *map = *_map; + + *_map = NULL; + if (--map->refcount > 0) + return; + + i_assert(map->refcount == 0); + mail_index_record_map_unlink(map); + + pool_unref(&map->extension_pool); + if (array_is_created(&map->keyword_idx_map)) + array_free(&map->keyword_idx_map); + buffer_free(&map->hdr_copy_buf); + i_free(map); +} + +static void mail_index_map_copy_records(struct mail_index_record_map *dest, + const struct mail_index_record_map *src, + unsigned int record_size) +{ + size_t size; + + size = src->records_count * record_size; + /* +1% so we have a bit of space to grow. useful for huge mailboxes. */ + dest->buffer = buffer_create_dynamic(default_pool, + size + I_MAX(size/100, 1024)); + buffer_append(dest->buffer, src->records, size); + + dest->records = buffer_get_modifiable_data(dest->buffer, NULL); + dest->records_count = src->records_count; +} + +static void mail_index_map_copy_header(struct mail_index_map *dest, + const struct mail_index_map *src) +{ + /* use src->hdr copy directly, because if we got here + from syncing it has the latest changes. */ + if (src != dest) + dest->hdr = src->hdr; + if (dest->hdr_copy_buf != NULL) { + if (src == dest) + return; + + buffer_set_used_size(dest->hdr_copy_buf, 0); + } else { + dest->hdr_copy_buf = + buffer_create_dynamic(default_pool, + dest->hdr.header_size); + } + buffer_append(dest->hdr_copy_buf, &dest->hdr, + I_MIN(sizeof(dest->hdr), src->hdr.base_header_size)); + if (src != dest) { + buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size, + MAIL_INDEX_MAP_HDR_OFFSET(src, src->hdr.base_header_size), + src->hdr.header_size - src->hdr.base_header_size); + } + i_assert(dest->hdr_copy_buf->used == dest->hdr.header_size); +} + +static struct mail_index_record_map * +mail_index_record_map_alloc(struct mail_index_map *map) +{ + struct mail_index_record_map *rec_map; + + rec_map = i_new(struct mail_index_record_map, 1); + i_array_init(&rec_map->maps, 4); + array_push_back(&rec_map->maps, &map); + return rec_map; +} + +struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map) +{ + struct mail_index_map *mem_map; + struct mail_index_ext *ext; + unsigned int count; + + mem_map = i_new(struct mail_index_map, 1); + mem_map->index = map->index; + mem_map->refcount = 1; + if (map->rec_map == NULL) { + mem_map->rec_map = mail_index_record_map_alloc(mem_map); + mem_map->rec_map->buffer = + buffer_create_dynamic(default_pool, 1024); + } else { + mem_map->rec_map = map->rec_map; + array_push_back(&mem_map->rec_map->maps, &mem_map); + } + + mail_index_map_copy_header(mem_map, map); + + /* copy extensions */ + if (array_is_created(&map->ext_id_map)) { + count = array_count(&map->ext_id_map); + mail_index_map_init_extbufs(mem_map, count + 2); + + array_append_array(&mem_map->extensions, &map->extensions); + array_append_array(&mem_map->ext_id_map, &map->ext_id_map); + + /* fix the name pointers to use our own pool */ + array_foreach_modifiable(&mem_map->extensions, ext) { + i_assert(ext->record_offset + ext->record_size <= + mem_map->hdr.record_size); + ext->name = p_strdup(mem_map->extension_pool, + ext->name); + } + } + + /* copy keyword map */ + if (array_is_created(&map->keyword_idx_map)) { + i_array_init(&mem_map->keyword_idx_map, + array_count(&map->keyword_idx_map) + 4); + array_append_array(&mem_map->keyword_idx_map, + &map->keyword_idx_map); + } + + return mem_map; +} + +void mail_index_record_map_move_to_private(struct mail_index_map *map) +{ + struct mail_index_record_map *new_map; + const struct mail_index_record *rec; + + if (array_count(&map->rec_map->maps) > 1) { + /* Multiple references to the rec_map. Create a clone of the + rec_map, which is in memory. */ + new_map = mail_index_record_map_alloc(map); + mail_index_map_copy_records(new_map, map->rec_map, + map->hdr.record_size); + mail_index_record_map_unlink(map); + map->rec_map = new_map; + if (map->rec_map->modseq != NULL) + new_map->modseq = mail_index_map_modseq_clone(map->rec_map->modseq); + } else { + new_map = map->rec_map; + } + + if (new_map->records_count != map->hdr.messages_count) { + /* The rec_map has more messages than what map contains. + These messages aren't necessary (and may confuse the caller), + so truncate them away. */ + i_assert(new_map->records_count > map->hdr.messages_count); + new_map->records_count = map->hdr.messages_count; + if (new_map->records_count == 0) + new_map->last_appended_uid = 0; + else { + rec = MAIL_INDEX_REC_AT_SEQ(map, new_map->records_count); + new_map->last_appended_uid = rec->uid; + } + buffer_set_used_size(new_map->buffer, new_map->records_count * + map->hdr.record_size); + } +} + +void mail_index_map_move_to_memory(struct mail_index_map *map) +{ + struct mail_index_record_map *new_map; + + if (map->rec_map->mmap_base == NULL) { + /* rec_map is already in memory */ + return; + } + + /* Move the rec_map contents to memory. If this is the only map that + refers to the rec_map, it can be directly replaced and the old + content munmap()ed. Otherwise, create a new rec_map for this map. */ + if (array_count(&map->rec_map->maps) == 1) + new_map = map->rec_map; + else { + new_map = mail_index_record_map_alloc(map); + new_map->modseq = map->rec_map->modseq == NULL ? NULL : + mail_index_map_modseq_clone(map->rec_map->modseq); + } + + mail_index_map_copy_records(new_map, map->rec_map, + map->hdr.record_size); + mail_index_map_copy_header(map, map); + + if (new_map != map->rec_map) { + mail_index_record_map_unlink(map); + map->rec_map = new_map; + } else { + if (munmap(new_map->mmap_base, new_map->mmap_size) < 0) + mail_index_set_syscall_error(map->index, "munmap()"); + new_map->mmap_base = NULL; + } +} + +bool mail_index_map_get_ext_idx(struct mail_index_map *map, + uint32_t ext_id, uint32_t *idx_r) +{ + const uint32_t *id; + + if (!array_is_created(&map->ext_id_map) || + ext_id >= array_count(&map->ext_id_map)) + return FALSE; + + id = array_idx(&map->ext_id_map, ext_id); + *idx_r = *id; + return *idx_r != (uint32_t)-1; +} + +static uint32_t mail_index_bsearch_uid(struct mail_index_map *map, + uint32_t uid, uint32_t left_idx, + int nearest_side) +{ + const struct mail_index_record *rec_base, *rec; + uint32_t idx, right_idx, record_size; + + i_assert(map->hdr.messages_count <= map->rec_map->records_count); + + rec_base = map->rec_map->records; + record_size = map->hdr.record_size; + + idx = left_idx; + right_idx = I_MIN(map->hdr.messages_count, uid); + + i_assert(right_idx < INT_MAX); + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + + rec = CONST_PTR_OFFSET(rec_base, idx * record_size); + if (rec->uid < uid) + left_idx = idx+1; + else if (rec->uid > uid) + right_idx = idx; + else + break; + } + i_assert(idx < map->hdr.messages_count); + + rec = CONST_PTR_OFFSET(rec_base, idx * record_size); + if (rec->uid != uid) { + if (nearest_side > 0) { + /* we want uid or larger */ + return rec->uid > uid ? idx+1 : + (idx == map->hdr.messages_count-1 ? 0 : idx+2); + } else { + /* we want uid or smaller */ + return rec->uid < uid ? idx + 1 : idx; + } + } + + return idx+1; +} + +void mail_index_map_lookup_seq_range(struct mail_index_map *map, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, + uint32_t *last_seq_r) +{ + i_assert(first_uid > 0); + i_assert(first_uid <= last_uid); + + if (map->hdr.messages_count == 0) { + *first_seq_r = *last_seq_r = 0; + return; + } + + *first_seq_r = mail_index_bsearch_uid(map, first_uid, 0, 1); + if (*first_seq_r == 0 || + MAIL_INDEX_REC_AT_SEQ(map, *first_seq_r)->uid > last_uid) { + *first_seq_r = *last_seq_r = 0; + return; + } + + if (last_uid >= map->hdr.next_uid-1) { + /* we want the last message */ + last_uid = map->hdr.next_uid-1; + if (first_uid > last_uid) { + *first_seq_r = *last_seq_r = 0; + return; + } + + *last_seq_r = map->hdr.messages_count; + return; + } + + if (first_uid == last_uid) + *last_seq_r = *first_seq_r; + else { + /* optimization - binary lookup only from right side: */ + *last_seq_r = mail_index_bsearch_uid(map, last_uid, + *first_seq_r - 1, -1); + } + i_assert(*last_seq_r >= *first_seq_r); +} diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c new file mode 100644 index 0000000..4285983 --- /dev/null +++ b/src/lib-index/mail-index-modseq.c @@ -0,0 +1,733 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-transaction-log-private.h" +#include "mail-index-private.h" +#include "mail-index-sync-private.h" +#include "mail-index-modseq.h" + +ARRAY_DEFINE_TYPE(modseqs, uint64_t); + +enum modseq_metadata_idx { + /* must be in the same order as enum mail_flags */ + METADATA_MODSEQ_IDX_ANSWERED = 0, + METADATA_MODSEQ_IDX_FLAGGED, + METADATA_MODSEQ_IDX_DELETED, + METADATA_MODSEQ_IDX_SEEN, + METADATA_MODSEQ_IDX_DRAFT, + + METADATA_MODSEQ_IDX_KEYWORD_START +}; + +struct metadata_modseqs { + ARRAY_TYPE(modseqs) modseqs; +}; + +struct mail_index_map_modseq { + /* indexes use enum modseq_metadata_idx */ + ARRAY(struct metadata_modseqs) metadata_modseqs; +}; + +struct mail_index_modseq_sync { + struct mail_index_sync_map_ctx *sync_map_ctx; + struct mail_index_view *view; + struct mail_transaction_log_view *log_view; + struct mail_index_map_modseq *mmap; +}; + +void mail_index_modseq_init(struct mail_index *index) +{ + index->modseq_ext_id = + mail_index_ext_register(index, MAIL_INDEX_MODSEQ_EXT_NAME, + sizeof(struct mail_index_modseq_header), + sizeof(uint64_t), sizeof(uint64_t)); +} + +static uint64_t mail_index_modseq_get_head(struct mail_index *index) +{ + return index->log->head == NULL ? 1 : + I_MAX(index->log->head->sync_highest_modseq, 1); +} + +void mail_index_modseq_enable(struct mail_index *index) +{ + struct mail_index_transaction *trans; + struct mail_index_view *view; + struct mail_index_modseq_header hdr; + uint32_t ext_map_idx; + + if (index->modseqs_enabled) + return; + + if (!mail_index_map_get_ext_idx(index->map, index->modseq_ext_id, + &ext_map_idx)) { + /* modseqs not enabled to the index yet, add them. */ + view = mail_index_view_open(index); + trans = mail_index_transaction_begin(view, 0); + + i_zero(&hdr); + hdr.highest_modseq = mail_index_modseq_get_head(index); + mail_index_update_header_ext(trans, index->modseq_ext_id, + 0, &hdr, sizeof(hdr)); + + /* commit also refreshes the index, which syncs the modseqs */ + (void)mail_index_transaction_commit(&trans); + mail_index_view_close(&view); + + /* get the modseq extension to index map */ + if (!mail_index_map_get_ext_idx(index->map, + index->modseq_ext_id, + &ext_map_idx)) { + /* didn't work for some reason */ + return; + } + } + index->modseqs_enabled = TRUE; +} + +bool mail_index_have_modseq_tracking(struct mail_index *index) +{ + return mail_index_map_get_modseq_header(index->map) != NULL; +} + +void mail_index_modseq_hdr_snapshot_update(struct mail_index_map *map) +{ + const struct mail_index_modseq_header *hdr = + mail_index_map_get_modseq_header(map); + if (hdr != NULL) + map->modseq_hdr_snapshot = *hdr; + else + i_zero(&map->modseq_hdr_snapshot); +} + +const struct mail_index_modseq_header * +mail_index_map_get_modseq_header(struct mail_index_map *map) +{ + const struct mail_index_ext *ext; + uint32_t idx; + + if (!mail_index_map_get_ext_idx(map, map->index->modseq_ext_id, &idx)) + return NULL; + + ext = array_idx(&map->extensions, idx); + if (ext->hdr_size != sizeof(struct mail_index_modseq_header)) + return NULL; + + return MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset); +} + +uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map) +{ + const struct mail_index_modseq_header *modseq_hdr; + + modseq_hdr = mail_index_map_get_modseq_header(map); + if (modseq_hdr != NULL && modseq_hdr->highest_modseq != 0) + return modseq_hdr->highest_modseq; + else { + /* fallback to returning the log head. if modseqs aren't + enabled, we return 0. */ + return map->index->log->head == NULL ? 0 : + map->index->log->head->sync_highest_modseq; + } +} + +uint64_t mail_index_modseq_get_highest(struct mail_index_view *view) +{ + return mail_index_map_modseq_get_highest(view->map); +} + +static struct mail_index_map_modseq * +mail_index_map_modseq(struct mail_index_view *view) +{ + struct mail_index_map_modseq *mmap = view->map->rec_map->modseq; + uint32_t ext_map_idx; + + if (mmap != NULL) + return mmap; + + /* don't start tracking until we've seen modseq extension intro */ + if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id, + &ext_map_idx)) + return NULL; + + mmap = i_new(struct mail_index_map_modseq, 1); + i_array_init(&mmap->metadata_modseqs, + METADATA_MODSEQ_IDX_KEYWORD_START + + array_count(&view->index->keywords)); + view->map->rec_map->modseq = mmap; + return mmap; +} + +uint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq) +{ + struct mail_index_map_modseq *mmap = mail_index_map_modseq(view); + struct mail_index_map *map; + const struct mail_index_ext *ext; + const struct mail_index_record *rec; + const uint64_t *modseqp; + uint32_t ext_map_idx; + + if (mmap == NULL) + return mail_index_modseq_get_head(view->index); + + rec = mail_index_lookup_full(view, seq, &map, NULL); + if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id, + &ext_map_idx)) { + /* not enabled yet */ + return mail_index_modseq_get_head(view->index); + } + + ext = array_idx(&map->extensions, ext_map_idx); + modseqp = CONST_PTR_OFFSET(rec, ext->record_offset); + if (*modseqp == 0) { + /* If we're here because we just enabled modseqs, we'll return + the same modseq (initial highestmodseq) for all messages. + The next sync will change these zeros to initial + highestmodseq or higher. + + If we're here because a message got appended but modseq + wasn't set (older Dovecot?), we'll again use the current + highest modseq. This isn't exactly correct, but it gets + fixed after the next sync and this situation shouldn't + normally happen anyway. */ + return mail_index_modseq_get_highest(view); + } + return *modseqp; +} + +int mail_index_modseq_set(struct mail_index_view *view, + uint32_t seq, uint64_t min_modseq) +{ + struct mail_index_map_modseq *mmap = mail_index_map_modseq(view); + const struct mail_index_ext *ext; + struct mail_index_record *rec; + uint64_t *modseqp; + uint32_t ext_map_idx; + + if (mmap == NULL) + return -1; + + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id, + &ext_map_idx)) + return -1; + + ext = array_idx(&view->map->extensions, ext_map_idx); + modseqp = PTR_OFFSET(rec, ext->record_offset); + if (*modseqp > min_modseq) + return 0; + else { + *modseqp = min_modseq; + return 1; + } +} + +static uint64_t +modseq_idx_lookup(struct mail_index_map_modseq *mmap, + unsigned int idx, uint32_t seq) +{ + const struct metadata_modseqs *metadata; + const uint64_t *modseqs; + unsigned int count; + + metadata = array_get(&mmap->metadata_modseqs, &count); + if (idx >= count || !array_is_created(&metadata[idx].modseqs)) + return 0; + + modseqs = array_get(&metadata[idx].modseqs, &count); + return seq > count ? 0 : modseqs[seq-1]; +} + +uint64_t mail_index_modseq_lookup_flags(struct mail_index_view *view, + enum mail_flags flags_mask, + uint32_t seq) +{ + struct mail_index_map_modseq *mmap = mail_index_map_modseq(view); + unsigned int i; + uint64_t modseq, highest_modseq = 0; + + if (mmap != NULL) { + /* first try to find a specific match */ + for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) { + if ((flags_mask & (1 << i)) != 0) { + modseq = modseq_idx_lookup(mmap, i, seq); + if (highest_modseq < modseq) + highest_modseq = modseq; + } + } + } + + if (highest_modseq == 0) { + /* no specific matches, fallback to using the highest */ + highest_modseq = mail_index_modseq_lookup(view, seq); + } + return highest_modseq; +} + +uint64_t mail_index_modseq_lookup_keywords(struct mail_index_view *view, + const struct mail_keywords *keywords, + uint32_t seq) +{ + struct mail_index_map_modseq *mmap = mail_index_map_modseq(view); + unsigned int i, metadata_idx; + uint64_t modseq, highest_modseq = 0; + + if (mmap != NULL) { + /* first try to find a specific match */ + for (i = 0; i < keywords->count; i++) { + metadata_idx = METADATA_MODSEQ_IDX_KEYWORD_START + + keywords->idx[i]; + + modseq = modseq_idx_lookup(mmap, metadata_idx, seq); + if (highest_modseq < modseq) + highest_modseq = modseq; + } + } + + if (highest_modseq == 0) { + /* no specific matches, fallback to using the highest */ + highest_modseq = mail_index_modseq_lookup(view, seq); + } + return highest_modseq; +} + +static void +mail_index_modseq_update(struct mail_index_modseq_sync *ctx, + uint64_t modseq, bool nonzeros, + uint32_t seq1, uint32_t seq2) +{ + const struct mail_index_ext *ext; + struct mail_index_record *rec; + uint32_t ext_map_idx; + uint64_t *modseqp; + + if (!mail_index_map_get_ext_idx(ctx->view->map, + ctx->view->index->modseq_ext_id, + &ext_map_idx)) + return; + + ext = array_idx(&ctx->view->map->extensions, ext_map_idx); + for (; seq1 <= seq2; seq1++) { + rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq1); + modseqp = PTR_OFFSET(rec, ext->record_offset); + if (*modseqp == 0 || (nonzeros && *modseqp < modseq)) + *modseqp = modseq; + } +} + +static bool +mail_index_modseq_update_to_highest(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2) +{ + uint64_t modseq; + + if (ctx->mmap == NULL) + return FALSE; + + modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view); + mail_index_modseq_update(ctx, modseq, TRUE, seq1, seq2); + return TRUE; +} + +static void +mail_index_modseq_update_old_rec(struct mail_index_modseq_sync *ctx, + const struct mail_transaction_header *thdr, + const void *tdata) +{ + ARRAY_TYPE(seq_range) uids = ARRAY_INIT; + const struct seq_range *rec; + buffer_t uid_buf; + unsigned int i, count; + uint32_t seq1, seq2; + + switch (thdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_APPEND: { + const struct mail_index_record *appends = tdata; + + count = thdr->size / sizeof(*appends); + for (i = 0; i < count; i++) { + if (mail_index_lookup_seq(ctx->view, + appends[i].uid, &seq1)) { + (void)mail_index_modseq_update_to_highest(ctx, seq1, seq1); + } + } + return; + } + case MAIL_TRANSACTION_FLAG_UPDATE: { + buffer_create_from_const_data(&uid_buf, tdata, thdr->size); + array_create_from_buffer(&uids, &uid_buf, + sizeof(struct mail_transaction_flag_update)); + break; + } + case MAIL_TRANSACTION_KEYWORD_UPDATE: { + const struct mail_transaction_keyword_update *rec = tdata; + unsigned int seqset_offset; + + seqset_offset = sizeof(*rec) + rec->name_size; + if ((seqset_offset % 4) != 0) + seqset_offset += 4 - (seqset_offset % 4); + + buffer_create_from_const_data(&uid_buf, + CONST_PTR_OFFSET(tdata, seqset_offset), + thdr->size - seqset_offset); + array_create_from_buffer(&uids, &uid_buf, sizeof(uint32_t)*2); + break; + } + case MAIL_TRANSACTION_KEYWORD_RESET: + buffer_create_from_const_data(&uid_buf, tdata, thdr->size); + array_create_from_buffer(&uids, &uid_buf, + sizeof(struct mail_transaction_keyword_reset)); + break; + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: + break; + default: + return; + } + + /* update modseqs */ + count = array_is_created(&uids) ? array_count(&uids) : 0; + for (i = 0; i < count; i++) { + rec = array_idx(&uids, i); + if (mail_index_lookup_seq_range(ctx->view, rec->seq1, rec->seq2, + &seq1, &seq2)) + (void)mail_index_modseq_update_to_highest(ctx, seq1, seq2); + } +} + +static void mail_index_modseq_sync_init(struct mail_index_modseq_sync *ctx) +{ + struct mail_index_map *map = ctx->view->map; + const struct mail_index_ext *ext; + const struct mail_index_modseq_header *hdr; + const struct mail_transaction_header *thdr; + const void *tdata; + const char *reason; + uint32_t ext_map_idx; + uint32_t end_seq; + uoff_t end_offset; + uint64_t cur_modseq; + bool reset; + int ret; + + if (!mail_index_map_get_ext_idx(map, ctx->view->index->modseq_ext_id, + &ext_map_idx)) + i_unreached(); + ext = array_idx(&map->extensions, ext_map_idx); + + /* get the current highest_modseq. don't change any modseq below it. */ + hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset); + + /* Scan logs for updates between ext_hdr.log_* .. view position. + There are two reasons why there could be any: + + 1) We just enabled modseqs and we're filling the initial values. + 2) A non-modseq-aware Dovecot version added new messages and wrote + dovecot.index file. */ + mail_transaction_log_view_get_prev_pos(ctx->view->log_view, + &end_seq, &end_offset); + if (end_seq < hdr->log_seq || + (end_seq == hdr->log_seq && end_offset <= hdr->log_offset)) { + /* modseqs are up to date */ + return; + } + + ctx->log_view = mail_transaction_log_view_open(ctx->view->index->log); + ret = mail_transaction_log_view_set(ctx->log_view, + I_MAX(1, hdr->log_seq), + hdr->log_offset, + end_seq, end_offset, &reset, &reason); + if (ret <= 0) { + /* missing files / error - try with only the last file */ + ret = mail_transaction_log_view_set(ctx->log_view, end_seq, 0, + end_seq, end_offset, + &reset, &reason); + /* since we don't know if we skipped some changes, set all + modseqs to beginning of the latest file. */ + cur_modseq = mail_transaction_log_view_get_prev_modseq( + ctx->log_view); + if (cur_modseq < hdr->highest_modseq) { + /* should happen only when setting initial modseqs. + we may already have returned highest_modseq as + some messages' modseq value. don't shrink it. */ + cur_modseq = hdr->highest_modseq; + } + mail_index_modseq_update(ctx, cur_modseq, TRUE, 1, + map->hdr.messages_count); + } else { + /* we have all the logs. replace zero modseqs with the current + highest modseq (we may have already returned it for them). */ + mail_index_modseq_update(ctx, hdr->highest_modseq, FALSE, 1, + map->hdr.messages_count); + } + if (ret > 0) { + while (mail_transaction_log_view_next(ctx->log_view, + &thdr, &tdata) > 0) { + T_BEGIN { + mail_index_modseq_update_old_rec(ctx, thdr, + tdata); + } T_END; + } + } + mail_transaction_log_view_close(&ctx->log_view); +} + +struct mail_index_modseq_sync * +mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx *sync_map_ctx) +{ + struct mail_index_modseq_sync *ctx; + + ctx = i_new(struct mail_index_modseq_sync, 1); + ctx->sync_map_ctx = sync_map_ctx; + ctx->view = sync_map_ctx->view; + ctx->mmap = mail_index_map_modseq(ctx->view); + if (ctx->mmap != NULL) { + mail_index_modseq_sync_init(ctx); + ctx->log_view = ctx->view->log_view; + } + return ctx; +} + +static void mail_index_modseq_update_header(struct mail_index_modseq_sync *ctx) +{ + struct mail_index_view *view = ctx->view; + struct mail_index_map *map = view->map; + const struct mail_index_ext *ext; + const struct mail_index_modseq_header *old_modseq_hdr; + struct mail_index_modseq_header new_modseq_hdr; + uint32_t ext_map_idx, log_seq; + uoff_t log_offset; + uint64_t highest_modseq; + + if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id, + &ext_map_idx)) + return; + + mail_transaction_log_view_get_prev_pos(view->log_view, + &log_seq, &log_offset); + highest_modseq = mail_transaction_log_view_get_prev_modseq(view->log_view); + + ext = array_idx(&map->extensions, ext_map_idx); + old_modseq_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset); + + if (old_modseq_hdr->log_seq < log_seq || + (old_modseq_hdr->log_seq == log_seq && + old_modseq_hdr->log_offset < log_offset)) { + i_zero(&new_modseq_hdr); + new_modseq_hdr.highest_modseq = highest_modseq; + new_modseq_hdr.log_seq = log_seq; + new_modseq_hdr.log_offset = log_offset; + + buffer_write(map->hdr_copy_buf, ext->hdr_offset, + &new_modseq_hdr, sizeof(new_modseq_hdr)); + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); + } +} + +void mail_index_modseq_sync_end(struct mail_index_modseq_sync **_ctx) +{ + struct mail_index_modseq_sync *ctx = *_ctx; + + *_ctx = NULL; + if (ctx->mmap != NULL) { + i_assert(ctx->mmap == ctx->view->map->rec_map->modseq); + mail_index_modseq_update_header(ctx); + } + i_free(ctx); +} + +void mail_index_modseq_sync_map_replaced(struct mail_index_modseq_sync *ctx) +{ + ctx->mmap = mail_index_map_modseq(ctx->view); +} + +void mail_index_modseq_hdr_update(struct mail_index_modseq_sync *ctx) +{ + if (ctx->mmap == NULL) { + ctx->mmap = mail_index_map_modseq(ctx->view); + i_assert(ctx->mmap != NULL); + mail_index_modseq_sync_init(ctx); + ctx->log_view = ctx->view->log_view; + } +} + +void mail_index_modseq_append(struct mail_index_modseq_sync *ctx, uint32_t seq) +{ + (void)mail_index_modseq_update_to_highest(ctx, seq, seq); +} + +void mail_index_modseq_expunge(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2) +{ + struct metadata_modseqs *metadata; + + if (ctx->mmap == NULL) + return; + + seq1--; + array_foreach_modifiable(&ctx->mmap->metadata_modseqs, metadata) { + if (array_is_created(&metadata->modseqs)) + array_delete(&metadata->modseqs, seq1, seq2-seq1); + } +} + +static void +modseqs_update(ARRAY_TYPE(modseqs) *array, uint32_t seq1, uint32_t seq2, + uint64_t value) +{ + uint64_t *modseqp; + + for (; seq1 <= seq2; seq1++) { + modseqp = array_idx_get_space(array, seq1-1); + if (*modseqp < value) + *modseqp = value; + } +} + +static void +modseqs_idx_update(struct mail_index_modseq_sync *ctx, unsigned int idx, + uint32_t seq1, uint32_t seq2) +{ + struct metadata_modseqs *metadata; + uint64_t modseq; + + if (!ctx->view->index->modseqs_enabled) { + /* we want to keep permanent modseqs updated, but don't bother + updating in-memory per-flag updates */ + return; + } + + modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view); + metadata = array_idx_get_space(&ctx->mmap->metadata_modseqs, idx); + if (!array_is_created(&metadata->modseqs)) + i_array_init(&metadata->modseqs, seq2 + 16); + modseqs_update(&metadata->modseqs, seq1, seq2, modseq); +} + +void mail_index_modseq_update_flags(struct mail_index_modseq_sync *ctx, + enum mail_flags flags_mask, + uint32_t seq1, uint32_t seq2) +{ + unsigned int i; + + if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2)) + return; + + for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) { + if ((flags_mask & (1 << i)) != 0) + modseqs_idx_update(ctx, i, seq1, seq2); + } +} + +void mail_index_modseq_update_keyword(struct mail_index_modseq_sync *ctx, + unsigned int keyword_idx, + uint32_t seq1, uint32_t seq2) +{ + if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2)) + return; + + modseqs_idx_update(ctx, METADATA_MODSEQ_IDX_KEYWORD_START + keyword_idx, + seq1, seq2); +} + +void mail_index_modseq_reset_keywords(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2) +{ + unsigned int i, count; + + if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2)) + return; + + count = array_count(&ctx->mmap->metadata_modseqs); + for (i = METADATA_MODSEQ_IDX_KEYWORD_START; i < count; i++) + modseqs_idx_update(ctx, i, seq1, seq2); +} + +struct mail_index_map_modseq * +mail_index_map_modseq_clone(const struct mail_index_map_modseq *mmap) +{ + struct mail_index_map_modseq *new_mmap; + const struct metadata_modseqs *src_metadata; + struct metadata_modseqs *dest_metadata; + unsigned int i, count; + + src_metadata = array_get(&mmap->metadata_modseqs, &count); + + new_mmap = i_new(struct mail_index_map_modseq, 1); + i_array_init(&new_mmap->metadata_modseqs, count + 16); + + for (i = 0; i < count; i++) { + dest_metadata = array_append_space(&new_mmap->metadata_modseqs); + if (array_is_created(&src_metadata[i].modseqs)) { + i_array_init(&dest_metadata->modseqs, + array_count(&src_metadata[i].modseqs)); + array_append_array(&dest_metadata->modseqs, + &src_metadata[i].modseqs); + } + } + return new_mmap; +} + +void mail_index_map_modseq_free(struct mail_index_map_modseq **_mmap) +{ + struct mail_index_map_modseq *mmap = *_mmap; + struct metadata_modseqs *metadata; + + *_mmap = NULL; + + array_foreach_modifiable(&mmap->metadata_modseqs, metadata) { + if (array_is_created(&metadata->modseqs)) + array_free(&metadata->modseqs); + } + array_free(&mmap->metadata_modseqs); + i_free(mmap); +} + +bool mail_index_modseq_get_next_log_offset(struct mail_index_view *view, + uint64_t modseq, uint32_t *log_seq_r, + uoff_t *log_offset_r) +{ + struct mail_transaction_log *log = view->index->log; + struct mail_transaction_log_file *file, *prev_file; + const char *reason; + int ret; + + if (log->files == NULL) { + /* we shouldn't normally get here */ + return FALSE; + } + while (modseq < log->files->hdr.initial_modseq) { + /* try to find the previous log file if it still exists */ + ret = mail_transaction_log_find_file(log, + log->files->hdr.file_seq - 1, FALSE, &file, &reason); + if (ret <= 0) + return FALSE; + } + + prev_file = NULL; + for (file = log->files; file != NULL; file = file->next) { + if (modseq < file->hdr.initial_modseq) + break; + prev_file = file; + } + + if (prev_file == NULL) { + /* the log file has been deleted already */ + return FALSE; + } + + *log_seq_r = prev_file->hdr.file_seq; + if (mail_transaction_log_file_get_modseq_next_offset(prev_file, modseq, + log_offset_r) < 0) + return FALSE; + + if (*log_seq_r > view->log_file_head_seq || + (*log_seq_r == view->log_file_head_seq && + *log_offset_r > view->log_file_head_offset)) { + /* modseq is already beyond our view. move it back so the + caller won't be confused. */ + *log_seq_r = view->log_file_head_seq; + *log_offset_r = view->log_file_head_offset; + } + return TRUE; +} diff --git a/src/lib-index/mail-index-modseq.h b/src/lib-index/mail-index-modseq.h new file mode 100644 index 0000000..f0fe48a --- /dev/null +++ b/src/lib-index/mail-index-modseq.h @@ -0,0 +1,66 @@ +#ifndef MAIL_INDEX_MODSEQ_H +#define MAIL_INDEX_MODSEQ_H + +#include "mail-types.h" + +#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq" + +struct mail_keywords; +struct mail_index; +struct mail_index_map; +struct mail_index_view; +struct mail_index_modseq; +struct mail_index_map_modseq; +struct mail_index_sync_map_ctx; + +void mail_index_modseq_init(struct mail_index *index); + +/* Save a copy of the current modseq header to map->modseq_hdr_snapshot. This + is expected to be called when reading the dovecot.index header before any + changes are applied on top of it from dovecot.index.log. */ +void mail_index_modseq_hdr_snapshot_update(struct mail_index_map *map); + +const struct mail_index_modseq_header * +mail_index_map_get_modseq_header(struct mail_index_map *map); +uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map); +void mail_index_modseq_enable(struct mail_index *index); +bool mail_index_have_modseq_tracking(struct mail_index *index); +uint64_t mail_index_modseq_get_highest(struct mail_index_view *view); + +uint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq); +uint64_t mail_index_modseq_lookup_flags(struct mail_index_view *view, + enum mail_flags flags_mask, + uint32_t seq); +uint64_t mail_index_modseq_lookup_keywords(struct mail_index_view *view, + const struct mail_keywords *keywords, + uint32_t seq); +int mail_index_modseq_set(struct mail_index_view *view, + uint32_t seq, uint64_t min_modseq); + +struct mail_index_modseq_sync * +mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx *sync_map_ctx); +void mail_index_modseq_sync_end(struct mail_index_modseq_sync **ctx); + +void mail_index_modseq_sync_map_replaced(struct mail_index_modseq_sync *ctx); +void mail_index_modseq_hdr_update(struct mail_index_modseq_sync *ctx); +void mail_index_modseq_append(struct mail_index_modseq_sync *ctx, uint32_t seq); +void mail_index_modseq_expunge(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2); +void mail_index_modseq_update_flags(struct mail_index_modseq_sync *ctx, + enum mail_flags flags_mask, + uint32_t seq1, uint32_t seq2); +void mail_index_modseq_update_keyword(struct mail_index_modseq_sync *ctx, + unsigned int keyword_idx, + uint32_t seq1, uint32_t seq2); +void mail_index_modseq_reset_keywords(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2); + +struct mail_index_map_modseq * +mail_index_map_modseq_clone(const struct mail_index_map_modseq *mmap); +void mail_index_map_modseq_free(struct mail_index_map_modseq **mmap); + +bool mail_index_modseq_get_next_log_offset(struct mail_index_view *view, + uint64_t modseq, uint32_t *log_seq_r, + uoff_t *log_offset_r); + +#endif diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h new file mode 100644 index 0000000..26f7e39 --- /dev/null +++ b/src/lib-index/mail-index-private.h @@ -0,0 +1,437 @@ +#ifndef MAIL_INDEX_PRIVATE_H +#define MAIL_INDEX_PRIVATE_H + +#include "file-lock.h" +#include "mail-index.h" +#include "mail-index-util.h" +#include "mail-index-view-private.h" +#include "mail-index-transaction-private.h" + +#include <sys/stat.h> + +struct mail_transaction_header; +struct mail_transaction_log_view; +struct mail_index_sync_map_ctx; + +/* How large index files to mmap() instead of reading to memory. */ +#define MAIL_INDEX_MMAP_MIN_SIZE (1024*64) +/* How many times to retry opening index files if read/fstat returns ESTALE. + This happens with NFS when the file has been deleted (ie. index file was + rewritten by another computer than us). */ +#define MAIL_INDEX_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT +/* Large extension header sizes are probably caused by file corruption, so + try to catch them by limiting the header size. */ +#define MAIL_INDEX_EXT_HEADER_MAX_SIZE (1024*1024*16-1) + +#define MAIL_INDEX_IS_IN_MEMORY(index) \ + ((index)->dir == NULL) + +#define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \ + ((map)->rec_map->mmap_base == NULL) + +#define MAIL_INDEX_MAP_IDX(map, idx) \ + ((struct mail_index_record *) \ + PTR_OFFSET((map)->rec_map->records, (idx) * (map)->hdr.record_size)) +#define MAIL_INDEX_REC_AT_SEQ(map, seq) \ + ((struct mail_index_record *) \ + PTR_OFFSET((map)->rec_map->records, ((seq)-1) * (map)->hdr.record_size)) + +#define MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u) \ + ((((u)->add_flags | (u)->remove_flags) & MAIL_INDEX_FLAGS_MASK) == 0 && \ + (u)->modseq_inc_flag == 0) + +#define MAIL_INDEX_EXT_KEYWORDS "keywords" +#define MAIL_INDEX_EXT_NAME_MAX_LENGTH 64 + +typedef int mail_index_expunge_handler_t(struct mail_index_sync_map_ctx *ctx, + const void *data, void **sync_context); + +#define MAIL_INDEX_HEADER_SIZE_ALIGN(size) \ + (((size) + 7) & ~7U) + +/* In-memory copy of struct mail_index_ext_header */ +struct mail_index_ext { + const char *name; + uint32_t index_idx; /* index ext_id */ + uint32_t reset_id; + uint32_t ext_offset; /* points to beginning of mail_index_ext_header */ + uint32_t hdr_offset; /* points to mail_index_ext_header.data[] */ + uint32_t hdr_size; /* size of mail_index_ext_header.data[] */ + uint16_t record_offset; + uint16_t record_size; + uint16_t record_align; +}; + +struct mail_index_ext_header { + /* Size of data[], i.e. the extension size in header */ + uint32_t hdr_size; + /* If reset_id changes, all of the extension record data is + invalidated. For example with cache files reset_id must match the + cache header's file_seq or the cache offsets aren't valid. */ + uint32_t reset_id; + /* Offset of this extension in struct mail_index_record. */ + uint16_t record_offset; + /* Size of this extension in struct mail_index_record. */ + uint16_t record_size; + /* Required alignment of this extension in struct mail_index_record. + It's expected that record_offset is correctly aligned. This is used + only when rearranging fields due to adding/removing other + extensions. */ + uint16_t record_align; + /* Size of name[], which contains the extension's unique name. */ + uint16_t name_size; + /* unsigned char name[name_size]; */ + /* Extension header data, if any. This starts from the next 64-bit + aligned offset after name[]. */ + /* unsigned char data[hdr_size]; */ +}; + +struct mail_index_keyword_header { + uint32_t keywords_count; + /* struct mail_index_keyword_header_rec[] */ + /* char name[][] */ +}; + +struct mail_index_keyword_header_rec { + uint32_t unused; /* for backwards compatibility */ + uint32_t name_offset; /* relative to beginning of name[] */ +}; + +enum mail_index_sync_handler_type { + MAIL_INDEX_SYNC_HANDLER_FILE = 0x01, + MAIL_INDEX_SYNC_HANDLER_HEAD = 0x02, + MAIL_INDEX_SYNC_HANDLER_VIEW = 0x04 +}; + +struct mail_index_registered_ext { + const char *name; + uint32_t index_idx; /* index ext_id */ + uint32_t hdr_size; /* size of mail_index_ext_header.data[] */ + uint16_t record_size; + uint16_t record_align; + + mail_index_expunge_handler_t *expunge_handler; +}; + +struct mail_index_modseq_header { + /* highest used modseq */ + uint64_t highest_modseq; + /* last tracked log file position */ + uint32_t log_seq; + uint32_t log_offset; +}; + +struct mail_index_record_map { + ARRAY(struct mail_index_map *) maps; + + void *mmap_base; + size_t mmap_size, mmap_used_size; + + buffer_t *buffer; + + void *records; /* struct mail_index_record[] */ + unsigned int records_count; + + struct mail_index_map_modseq *modseq; + uint32_t last_appended_uid; +}; + +#define MAIL_INDEX_MAP_HDR_OFFSET(map, hdr_offset) \ + CONST_PTR_OFFSET((map)->hdr_copy_buf->data, hdr_offset) +struct mail_index_map { + struct mail_index *index; + int refcount; + + /* Copy of the base header for convenience. Note that base_header_size + may be smaller or larger than this struct. If it's smaller, the last + fields in the struct are filled with zeroes. */ + struct mail_index_header hdr; + /* Copy of the full header. */ + buffer_t *hdr_copy_buf; + + pool_t extension_pool; + ARRAY(struct mail_index_ext) extensions; + ARRAY(uint32_t) ext_id_map; /* index -> file */ + + ARRAY(unsigned int) keyword_idx_map; /* file -> index */ + + struct mail_index_modseq_header modseq_hdr_snapshot; + + struct mail_index_record_map *rec_map; +}; + +struct mail_index_module_register { + unsigned int id; +}; + +union mail_index_module_context { + struct mail_index_module_register *reg; +}; + +struct mail_index_settings { + /* Directory path for .cache file. Set via + mail_index_set_cache_dir(). */ + char *cache_dir; + + /* fsyncing behavior. Set via mail_index_set_fsync_mode(). */ + enum fsync_mode fsync_mode; + enum mail_index_fsync_mask fsync_mask; + + /* Index file permissions. Set via mail_index_set_permissions(). */ + mode_t mode; + gid_t gid; + char *gid_origin; + + /* Lock settings. Set via mail_index_set_lock_method(). */ + enum file_lock_method lock_method; + unsigned int max_lock_timeout_secs; + + /* Initial extension added to newly created indexes. Set via + mail_index_set_ext_init_data(). */ + uint32_t ext_hdr_init_id; + void *ext_hdr_init_data; +}; + +struct mail_index_error { + /* Human-readable error text */ + char *text; + + /* Error happened because there's no disk space, i.e. syscall failed + with ENOSPC or EDQUOT. */ + bool nodiskspace:1; +}; + +struct mail_index { + /* Directory path for the index, or NULL for in-memory indexes. */ + char *dir; + /* Filename prefix for the index, e.g. "dovecot.index." */ + char *prefix; + struct event *event; + enum mail_index_open_flags flags; + struct mail_index_settings set; + struct mail_index_optimization_settings optimization_set; + + struct mail_cache *cache; + struct mail_transaction_log *log; + + char *filepath; + int fd; + /* Linked list of currently opened views */ + struct mail_index_view *views; + /* Latest map */ + struct mail_index_map *map; + + /* ID number that permanently identifies the index. This is stored in + the index files' headers. If the indexids suddenly changes, it means + that the index has been completely recreated and needs to be + reopened (e.g. the mailbox was deleted and recreated while it + was open). */ + uint32_t indexid; + /* Views initially use this same ID value. This ID is incremented + whenever something unexpected happens to the index that prevents + syncing existing views. When the view's inconsistency_id doesn't + match this one, the view is marked as inconsistent. */ + unsigned int inconsistency_id; + /* How many times this index has been opened with mail_index_open(). */ + unsigned int open_count; + + /* These contain the log_file_seq and log_file_tail_offset that exists + in dovecot.index file's header. These are used to figure out if it's + time to rewrite the dovecot.index file. Note that these aren't + available in index->map->hdr, because it gets updated when + transaction log file is read. */ + uint32_t main_index_hdr_log_file_seq; + uint32_t main_index_hdr_log_file_tail_offset; + + /* log file which last updated index_deleted */ + uint32_t index_delete_changed_file_seq; + + /* transaction log head seq/offset when we last fscked */ + uint32_t fsck_log_head_file_seq; + uoff_t fsck_log_head_file_offset; + + /* syncing will update this if non-NULL */ + struct mail_index_transaction_commit_result *sync_commit_result; + /* Delayed log2_rotate_time update to mail_index_header. This is set + and unset within the same sync. */ + uint32_t hdr_log2_rotate_time_delayed_update; + + /* Registered extensions */ + pool_t extension_pool; + ARRAY(struct mail_index_registered_ext) extensions; + + /* All keywords that have ever been used in this index. Keywords are + only added here, never removed. */ + pool_t keywords_pool; + ARRAY_TYPE(keywords) keywords; + HASH_TABLE(char *, void *) keywords_hash; /* name -> unsigned int idx */ + + /* Registered extension IDs */ + uint32_t keywords_ext_id; + uint32_t modseq_ext_id; + + /* Module-specific contexts. */ + ARRAY(union mail_index_module_context *) module_contexts; + + /* Last error returned by mail_index_get_error_message(). + Cleared by mail_index_reset_error(). */ + struct mail_index_error last_error; + /* Timestamp when mmap() failure was logged the last time. This is used + to prevent logging the same error too rapidly. This could happen + e.g. if mmap()ing a large cache file that exceeeds process's + VSZ limit. */ + time_t last_mmap_error_time; + /* If non-NULL, dovecot.index should be recreated as soon as possible. + The reason for why the recreation is wanted is stored as human- + readable text. */ + char *need_recreate; + + /* Mapping has noticed non-external MAIL_TRANSACTION_INDEX_DELETED + record, i.e. a request to mark the index deleted. The next sync + will finish the deletion by writing external + MAIL_TRANSACTION_INDEX_DELETED record. */ + bool index_delete_requested:1; + /* Mapping has noticed external MAIL_TRANSACTION_INDEX_DELETED record, + or index was unexpectedly deleted under us. No more changes are + allowed to the index, except undeletion. */ + bool index_deleted:1; + /* .log is locked for syncing. This is the main exclusive lock for + indexes. */ + bool log_sync_locked:1; + /* Main index or .log couldn't be opened read-write */ + bool readonly:1; + /* mail_index_map() is running */ + bool mapping:1; + /* mail_index_sync_*() is running */ + bool syncing:1; + /* Mapping has read more from .log than it preferred. Use + mail_index_base_optimization_settings.rewrite_min_log_bytes the next + time when checking if index needs a rewrite. */ + bool index_min_write:1; + /* mail_index_modseq_enable() has been called. Track per-flag + modseq numbers in memory (global modseqs are tracked anyway). */ + bool modseqs_enabled:1; + /* mail_index_open() is creating new index files */ + bool initial_create:1; + /* TRUE after mail_index_map() has succeeded */ + bool initial_mapped:1; + /* The next mail_index_map() must reopen the main index, because the + currently opened one is too old. */ + bool reopen_main_index:1; + /* Index has been fsck'd, but mail_index_reset_fscked() hasn't been + called yet. */ + bool fscked:1; +}; + +extern struct mail_index_module_register mail_index_module_register; +extern struct event_category event_category_mail_index; + +/* Add/replace expunge handler for specified extension. */ +void mail_index_register_expunge_handler(struct mail_index *index, + uint32_t ext_id, + mail_index_expunge_handler_t *callback); +void mail_index_unregister_expunge_handler(struct mail_index *index, + uint32_t ext_id); + +int mail_index_create_tmp_file(struct mail_index *index, + const char *path_prefix, const char **path_r); + +int mail_index_try_open_only(struct mail_index *index); +void mail_index_close_file(struct mail_index *index); +/* Returns 1 if index was successfully (re-)opened, 0 if the index no longer + exists, -1 if I/O error. If 1 is returned, reopened_r=TRUE if a new index + was actually reopened (or if index wasn't even open before this call). */ +int mail_index_reopen_if_changed(struct mail_index *index, bool *reopened_r, + const char **reason_r); +/* Update/rewrite the main index file from index->map */ +void mail_index_write(struct mail_index *index, bool want_rotate, + const char *reason); + +void mail_index_flush_read_cache(struct mail_index *index, const char *path, + int fd, bool locked); + +int mail_index_lock_fd(struct mail_index *index, const char *path, int fd, + int lock_type, unsigned int timeout_secs, + struct file_lock **lock_r); + +/* Allocate a new empty map. */ +struct mail_index_map *mail_index_map_alloc(struct mail_index *index); +/* Replace index->map with the latest index changes. This may reopen the index + file and/or it may read the latest changes from transaction log. The log is + read up to EOF, but non-synced expunges are skipped. + + If we mmap()ed the index file, the map is returned locked. + + Returns 1 = ok, 0 = corrupted, -1 = error. */ +int mail_index_map(struct mail_index *index, + enum mail_index_sync_handler_type type); +/* Unreference given mapping and unmap it if it's dropped to zero. */ +void mail_index_unmap(struct mail_index_map **map); +/* Clone a map. It still points to the original rec_map. */ +struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map); +/* Make sure the map has its own private rec_map, cloning it if necessary. */ +void mail_index_record_map_move_to_private(struct mail_index_map *map); +/* If map points to mmap()ed index, copy it to the memory. */ +void mail_index_map_move_to_memory(struct mail_index_map *map); + +void mail_index_fchown(struct mail_index *index, int fd, const char *path); + +bool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name, + uint32_t *idx_r); +bool mail_index_ext_name_is_valid(const char *name); +uint32_t +mail_index_map_register_ext(struct mail_index_map *map, + const char *name, uint32_t ext_offset, + const struct mail_index_ext_header *ext_hdr); +bool mail_index_map_get_ext_idx(struct mail_index_map *map, + uint32_t ext_id, uint32_t *idx_r); +const struct mail_index_ext * +mail_index_view_get_ext(struct mail_index_view *view, uint32_t ext_id); + +void mail_index_map_lookup_seq_range(struct mail_index_map *map, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, + uint32_t *last_seq_r); + +/* Returns 1 on success, 0 on non-critical errors we want to silently fix, + -1 if map isn't usable. The caller is responsible for logging the errors + if -1 is returned. */ +int mail_index_map_check_header(struct mail_index_map *map, + const char **error_r); +/* Returns 1 if header is usable, 0 or -1 if not. The caller should log an + error if -1 is returned, but not if 0 is returned. */ +bool mail_index_check_header_compat(struct mail_index *index, + const struct mail_index_header *hdr, + uoff_t file_size, const char **error_r); +int mail_index_map_parse_extensions(struct mail_index_map *map); +int mail_index_map_parse_keywords(struct mail_index_map *map); + +void mail_index_map_init_extbufs(struct mail_index_map *map, + unsigned int initial_count); +int mail_index_map_ext_get_next(struct mail_index_map *map, + unsigned int *offset, + const struct mail_index_ext_header **ext_hdr_r, + const char **name_r); +int mail_index_map_ext_hdr_check(const struct mail_index_header *hdr, + const struct mail_index_ext_header *ext_hdr, + const char *name, const char **error_r); +unsigned int mail_index_map_ext_hdr_offset(unsigned int name_len); + +void mail_index_fsck_locked(struct mail_index *index); + +/* Log an error and set it as the index's current error that is available + with mail_index_get_error_message(). */ +void mail_index_set_error(struct mail_index *index, const char *fmt, ...) + ATTR_FORMAT(2, 3) ATTR_COLD; +/* Same as mail_index_set_error(), but don't log the error. */ +void mail_index_set_error_nolog(struct mail_index *index, const char *str) + ATTR_COLD; +/* "%s failed with index file %s: %m" */ +void mail_index_set_syscall_error(struct mail_index *index, + const char *function) ATTR_COLD; +/* "%s failed with file %s: %m" */ +void mail_index_file_set_syscall_error(struct mail_index *index, + const char *filepath, + const char *function) ATTR_COLD; + +#endif diff --git a/src/lib-index/mail-index-strmap.c b/src/lib-index/mail-index-strmap.c new file mode 100644 index 0000000..1287c73 --- /dev/null +++ b/src/lib-index/mail-index-strmap.c @@ -0,0 +1,1259 @@ +/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "bsearch-insert-pos.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "file-dotlock.h" +#include "crc32.h" +#include "safe-mkstemp.h" +#include "str.h" +#include "mail-index-private.h" +#include "mail-index-strmap.h" + +#include <stdio.h> + +struct mail_index_strmap { + struct mail_index *index; + char *path; + int fd; + struct istream *input; + + struct file_lock *file_lock; + struct dotlock *dotlock; + struct dotlock_settings dotlock_settings; +}; + +struct mail_index_strmap_view { + struct mail_index_strmap *strmap; + struct mail_index_view *view; + + ARRAY_TYPE(mail_index_strmap_rec) recs; + ARRAY(uint32_t) recs_crc32; + struct hash2_table *hash; + + mail_index_strmap_key_cmp_t *key_compare; + mail_index_strmap_rec_cmp_t *rec_compare; + mail_index_strmap_remap_t *remap_cb; + void *cb_context; + + uoff_t last_read_block_offset; + uint32_t last_read_uid; + uint32_t last_added_uid; + uint32_t total_ref_count; + + uint32_t last_ref_index; + uint32_t next_str_idx; + uint32_t lost_expunged_uid; + + bool desynced:1; +}; + +struct mail_index_strmap_read_context { + struct mail_index_strmap_view *view; + + struct istream *input; + uoff_t end_offset; + uint32_t highest_str_idx; + uint32_t uid_lookup_seq; + uint32_t lost_expunged_uid; + + const unsigned char *data, *end, *str_idx_base; + struct mail_index_strmap_rec rec; + uint32_t next_ref_index; + unsigned int rec_size; + + bool too_large_uids:1; +}; + +struct mail_index_strmap_view_sync { + struct mail_index_strmap_view *view; +}; + +struct mail_index_strmap_hash_key { + const char *str; + uint32_t crc32; +}; + +/* number of bytes required to store one string idx */ +#define STRMAP_FILE_STRIDX_SIZE (sizeof(uint32_t)*2) + +/* renumber the string indexes when highest string idx becomes larger than + <number of indexes>*STRMAP_FILE_MAX_STRIDX_MULTIPLIER */ +#define STRMAP_FILE_MAX_STRIDX_MULTIPLIER 2 + +#define STRIDX_MUST_RENUMBER(highest_idx, n_unique_indexes) \ + (highest_idx > n_unique_indexes * STRMAP_FILE_MAX_STRIDX_MULTIPLIER) + +#define MAIL_INDEX_STRMAP_TIMEOUT_SECS 10 + +static const struct dotlock_settings default_dotlock_settings = { + .timeout = MAIL_INDEX_STRMAP_TIMEOUT_SECS, + .stale_timeout = 30 +}; + +struct mail_index_strmap * +mail_index_strmap_init(struct mail_index *index, const char *suffix) +{ + struct mail_index_strmap *strmap; + + i_assert(index->open_count > 0); + + strmap = i_new(struct mail_index_strmap, 1); + strmap->index = index; + strmap->path = i_strconcat(index->filepath, suffix, NULL); + strmap->fd = -1; + + strmap->dotlock_settings = default_dotlock_settings; + strmap->dotlock_settings.use_excl_lock = + (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0; + strmap->dotlock_settings.nfs_flush = + (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0; + return strmap; +} + +static bool +mail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx, + uint32_t *crc32_r); + +static void +mail_index_strmap_set_syscall_error(struct mail_index_strmap *strmap, + const char *function) +{ + i_assert(function != NULL); + + if (ENOSPACE(errno)) { + strmap->index->last_error.nodiskspace = TRUE; + if ((strmap->index->flags & + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) == 0) + return; + } + + mail_index_set_error(strmap->index, + "%s failed with strmap index file %s: %m", + function, strmap->path); +} + +static void mail_index_strmap_close(struct mail_index_strmap *strmap) +{ + if (strmap->file_lock != NULL) + file_lock_free(&strmap->file_lock); + else if (strmap->dotlock != NULL) + file_dotlock_delete(&strmap->dotlock); + + if (strmap->fd != -1) { + if (close(strmap->fd) < 0) + mail_index_strmap_set_syscall_error(strmap, "close()"); + strmap->fd = -1; + } + i_stream_unref(&strmap->input); +} + +void mail_index_strmap_deinit(struct mail_index_strmap **_strmap) +{ + struct mail_index_strmap *strmap = *_strmap; + + *_strmap = NULL; + mail_index_strmap_close(strmap); + i_free(strmap->path); + i_free(strmap); +} + +static unsigned int mail_index_strmap_hash_key(const void *_key) +{ + const struct mail_index_strmap_hash_key *key = _key; + + return key->crc32; +} + +static bool +mail_index_strmap_hash_cmp(const void *_key, const void *_value, void *context) +{ + const struct mail_index_strmap_hash_key *key = _key; + const struct mail_index_strmap_rec *rec = _value; + struct mail_index_strmap_view *view = context; + + return view->key_compare(key->str, rec, view->cb_context); +} + +struct mail_index_strmap_view * +mail_index_strmap_view_open(struct mail_index_strmap *strmap, + struct mail_index_view *idx_view, + mail_index_strmap_key_cmp_t *key_compare_cb, + mail_index_strmap_rec_cmp_t *rec_compare_cb, + mail_index_strmap_remap_t *remap_cb, + void *context, + const ARRAY_TYPE(mail_index_strmap_rec) **recs_r, + const struct hash2_table **hash_r) +{ + struct mail_index_strmap_view *view; + + view = i_new(struct mail_index_strmap_view, 1); + view->strmap = strmap; + view->view = idx_view; + view->key_compare = key_compare_cb; + view->rec_compare = rec_compare_cb; + view->remap_cb = remap_cb; + view->cb_context = context; + view->next_str_idx = 1; + + i_array_init(&view->recs, 64); + i_array_init(&view->recs_crc32, 64); + view->hash = hash2_create(0, sizeof(struct mail_index_strmap_rec), + mail_index_strmap_hash_key, + mail_index_strmap_hash_cmp, view); + *recs_r = &view->recs; + *hash_r = view->hash; + return view; +} + +void mail_index_strmap_view_close(struct mail_index_strmap_view **_view) +{ + struct mail_index_strmap_view *view = *_view; + + *_view = NULL; + array_free(&view->recs); + array_free(&view->recs_crc32); + hash2_destroy(&view->hash); + i_free(view); +} + +uint32_t mail_index_strmap_view_get_highest_idx(struct mail_index_strmap_view *view) +{ + return view->next_str_idx-1; +} + +static void mail_index_strmap_view_reset(struct mail_index_strmap_view *view) +{ + view->remap_cb(NULL, 0, 0, view->cb_context); + array_clear(&view->recs); + array_clear(&view->recs_crc32); + hash2_clear(view->hash); + + view->last_added_uid = 0; + view->lost_expunged_uid = 0; + view->desynced = FALSE; +} + +void mail_index_strmap_view_set_corrupted(struct mail_index_strmap_view *view) +{ + mail_index_set_error(view->strmap->index, + "Corrupted strmap index file: %s", + view->strmap->path); + i_unlink(view->strmap->path); + mail_index_strmap_close(view->strmap); + mail_index_strmap_view_reset(view); +} + +static int mail_index_strmap_open(struct mail_index_strmap_view *view) +{ + struct mail_index_strmap *strmap = view->strmap; + const struct mail_index_header *idx_hdr; + struct mail_index_strmap_header hdr; + const unsigned char *data; + size_t size; + int ret; + + i_assert(strmap->fd == -1); + + strmap->fd = open(strmap->path, O_RDWR); + if (strmap->fd == -1) { + if (errno == ENOENT) + return 0; + mail_index_strmap_set_syscall_error(strmap, "open()"); + return -1; + } + strmap->input = i_stream_create_fd(strmap->fd, SIZE_MAX); + ret = i_stream_read_bytes(strmap->input, &data, &size, sizeof(hdr)); + if (ret <= 0) { + if (ret < 0) { + mail_index_strmap_set_syscall_error(strmap, "read()"); + mail_index_strmap_close(strmap); + } else { + i_assert(ret == 0); + mail_index_strmap_view_set_corrupted(view); + } + return ret; + } + memcpy(&hdr, data, sizeof(hdr)); + + idx_hdr = mail_index_get_header(view->view); + if (hdr.version != MAIL_INDEX_STRMAP_VERSION || + hdr.uid_validity != idx_hdr->uid_validity) { + /* need to rebuild. if we already had something in the strmap, + we can keep it. */ + i_unlink(strmap->path); + mail_index_strmap_close(strmap); + return 0; + } + + /* we'll read the entire file from the beginning */ + view->last_added_uid = 0; + view->last_read_uid = 0; + view->total_ref_count = 0; + view->last_read_block_offset = sizeof(struct mail_index_strmap_header); + view->next_str_idx = 1; + + mail_index_strmap_view_reset(view); + return 0; +} + +static bool mail_index_strmap_need_reopen(struct mail_index_strmap *strmap) +{ + struct stat st1, st2; + + /* FIXME: nfs flush */ + if (fstat(strmap->fd, &st1) < 0) { + if (!ESTALE_FSTAT(errno)) + mail_index_strmap_set_syscall_error(strmap, "fstat()"); + return TRUE; + } + if (stat(strmap->path, &st2) < 0) { + mail_index_strmap_set_syscall_error(strmap, "stat()"); + return TRUE; + } + return st1.st_ino != st2.st_ino || !CMP_DEV_T(st1.st_dev, st2.st_dev); +} + +static int mail_index_strmap_refresh(struct mail_index_strmap_view *view) +{ + uint32_t seq; + + if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index)) + return -1; + + if (view->strmap->fd != -1) { + if (!mail_index_strmap_need_reopen(view->strmap)) { + if (view->lost_expunged_uid != 0) { + /* last read failed because view had a message + that didn't exist in the strmap (because it + was expunged by another session). if the + message still isn't expunged in this view, + just continue using the current strmap. */ + if (mail_index_lookup_seq(view->view, + view->lost_expunged_uid, &seq)) + return -1; + } else if (view->desynced) { + /* our view isn't synced with the disk, we + can't read strmap without first resetting + the view */ + } else { + i_stream_sync(view->strmap->input); + return 0; + } + } + mail_index_strmap_close(view->strmap); + } + + return mail_index_strmap_open(view); +} + +static int +mail_index_strmap_read_packed(struct mail_index_strmap_read_context *ctx, + uint32_t *num_r) +{ + const unsigned char *data; + const uint8_t *bytes, *p, *end; + size_t size; + int ret; + + ret = i_stream_read_bytes(ctx->input, &data, &size, sizeof(*num_r)); + if (ret <= 0) + return ret; + + if (ctx->input->v_offset + size > ctx->end_offset) + size = ctx->end_offset - ctx->input->v_offset; + bytes = p = (const uint8_t *)data; + end = bytes + size; + + if (mail_index_unpack_num(&p, end, num_r) < 0) + return -1; + i_stream_skip(ctx->input, p - bytes); + return 1; +} + +static int +mail_index_strmap_uid_exists(struct mail_index_strmap_read_context *ctx, + uint32_t uid) +{ + const struct mail_index_record *rec; + + i_assert(ctx->uid_lookup_seq > 0); + + if (ctx->uid_lookup_seq > ctx->view->view->map->hdr.messages_count) { + if (uid >= ctx->view->view->map->hdr.next_uid) { + /* thread index has larger UIDs than what we've seen + in our view. we'll have to read them again later + when we know about them */ + ctx->too_large_uids = TRUE; + } + return 0; + } + + rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->view->map, ctx->uid_lookup_seq); + if (rec->uid == uid) { + ctx->uid_lookup_seq++; + return 1; + } else if (rec->uid > uid) { + return 0; + } else { + /* record that exists in index is missing from strmap. + see if it's because the strmap is corrupted or because + our current view is a bit stale and the message has already + been expunged. */ + mail_index_refresh(ctx->view->view->index); + if (mail_index_is_expunged(ctx->view->view, + ctx->uid_lookup_seq)) + ctx->lost_expunged_uid = rec->uid; + return -1; + } +} + +static int +mail_index_strmap_read_rec_first(struct mail_index_strmap_read_context *ctx, + uint32_t *crc32_r) +{ + size_t size; + uint32_t n, i, count, str_idx; + int ret; + + /* <uid> <n> <crc32>*count <str_idx>*count + where + n = 0 -> count=1 (only Message-ID:) + n = 1 -> count=2 (Message-ID: + In-Reply-To:) + n = 2+ -> count=n (Message-ID: + References:) + */ + if (mail_index_strmap_read_packed(ctx, &n) <= 0) + return -1; + count = n < 2 ? n + 1 : n; + ctx->view->total_ref_count += count; + + ctx->rec_size = count * (sizeof(ctx->rec.str_idx) + sizeof(*crc32_r)); + ret = mail_index_strmap_uid_exists(ctx, ctx->rec.uid); + if (ret < 0) + return -1; + if (i_stream_read_bytes(ctx->view->strmap->input, &ctx->data, &size, ctx->rec_size) <= 0) + return -1; + ctx->str_idx_base = ctx->data + count * sizeof(uint32_t); + + if (ret == 0) { + /* this message has already been expunged, ignore it. + update highest string indexes anyway. */ + for (i = 0; i < count; i++) { + memcpy(&str_idx, ctx->str_idx_base, sizeof(str_idx)); + if (ctx->highest_str_idx < str_idx) + ctx->highest_str_idx = str_idx; + ctx->str_idx_base += sizeof(str_idx); + } + i_stream_skip(ctx->view->strmap->input, ctx->rec_size); + return 0; + } + + /* everything exists. save it. FIXME: these ref_index values + are thread index specific, perhaps something more generic + should be used some day */ + ctx->end = ctx->data + count * sizeof(*crc32_r); + + ctx->next_ref_index = 0; + if (!mail_index_strmap_read_rec_next(ctx, crc32_r)) + i_unreached(); + ctx->next_ref_index = n == 1 ? 1 : 2; + return 1; +} + +static bool +mail_index_strmap_read_rec_next(struct mail_index_strmap_read_context *ctx, + uint32_t *crc32_r) +{ + if (ctx->data == ctx->end) { + i_stream_skip(ctx->view->strmap->input, ctx->rec_size); + return FALSE; + } + + /* FIXME: str_idx could be stored as packed relative values + (first relative to highest_idx, the rest relative to the + previous str_idx) */ + + /* read the record contents */ + memcpy(&ctx->rec.str_idx, ctx->str_idx_base, sizeof(ctx->rec.str_idx)); + memcpy(crc32_r, ctx->data, sizeof(*crc32_r)); + + ctx->rec.ref_index = ctx->next_ref_index++; + + if (ctx->highest_str_idx < ctx->rec.str_idx) + ctx->highest_str_idx = ctx->rec.str_idx; + + /* get to the next record */ + ctx->data += sizeof(*crc32_r); + ctx->str_idx_base += sizeof(ctx->rec.str_idx); + return TRUE; +} + +static int +strmap_read_block_init(struct mail_index_strmap_view *view, + struct mail_index_strmap_read_context *ctx) +{ + struct mail_index_strmap *strmap = view->strmap; + const unsigned char *data; + size_t size; + uint32_t block_size, seq1, seq2; + int ret; + + if (view->last_read_uid + 1 >= view->view->map->hdr.next_uid) { + /* come back later when we know about the new UIDs */ + return 0; + } + + i_zero(ctx); + ret = i_stream_read_bytes(strmap->input, &data, &size, + sizeof(block_size)); + if (ret <= 0) { + if (strmap->input->stream_errno == 0) { + /* no new data */ + return 0; + } + mail_index_strmap_set_syscall_error(strmap, "read()"); + return -1; + } + memcpy(&block_size, data, sizeof(block_size)); + block_size = mail_index_offset_to_uint32(block_size) >> 2; + if (block_size == 0) { + /* the rest of the file is either not written, or the previous + write didn't finish */ + return 0; + } + i_stream_skip(strmap->input, sizeof(block_size)); + + ctx->view = view; + ctx->input = strmap->input; + ctx->end_offset = strmap->input->v_offset + block_size; + if (ctx->end_offset < strmap->input->v_offset) { + /* block size too large */ + mail_index_strmap_view_set_corrupted(view); + return -1; + } + ctx->rec.uid = view->last_read_uid + 1; + + /* FIXME: when reading multiple blocks we shouldn't have to calculate + this every time */ + if (!mail_index_lookup_seq_range(view->view, ctx->rec.uid, (uint32_t)-1, + &seq1, &seq2)) + seq1 = mail_index_view_get_messages_count(view->view) + 1; + ctx->uid_lookup_seq = seq1; + return 1; +} + +static int +strmap_read_block_next(struct mail_index_strmap_read_context *ctx, + uint32_t *crc32_r) +{ + uint32_t uid_diff; + int ret; + + if (mail_index_strmap_read_rec_next(ctx, crc32_r)) + return 1; + + /* get next UID */ + do { + if (ctx->input->v_offset == ctx->end_offset) { + /* this block is done */ + return 0; + } + if (mail_index_strmap_read_packed(ctx, &uid_diff) <= 0) + return -1; + + ctx->rec.uid += uid_diff; + ret = mail_index_strmap_read_rec_first(ctx, crc32_r); + } while (ret == 0); + return ret; +} + +static int +strmap_read_block_deinit(struct mail_index_strmap_read_context *ctx, int ret, + bool update_block_offset) +{ + struct mail_index_strmap_view *view = ctx->view; + struct mail_index_strmap *strmap = view->strmap; + + if (ctx->highest_str_idx > view->total_ref_count) { + /* if all string indexes are unique, highest_str_index equals + total_ref_count. otherwise it's always lower. */ + mail_index_set_error(strmap->index, + "Corrupted strmap index file %s: " + "String indexes too high " + "(highest=%u max=%u)", + strmap->path, ctx->highest_str_idx, + view->total_ref_count); + mail_index_strmap_view_set_corrupted(view); + return -1; + } + if (ctx->lost_expunged_uid != 0) { + /* our view contained a message that had since been expunged. */ + i_assert(ret < 0); + view->lost_expunged_uid = ctx->lost_expunged_uid; + } else if (ret < 0) { + if (strmap->input->stream_errno != 0) + mail_index_strmap_set_syscall_error(strmap, "read()"); + else + mail_index_strmap_view_set_corrupted(view); + return -1; + } else if (update_block_offset && !ctx->too_large_uids) { + view->last_read_block_offset = strmap->input->v_offset; + view->last_read_uid = ctx->rec.uid; + } + if (view->next_str_idx <= ctx->highest_str_idx) + view->next_str_idx = ctx->highest_str_idx + 1; + return ret; +} + +static bool +strmap_view_sync_handle_conflict(struct mail_index_strmap_read_context *ctx, + const struct mail_index_strmap_rec *hash_rec, + struct hash2_iter *iter) +{ + uint32_t seq; + + /* hopefully it's a message that has since been expunged */ + if (!mail_index_lookup_seq(ctx->view->view, hash_rec->uid, &seq)) { + /* message is no longer in our view. remove it completely. */ + hash2_remove_iter(ctx->view->hash, iter); + return TRUE; + } + if (mail_index_is_expunged(ctx->view->view, seq)) { + /* it's quite likely a conflict. we may not be able to verify + it, so just assume it is. nothing breaks even if we guess + wrong, the performance just suffers a bit. */ + return FALSE; + } + + /* 0 means "doesn't match", which is the only acceptable case */ + return ctx->view->rec_compare(&ctx->rec, hash_rec, + ctx->view->cb_context) == 0; +} + +static int +strmap_view_sync_block_check_conflicts(struct mail_index_strmap_read_context *ctx, + uint32_t crc32) +{ + struct mail_index_strmap_rec *hash_rec; + struct hash2_iter iter; + + if (crc32 == 0) { + /* unique string - there are no conflicts */ + return 0; + } + + /* check for conflicting string indexes. they may happen if + + 1) msgid exists only for a message X that has been expunged + 2) another process doesn't see X, but sees msgid for another + message and writes it using a new string index + 3) if we still see X, we now see the same msgid with two + string indexes. + + if we detect such a conflict, we can't continue using the + strmap index until X has been expunged. */ + i_zero(&iter); + while ((hash_rec = hash2_iterate(ctx->view->hash, + crc32, &iter)) != NULL && + hash_rec->str_idx != ctx->rec.str_idx) { + /* CRC32 matches, but string index doesn't */ + if (!strmap_view_sync_handle_conflict(ctx, hash_rec, &iter)) { + ctx->lost_expunged_uid = hash_rec->uid; + return -1; + } + } + return 0; +} + +static int +mail_index_strmap_view_sync_block(struct mail_index_strmap_read_context *ctx) +{ + struct mail_index_strmap_rec *hash_rec; + uint32_t crc32, prev_uid = 0; + int ret; + + while ((ret = strmap_read_block_next(ctx, &crc32)) > 0) { + if (ctx->rec.uid <= ctx->view->last_added_uid) { + if (ctx->rec.uid < ctx->view->last_added_uid || + prev_uid != ctx->rec.uid) { + /* we've already added this */ + continue; + } + } + prev_uid = ctx->rec.uid; + + if (strmap_view_sync_block_check_conflicts(ctx, crc32) < 0) { + ret = -1; + break; + } + ctx->view->last_added_uid = ctx->rec.uid; + + /* add the record to records array */ + array_push_back(&ctx->view->recs, &ctx->rec); + array_push_back(&ctx->view->recs_crc32, &crc32); + + /* add a separate copy of the record to hash */ + hash_rec = hash2_insert_hash(ctx->view->hash, crc32); + memcpy(hash_rec, &ctx->rec, sizeof(*hash_rec)); + } + return strmap_read_block_deinit(ctx, ret, TRUE); +} + +struct mail_index_strmap_view_sync * +mail_index_strmap_view_sync_init(struct mail_index_strmap_view *view, + uint32_t *last_uid_r) +{ + struct mail_index_strmap_view_sync *sync; + struct mail_index_strmap_read_context ctx; + int ret; + + sync = i_new(struct mail_index_strmap_view_sync, 1); + sync->view = view; + + if (mail_index_strmap_refresh(view) < 0) { + /* reading the strmap failed - just ignore and do + this in-memory based on whatever we knew last */ + } else if (view->strmap->input != NULL) { + i_stream_seek(view->strmap->input, + view->last_read_block_offset); + while ((ret = strmap_read_block_init(view, &ctx)) > 0) { + if (mail_index_strmap_view_sync_block(&ctx) < 0) { + ret = -1; + break; + } + if (ctx.too_large_uids) + break; + } + + if (ret < 0) { + /* something failed - we can still use the strmap as far + as we managed to read it, but our view is now out + of sync */ + view->desynced = TRUE; + } else { + i_assert(view->lost_expunged_uid == 0); + } + } + *last_uid_r = view->last_added_uid; + return sync; +} + +static inline uint32_t crc32_str_nonzero(const char *str) +{ + /* we'll flip the bits because of a bug in our old crc32 code. + this keeps the index format backwards compatible with the new fixed + crc32 code. */ + uint32_t value = crc32_str(str) ^ 0xffffffffU; + return value == 0 ? 1 : value; +} + +void mail_index_strmap_view_sync_add(struct mail_index_strmap_view_sync *sync, + uint32_t uid, uint32_t ref_index, + const char *key) +{ + struct mail_index_strmap_view *view = sync->view; + struct mail_index_strmap_rec *rec, *old_rec; + struct mail_index_strmap_hash_key hash_key; + uint32_t str_idx; + + i_assert(uid > view->last_added_uid || + (uid == view->last_added_uid && + ref_index > view->last_ref_index)); + + hash_key.str = key; + hash_key.crc32 = crc32_str_nonzero(key); + + old_rec = hash2_lookup(view->hash, &hash_key); + if (old_rec != NULL) { + /* The string already exists, use the same unique idx */ + str_idx = old_rec->str_idx; + } else { + /* Newly seen string, assign a new unique idx to it */ + str_idx = view->next_str_idx++; + } + i_assert(str_idx != 0); + + rec = hash2_insert(view->hash, &hash_key); + rec->uid = uid; + rec->ref_index = ref_index; + rec->str_idx = str_idx; + array_push_back(&view->recs, rec); + array_push_back(&view->recs_crc32, &hash_key.crc32); + + view->last_added_uid = uid; + view->last_ref_index = ref_index; +} + +void mail_index_strmap_view_sync_add_unique(struct mail_index_strmap_view_sync *sync, + uint32_t uid, uint32_t ref_index) +{ + struct mail_index_strmap_view *view = sync->view; + struct mail_index_strmap_rec rec; + + i_assert(uid > view->last_added_uid || + (uid == view->last_added_uid && + ref_index > view->last_ref_index)); + + i_zero(&rec); + rec.uid = uid; + rec.ref_index = ref_index; + rec.str_idx = view->next_str_idx++; + array_push_back(&view->recs, &rec); + array_append_zero(&view->recs_crc32); + + view->last_added_uid = uid; + view->last_ref_index = ref_index; +} + +static void +mail_index_strmap_zero_terminate(struct mail_index_strmap_view *view) +{ + /* zero-terminate the records array */ + array_append_zero(&view->recs); + array_pop_back(&view->recs); +} + +static void mail_index_strmap_view_renumber(struct mail_index_strmap_view *view) +{ + struct mail_index_strmap_read_context ctx; + struct mail_index_strmap_rec *recs, *hash_rec; + uint32_t prev_uid, str_idx, *recs_crc32, *renumber_map; + unsigned int i, dest, count, count2; + int ret; + + i_zero(&ctx); + ctx.view = view; + ctx.uid_lookup_seq = 1; + + /* create a map of old -> new index and remove records of + expunged messages */ + renumber_map = i_new(uint32_t, view->next_str_idx); + str_idx = 0; prev_uid = 0; + recs = array_get_modifiable(&view->recs, &count); + recs_crc32 = array_get_modifiable(&view->recs_crc32, &count2); + i_assert(count == count2); + + for (i = dest = 0; i < count; ) { + if (prev_uid != recs[i].uid) { + /* see if this record should be removed */ + prev_uid = recs[i].uid; + ret = mail_index_strmap_uid_exists(&ctx, prev_uid); + i_assert(ret >= 0); + if (ret == 0) { + /* message expunged */ + do { + i++; + } while (i < count && recs[i].uid == prev_uid); + continue; + } + } + + i_assert(recs[i].str_idx < view->next_str_idx); + if (renumber_map[recs[i].str_idx] == 0) + renumber_map[recs[i].str_idx] = ++str_idx; + if (i != dest) { + recs[dest] = recs[i]; + recs_crc32[dest] = recs_crc32[i]; + } + i++; dest++; + } + i_assert(renumber_map[0] == 0); + array_delete(&view->recs, dest, i-dest); + array_delete(&view->recs_crc32, dest, i-dest); + mail_index_strmap_zero_terminate(view); + + /* notify caller of the renumbering */ + i_assert(str_idx <= view->next_str_idx); + view->remap_cb(renumber_map, view->next_str_idx, str_idx + 1, + view->cb_context); + + /* renumber the indexes in-place and recreate the hash */ + recs = array_get_modifiable(&view->recs, &count); + hash2_clear(view->hash); + for (i = 0; i < count; i++) { + recs[i].str_idx = renumber_map[recs[i].str_idx]; + hash_rec = hash2_insert_hash(view->hash, recs_crc32[i]); + memcpy(hash_rec, &recs[i], sizeof(*hash_rec)); + } + + /* update the new next_str_idx only after remapping */ + view->next_str_idx = str_idx + 1; + i_free(renumber_map); +} + +static void mail_index_strmap_write_block(struct mail_index_strmap_view *view, + struct ostream *output, + unsigned int i, uint32_t base_uid) +{ + const struct mail_index_strmap_rec *recs; + const uint32_t *crc32; + unsigned int j, n, count, count2, uid_rec_count; + uint32_t block_size; + uint8_t *p, packed[MAIL_INDEX_PACK_MAX_SIZE*2]; + uoff_t block_offset, end_offset; + + /* skip over the block size for now, we don't know it yet */ + block_offset = output->offset; + block_size = 0; + o_stream_nsend(output, &block_size, sizeof(block_size)); + + /* write records */ + recs = array_get(&view->recs, &count); + crc32 = array_get(&view->recs_crc32, &count2); + i_assert(count == count2); + while (i < count) { + /* @UNSAFE: <uid diff> */ + p = packed; + mail_index_pack_num(&p, recs[i].uid - base_uid); + base_uid = recs[i].uid; + + /* find how many records belong to this UID */ + uid_rec_count = 1; + for (j = i + 1; j < count; j++) { + if (recs[j].uid != base_uid) + break; + uid_rec_count++; + } + view->total_ref_count += uid_rec_count; + + /* <n> <crc32>*count <str_idx>*count - + FIXME: thread index specific code */ + i_assert(recs[i].ref_index == 0); + if (uid_rec_count == 1) { + /* Only Message-ID: header */ + n = 0; + } else if (recs[i+1].ref_index == 1) { + /* In-Reply-To: header */ + n = 1; + i_assert(uid_rec_count == 2); + } else { + /* References: header */ + n = uid_rec_count; + i_assert(recs[i+1].ref_index == 2); + } + + mail_index_pack_num(&p, n); + o_stream_nsend(output, packed, p-packed); + for (j = 0; j < uid_rec_count; j++) + o_stream_nsend(output, &crc32[i+j], sizeof(crc32[i+j])); + for (j = 0; j < uid_rec_count; j++) { + i_assert(j < 2 || recs[i+j].ref_index == j+1); + o_stream_nsend(output, &recs[i+j].str_idx, + sizeof(recs[i+j].str_idx)); + } + i += uid_rec_count; + } + + /* we know the block size now - write it */ + block_size = output->offset - (block_offset + sizeof(block_size)); + block_size = mail_index_uint32_to_offset(block_size << 2); + i_assert(block_size != 0); + + end_offset = output->offset; + (void)o_stream_seek(output, block_offset); + o_stream_nsend(output, &block_size, sizeof(block_size)); + (void)o_stream_seek(output, end_offset); + + if (output->stream_errno != 0) + return; + + i_assert(view->last_added_uid == recs[count-1].uid); + view->last_read_uid = recs[count-1].uid; + view->last_read_block_offset = output->offset; +} + +static void +mail_index_strmap_recreate_write(struct mail_index_strmap_view *view, + struct ostream *output) +{ + const struct mail_index_header *idx_hdr; + struct mail_index_strmap_header hdr; + + idx_hdr = mail_index_get_header(view->view); + + /* write header */ + i_zero(&hdr); + hdr.version = MAIL_INDEX_STRMAP_VERSION; + hdr.uid_validity = idx_hdr->uid_validity; + o_stream_nsend(output, &hdr, sizeof(hdr)); + + view->total_ref_count = 0; + mail_index_strmap_write_block(view, output, 0, 1); +} + +static int mail_index_strmap_recreate(struct mail_index_strmap_view *view) +{ + struct mail_index_strmap *strmap = view->strmap; + string_t *str; + struct ostream *output; + const char *temp_path; + int fd, ret = 0; + + if (array_count(&view->recs) == 0) { + /* everything expunged - just unlink the existing index */ + if (unlink(strmap->path) < 0 && errno != ENOENT) + mail_index_strmap_set_syscall_error(strmap, "unlink()"); + return 0; + } + + str = t_str_new(256); + str_append(str, strmap->path); + fd = safe_mkstemp_hostpid_group(str, view->view->index->set.mode, + view->view->index->set.gid, + view->view->index->set.gid_origin); + temp_path = str_c(str); + + if (fd == -1) { + mail_index_set_error(strmap->index, + "safe_mkstemp_hostpid(%s) failed: %m", + temp_path); + return -1; + } + output = o_stream_create_fd(fd, 0); + o_stream_cork(output); + mail_index_strmap_recreate_write(view, output); + if (o_stream_finish(output) < 0) { + mail_index_set_error(strmap->index, "write(%s) failed: %s", + temp_path, o_stream_get_error(output)); + ret = -1; + } + o_stream_destroy(&output); + if (close(fd) < 0) { + mail_index_set_error(strmap->index, + "close(%s) failed: %m", temp_path); + ret = -1; + } else if (ret == 0 && rename(temp_path, strmap->path) < 0) { + mail_index_set_error(strmap->index, + "rename(%s, %s) failed: %m", + temp_path, strmap->path); + ret = -1; + } + if (ret < 0) + i_unlink(temp_path); + return ret; +} + +static int mail_index_strmap_lock(struct mail_index_strmap *strmap) +{ + unsigned int timeout_secs; + const char *error; + int ret; + + i_assert(strmap->fd != -1); + + if (strmap->index->set.lock_method != FILE_LOCK_METHOD_DOTLOCK) { + i_assert(strmap->file_lock == NULL); + + struct file_lock_settings lock_set = { + .lock_method = strmap->index->set.lock_method, + }; + timeout_secs = I_MIN(MAIL_INDEX_STRMAP_TIMEOUT_SECS, + strmap->index->set.max_lock_timeout_secs); + ret = file_wait_lock(strmap->fd, strmap->path, F_WRLCK, + &lock_set, timeout_secs, + &strmap->file_lock, &error); + if (ret <= 0) { + mail_index_set_error(strmap->index, + "file_wait_lock() failed with strmap index file %s: %s", + strmap->path, error); + } + } else { + i_assert(strmap->dotlock == NULL); + + ret = file_dotlock_create(&strmap->dotlock_settings, + strmap->path, 0, &strmap->dotlock); + if (ret <= 0) { + mail_index_strmap_set_syscall_error(strmap, + "file_dotlock_create()"); + } + } + return ret; +} + +static void mail_index_strmap_unlock(struct mail_index_strmap *strmap) +{ + if (strmap->file_lock != NULL) + file_unlock(&strmap->file_lock); + else if (strmap->dotlock != NULL) + file_dotlock_delete(&strmap->dotlock); +} + +static int +strmap_rec_cmp(const uint32_t *uid, const struct mail_index_strmap_rec *rec) +{ + return *uid < rec->uid ? -1 : + (*uid > rec->uid ? 1 : 0); +} + +static int +mail_index_strmap_write_append(struct mail_index_strmap_view *view) +{ + struct mail_index_strmap_read_context ctx; + const struct mail_index_strmap_rec *old_recs; + unsigned int i, old_count; + struct ostream *output; + uint32_t crc32, next_uid; + bool full_block; + int ret; + + /* Check first if another process had written new records to the file. + If there are any, hopefully they're the same as what we would be + writing. There are two problematic cases when messages have been + expunged recently: + + 1) The file contains UIDs that we don't have. This means the string + indexes won't be compatible anymore, so we'll have to renumber ours + to match the ones in the strmap file. + + Currently we don't bother handling 1) case. If indexes don't match + what we have, we just don't write anything. + + 2) We have UIDs that don't exist in the file. We can't simply skip + those records, because other records may have pointers to them using + different string indexes than we have. Even if we renumbered those, + future appends by other processes might cause the same problem (they + see the string for the first time and assign it a new index, but we + already have internally given it another index). So the only + sensible choice is to write nothing and hope that the message goes + away soon. */ + next_uid = view->last_read_uid + 1; + (void)array_bsearch_insert_pos(&view->recs, &next_uid, + strmap_rec_cmp, &i); + + old_recs = array_get(&view->recs, &old_count); + if (i < old_count) { + while (i > 0 && old_recs[i-1].uid == old_recs[i].uid) + i--; + } + + i_stream_sync(view->strmap->input); + i_stream_seek(view->strmap->input, view->last_read_block_offset); + full_block = TRUE; ret = 0; + while (i < old_count && + (ret = strmap_read_block_init(view, &ctx)) > 0) { + while ((ret = strmap_read_block_next(&ctx, &crc32)) > 0) { + if (ctx.rec.uid != old_recs[i].uid || + ctx.rec.str_idx != old_recs[i].str_idx) { + /* mismatch */ + if (ctx.rec.uid > old_recs[i].uid) { + /* 1) case */ + ctx.lost_expunged_uid = ctx.rec.uid; + } else if (ctx.rec.uid < old_recs[i].uid) { + /* 2) case */ + ctx.lost_expunged_uid = old_recs[i].uid; + } else { + /* string index mismatch, + shouldn't happen */ + } + ret = -1; + break; + } + if (++i == old_count) { + full_block = FALSE; + break; + } + } + if (strmap_read_block_deinit(&ctx, ret, full_block) < 0) { + ret = -1; + break; + } + } + if (ret < 0) + return -1; + if (i == old_count) { + /* nothing new to write */ + return 0; + } + i_assert(full_block); + i_assert(old_recs[i].uid > view->last_read_uid); + + /* write the new records */ + output = o_stream_create_fd(view->strmap->fd, 0); + (void)o_stream_seek(output, view->last_read_block_offset); + o_stream_cork(output); + mail_index_strmap_write_block(view, output, i, + view->last_read_uid + 1); + if (o_stream_finish(output) < 0) { + mail_index_strmap_set_syscall_error(view->strmap, "write()"); + ret = -1; + } + o_stream_destroy(&output); + return ret; +} + +static int mail_index_strmap_write(struct mail_index_strmap_view *view) +{ + int ret; + + /* FIXME: this renumbering doesn't work well when running for a long + time since records aren't removed from hash often enough */ + if (STRIDX_MUST_RENUMBER(view->next_str_idx - 1, + hash2_count(view->hash))) { + mail_index_strmap_view_renumber(view); + if (!MAIL_INDEX_IS_IN_MEMORY(view->strmap->index)) { + if (mail_index_strmap_recreate(view) < 0) { + view->desynced = TRUE; + return -1; + } + } + return 0; + } + + if (MAIL_INDEX_IS_IN_MEMORY(view->strmap->index) || view->desynced) + return 0; + + if (view->strmap->fd == -1) { + /* initial file creation */ + if (mail_index_strmap_recreate(view) < 0) { + view->desynced = TRUE; + return -1; + } + return 0; + } + + /* append the new records to the strmap file */ + if (mail_index_strmap_lock(view->strmap) <= 0) { + /* timeout / error */ + ret = -1; + } else if (mail_index_strmap_need_reopen(view->strmap)) { + /* the file was already recreated - leave the syncing as it is + for now and let the next sync re-read the file. */ + ret = 0; + } else { + ret = mail_index_strmap_write_append(view); + } + mail_index_strmap_unlock(view->strmap); + if (ret < 0) + view->desynced = TRUE; + return ret; +} + +void mail_index_strmap_view_sync_commit(struct mail_index_strmap_view_sync **_sync) +{ + struct mail_index_strmap_view_sync *sync = *_sync; + struct mail_index_strmap_view *view = sync->view; + + *_sync = NULL; + i_free(sync); + + (void)mail_index_strmap_write(view); + mail_index_strmap_zero_terminate(view); + + /* zero-terminate the records array */ + array_append_zero(&view->recs); + array_pop_back(&view->recs); +} + +void mail_index_strmap_view_sync_rollback(struct mail_index_strmap_view_sync **_sync) +{ + struct mail_index_strmap_view_sync *sync = *_sync; + + *_sync = NULL; + + mail_index_strmap_view_reset(sync->view); + mail_index_strmap_zero_terminate(sync->view); + i_free(sync); +} diff --git a/src/lib-index/mail-index-strmap.h b/src/lib-index/mail-index-strmap.h new file mode 100644 index 0000000..c61afa6 --- /dev/null +++ b/src/lib-index/mail-index-strmap.h @@ -0,0 +1,81 @@ +#ifndef MAIL_INDEX_STRMAP_H +#define MAIL_INDEX_STRMAP_H + +#include "hash2.h" + +struct mail_index; +struct mail_index_view; + +struct mail_index_strmap_header { +#define MAIL_INDEX_STRMAP_VERSION 1 + uint8_t version; + uint8_t unused[3]; + + uint32_t uid_validity; +}; + +struct mail_index_strmap_rec { + uint32_t uid; + uint32_t ref_index; + /* unique index number for the string */ + uint32_t str_idx; +}; +ARRAY_DEFINE_TYPE(mail_index_strmap_rec, struct mail_index_strmap_rec); + +typedef bool +mail_index_strmap_key_cmp_t(const char *key, + const struct mail_index_strmap_rec *rec, + void *context); +/* Returns 1 if matches, 0 if not, -1 if one of the records is expunged and + the result can't be determined */ +typedef int +mail_index_strmap_rec_cmp_t(const struct mail_index_strmap_rec *rec1, + const struct mail_index_strmap_rec *rec2, + void *context); +/* called when string indexes are renumbered. idx_map[old_idx] = new_idx. + if new_idx is 0, the record was expunged. As a special case if count=0, + the strmap was reset. */ +typedef void mail_index_strmap_remap_t(const uint32_t *idx_map, + unsigned int old_count, + unsigned int new_count, void *context); + +struct mail_index_strmap * +mail_index_strmap_init(struct mail_index *index, const char *suffix); +void mail_index_strmap_deinit(struct mail_index_strmap **strmap); + +/* Returns strmap records and hash that can be used for read-only access. + The records array always terminates with a record containing zeros (but it's + not counted in the array count). */ +struct mail_index_strmap_view * +mail_index_strmap_view_open(struct mail_index_strmap *strmap, + struct mail_index_view *idx_view, + mail_index_strmap_key_cmp_t *key_compare_cb, + mail_index_strmap_rec_cmp_t *rec_compare_cb, + mail_index_strmap_remap_t *remap_cb, + void *context, + const ARRAY_TYPE(mail_index_strmap_rec) **recs_r, + const struct hash2_table **hash_r); +void mail_index_strmap_view_close(struct mail_index_strmap_view **view); +void mail_index_strmap_view_set_corrupted(struct mail_index_strmap_view *view) + ATTR_COLD; + +/* Return the highest used string index. */ +uint32_t mail_index_strmap_view_get_highest_idx(struct mail_index_strmap_view *view); + +/* Synchronize strmap: Caller adds missing entries, expunged messages may be + removed internally and the changes are written to disk. Note that the strmap + recs/hash shouldn't be used until _sync_commit() is called, because the + string indexes may be renumbered if another process had already written the + same changes as us. */ +struct mail_index_strmap_view_sync * +mail_index_strmap_view_sync_init(struct mail_index_strmap_view *view, + uint32_t *last_uid_r); +void mail_index_strmap_view_sync_add(struct mail_index_strmap_view_sync *sync, + uint32_t uid, uint32_t ref_index, + const char *key); +void mail_index_strmap_view_sync_add_unique(struct mail_index_strmap_view_sync *sync, + uint32_t uid, uint32_t ref_index); +void mail_index_strmap_view_sync_commit(struct mail_index_strmap_view_sync **sync); +void mail_index_strmap_view_sync_rollback(struct mail_index_strmap_view_sync **sync); + +#endif diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c new file mode 100644 index 0000000..5d453f3 --- /dev/null +++ b/src/lib-index/mail-index-sync-ext.c @@ -0,0 +1,735 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log.h" + + +void mail_index_sync_init_expunge_handlers(struct mail_index_sync_map_ctx *ctx) +{ + const struct mail_index_ext *ext; + const struct mail_index_registered_ext *rext; + const uint32_t *id_map; + void **contexts; + struct mail_index_expunge_handler eh; + unsigned int ext_count, id_map_count; + unsigned int rext_count, context_count, count; + uint32_t idx_ext_id, map_ext_id; + + if (!array_is_created(&ctx->view->map->extensions)) + return; + + i_zero(&eh); + if (array_is_created(&ctx->expunge_handlers)) + array_clear(&ctx->expunge_handlers); + else + i_array_init(&ctx->expunge_handlers, 64); + + rext = array_get(&ctx->view->index->extensions, &rext_count); + ext = array_get(&ctx->view->map->extensions, &ext_count); + id_map = array_get(&ctx->view->map->ext_id_map, &id_map_count); + contexts = array_get_modifiable(&ctx->extra_contexts, &context_count); + + i_assert(context_count >= rext_count); + count = I_MIN(rext_count, id_map_count); + for (idx_ext_id = 0; idx_ext_id < count; idx_ext_id++) { + if (rext[idx_ext_id].expunge_handler == NULL) + continue; + map_ext_id = id_map[idx_ext_id]; + if (map_ext_id == (uint32_t)-1) + continue; + + eh.handler = rext[idx_ext_id].expunge_handler; + eh.sync_context = &contexts[idx_ext_id]; + eh.record_offset = ext[map_ext_id].record_offset; + array_push_back(&ctx->expunge_handlers, &eh); + } + ctx->expunge_handlers_set = TRUE; + ctx->expunge_handlers_used = TRUE; +} + +void +mail_index_sync_deinit_expunge_handlers(struct mail_index_sync_map_ctx *ctx) +{ + const struct mail_index_expunge_handler *eh; + + if (!array_is_created(&ctx->expunge_handlers)) + return; + + array_foreach(&ctx->expunge_handlers, eh) { + if (eh->sync_context != NULL) + eh->handler(ctx, NULL, eh->sync_context); + } + array_free(&ctx->expunge_handlers); +} + +void mail_index_sync_init_handlers(struct mail_index_sync_map_ctx *ctx) +{ + unsigned int count; + + if (!array_is_created(&ctx->view->map->extensions)) + return; + + /* set space for extra contexts */ + count = array_count(&ctx->view->index->extensions); + i_assert(count > 0); + + if (!array_is_created(&ctx->extra_contexts)) + i_array_init(&ctx->extra_contexts, count); + + /* make sure the extra_contexts contains everything */ + (void)array_idx_get_space(&ctx->extra_contexts, count - 1); + /* we need to update the expunge handler list in case they had + already been called */ + ctx->expunge_handlers_set = FALSE; +} + +void mail_index_sync_deinit_handlers(struct mail_index_sync_map_ctx *ctx) +{ + if (array_is_created(&ctx->extra_contexts)) + array_free(&ctx->extra_contexts); +} + +static struct mail_index_ext_header * +get_ext_header(struct mail_index_map *map, const struct mail_index_ext *ext) +{ + struct mail_index_ext_header *ext_hdr; + void *hdr_base; + + /* do some kludgy jumping to get to it. */ + hdr_base = buffer_get_modifiable_data(map->hdr_copy_buf, NULL); + ext_hdr = PTR_OFFSET(hdr_base, ext->ext_offset); + i_assert(memcmp((char *)(ext_hdr + 1), + ext->name, strlen(ext->name)) == 0); + return ext_hdr; +} + +static int mail_index_ext_align_cmp(const void *p1, const void *p2) +{ + const struct mail_index_ext *const *e1 = p1, *const *e2 = p2; + + return (int)(*e2)->record_align - (int)(*e1)->record_align; +} + +static void sync_ext_reorder(struct mail_index_map *map, uint32_t ext_map_idx, + uint16_t old_ext_size) +{ + struct mail_index_ext *ext, **sorted; + struct mail_index_ext_header *ext_hdr; + uint16_t *old_offsets, *copy_sizes, min_align, max_align; + uint32_t offset, new_record_size, rec_idx; + unsigned int i, count; + const void *src; + buffer_t *new_buffer; + size_t new_buffer_size; + + i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) && map->refcount == 1); + + ext = array_get_modifiable(&map->extensions, &count); + i_assert(ext_map_idx < count); + + /* @UNSAFE */ + old_offsets = t_new(uint16_t, count); + copy_sizes = t_new(uint16_t, count); + sorted = t_new(struct mail_index_ext *, count); + for (i = 0; i < count; i++) { + old_offsets[i] = ext[i].record_offset; + copy_sizes[i] = ext[i].record_size; + ext[i].record_offset = 0; + sorted[i] = &ext[i]; + } + qsort(sorted, count, sizeof(struct mail_index_ext *), + mail_index_ext_align_cmp); + + if (copy_sizes[ext_map_idx] > old_ext_size) { + /* we are growing the extension record. remember this + so we don't write extra data while copying the record */ + copy_sizes[ext_map_idx] = old_ext_size; + } + + /* we simply try to use the extensions with largest alignment + requirement first. FIXME: if the extension sizes don't match + alignment, this may not give the minimal layout. */ + offset = MAIL_INDEX_RECORD_MIN_SIZE; + max_align = sizeof(uint32_t); + for (;;) { + min_align = (uint16_t)-1; + for (i = 0; i < count; i++) { + if (sorted[i]->record_align > max_align) + max_align = sorted[i]->record_align; + + if (sorted[i]->record_offset == 0 && + sorted[i]->record_size > 0) { + if ((offset % sorted[i]->record_align) == 0) + break; + if (sorted[i]->record_align < min_align) + min_align = sorted[i]->record_align; + } + } + if (i == count) { + if (min_align == (uint16_t)-1) { + /* all done */ + break; + } + /* we have to leave space here */ + i_assert(min_align > 1 && min_align < (uint16_t)-1); + offset += min_align - (offset % min_align); + } else { + sorted[i]->record_offset = offset; + offset += sorted[i]->record_size; + } + + i_assert(offset < (uint16_t)-1); + } + + if ((offset % max_align) != 0) { + /* keep record size divisible with maximum alignment */ + offset += max_align - (offset % max_align); + } + new_record_size = offset; + i_assert(new_record_size >= sizeof(struct mail_index_record)); + + /* copy the records to new buffer */ + new_buffer_size = map->rec_map->records_count * new_record_size; + new_buffer = buffer_create_dynamic(default_pool, new_buffer_size); + src = map->rec_map->records; + offset = 0; + for (rec_idx = 0; rec_idx < map->rec_map->records_count; rec_idx++) { + /* write the base record */ + buffer_write(new_buffer, offset, src, + sizeof(struct mail_index_record)); + + /* write extensions */ + for (i = 0; i < count; i++) { + buffer_write(new_buffer, offset + ext[i].record_offset, + CONST_PTR_OFFSET(src, old_offsets[i]), + copy_sizes[i]); + } + src = CONST_PTR_OFFSET(src, map->hdr.record_size); + offset += new_record_size; + } + + if (new_buffer->used != new_buffer_size) { + /* we didn't fully write the last record */ + size_t space = new_buffer_size - new_buffer->used; + i_assert(space < new_record_size); + buffer_append_zero(new_buffer, space); + } + + buffer_free(&map->rec_map->buffer); + map->rec_map->buffer = new_buffer; + map->rec_map->records = + buffer_get_modifiable_data(map->rec_map->buffer, NULL); + map->hdr.record_size = new_record_size; + + /* update record offsets in headers */ + for (i = 0; i < count; i++) { + ext_hdr = get_ext_header(map, &ext[i]); + ext_hdr->record_offset = ext[i].record_offset; + } +} + +static void +sync_ext_resize(const struct mail_transaction_ext_intro *u, + uint32_t ext_map_idx, struct mail_index_sync_map_ctx *ctx, + bool no_shrink) +{ + struct mail_index_map *map; + struct mail_index_ext *ext; + struct mail_index_ext_header *ext_hdr; + uint32_t old_padded_hdr_size, new_padded_hdr_size, old_record_size; + bool reorder = FALSE; + + ext = array_idx_modifiable(&ctx->view->map->extensions, ext_map_idx); + old_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size); + new_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size); + + if (ext->record_align != u->record_align || + ext->record_size != u->record_size) { + /* record changed */ + } else if (new_padded_hdr_size < old_padded_hdr_size) { + /* header is shrunk. do we allow? */ + if (no_shrink) + return; + } else if (ext->hdr_size == u->hdr_size) { + /* no changes */ + return; + } + /* something changed. get ourself a new map before we start changing + anything in it. */ + map = mail_index_sync_get_atomic_map(ctx); + /* ext was duplicated to the new map. */ + ext = array_idx_modifiable(&map->extensions, ext_map_idx); + + if (new_padded_hdr_size < old_padded_hdr_size) { + /* header shrank */ + if (no_shrink) + new_padded_hdr_size = old_padded_hdr_size; + else { + buffer_delete(map->hdr_copy_buf, + ext->hdr_offset + new_padded_hdr_size, + old_padded_hdr_size - new_padded_hdr_size); + ext->hdr_size = u->hdr_size; + } + } else if (new_padded_hdr_size > old_padded_hdr_size) { + /* header grown */ + buffer_insert_zero(map->hdr_copy_buf, + ext->hdr_offset + old_padded_hdr_size, + new_padded_hdr_size - old_padded_hdr_size); + ext->hdr_size = u->hdr_size; + } else { + if (ext->hdr_size != u->hdr_size) { + /* aligned sizes were the same, but the actual sizes + had changed */ + ext->hdr_size = u->hdr_size; + } + } + + if (ext->record_align < u->record_align || + (ext->record_align > u->record_align && !no_shrink)) { + ext->record_align = u->record_align; + reorder = TRUE; + } + + old_record_size = ext->record_size; + if (ext->record_size < u->record_size || + (ext->record_size > u->record_size && !no_shrink)) { + ext->record_size = u->record_size; + reorder = TRUE; + } + + i_assert((map->hdr_copy_buf->used % sizeof(uint64_t)) == 0); + map->hdr.header_size = map->hdr_copy_buf->used; + + ext_hdr = get_ext_header(map, ext); + ext_hdr->reset_id = ext->reset_id; + ext_hdr->hdr_size = ext->hdr_size; + ext_hdr->record_offset = ext->record_offset; + ext_hdr->record_size = ext->record_size; + ext_hdr->record_align = ext->record_align; + + if (new_padded_hdr_size != old_padded_hdr_size) { + /* move all hdr_offset of all extensions after this one */ + unsigned int i, count = array_count(&map->extensions); + ssize_t diff = (ssize_t)new_padded_hdr_size - + (ssize_t)old_padded_hdr_size; + + ext = array_front_modifiable(&map->extensions); + for (i = ext_map_idx + 1; i < count; i++) { + ext[i].ext_offset += diff; + ext[i].hdr_offset += diff; + } + } + + if (reorder) + sync_ext_reorder(map, ext_map_idx, old_record_size); +} + +static bool +mail_index_sync_ext_unknown_complain(struct mail_index_sync_map_ctx *ctx, + uint32_t ext_map_idx) +{ + unsigned char *p; + + if (ext_map_idx >= 1024) { + /* don't try to track too high values */ + return TRUE; + } + + if (ctx->unknown_extensions == NULL) { + ctx->unknown_extensions = + buffer_create_dynamic(default_pool, ext_map_idx + 8); + } + p = buffer_get_space_unsafe(ctx->unknown_extensions, ext_map_idx, 1); + if (*p != 0) { + /* we've already complained once */ + return FALSE; + } + *p = 1; + return TRUE; +} + +static void +mail_index_sync_ext_init_new(struct mail_index_sync_map_ctx *ctx, + const char *name, + const struct mail_index_ext_header *ext_hdr, + uint32_t *ext_map_idx_r) +{ + struct mail_index_map *map; + const struct mail_index_ext *ext; + buffer_t *hdr_buf; + uint32_t ext_map_idx; + + i_assert(mail_index_ext_name_is_valid(name)); + + /* be sure to get a unique mapping before we modify the extensions, + otherwise other map users will see the new extension but not the + data records that sync_ext_reorder() adds. */ + map = mail_index_sync_get_atomic_map(ctx); + + hdr_buf = map->hdr_copy_buf; + i_assert(hdr_buf->used == map->hdr.header_size); + + if (MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) != hdr_buf->used) { + /* we need to add padding between base header and extensions */ + buffer_append_zero(hdr_buf, + MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) - + hdr_buf->used); + } + + /* register record offset initially using zero, + sync_ext_reorder() will fix it. */ + ext_map_idx = mail_index_map_register_ext(map, name, hdr_buf->used, + ext_hdr); + ext = array_idx(&map->extensions, ext_map_idx); + + /* <ext_hdr> <name> [padding] [header data] */ + i_assert(ext_hdr->name_size == strlen(name)); + buffer_append(hdr_buf, ext_hdr, sizeof(*ext_hdr)); + buffer_append(hdr_buf, name, ext_hdr->name_size); + /* header must begin and end in correct alignment */ + buffer_append_zero(hdr_buf, + MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) - hdr_buf->used + + MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size)); + i_assert(hdr_buf->used == + ext->hdr_offset + MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size)); + i_assert((hdr_buf->used % sizeof(uint64_t)) == 0); + + map->hdr.header_size = hdr_buf->used; + + mail_index_sync_init_handlers(ctx); + sync_ext_reorder(map, ext_map_idx, 0); + i_assert(ext->record_offset != 0 || ext->record_size == 0); + + *ext_map_idx_r = ext_map_idx; +} + +int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_intro *u) +{ + struct mail_index_map *map = ctx->view->map; + struct mail_index_ext_header ext_hdr; + const struct mail_index_ext *ext; + const char *name, *error; + uint32_t ext_map_idx; + bool no_shrink; + + /* default to ignoring the following extension updates in case this + intro is corrupted */ + ctx->cur_ext_map_idx = (uint32_t)-2; + ctx->cur_ext_ignore = TRUE; + ctx->cur_ext_record_size = 0; + + if (u->ext_id != (uint32_t)-1 && + (!array_is_created(&map->extensions) || + u->ext_id >= array_count(&map->extensions))) { + /* The extension ID is unknown in this map. */ + if (map->hdr.log_file_seq == 0) { + /* This map was generated by + view_sync_get_log_lost_changes(). There's no need to + update any extensions, because they won't be used + anyway. Any extension lookups will be accessed via + the latest index map. */ + i_assert(map->rec_map != ctx->view->index->map->rec_map); + return 1; + } + if (!mail_index_sync_ext_unknown_complain(ctx, u->ext_id)) + return -1; + mail_index_sync_set_corrupted(ctx, + "Extension introduction for unknown id %u", u->ext_id); + return -1; + } + + if (u->ext_id == (uint32_t)-1 && u->name_size == 0) { + mail_index_sync_set_corrupted(ctx, + "Extension introduction without id or name"); + return -1; + } + + if (u->ext_id != (uint32_t)-1) { + name = NULL; + ext_map_idx = u->ext_id; + } else { + name = t_strndup(u + 1, u->name_size); + if (!mail_index_map_lookup_ext(map, name, &ext_map_idx)) + ext_map_idx = (uint32_t)-1; + } + if (ext_map_idx == (uint32_t)-1) + ext = NULL; + else { + ext = array_idx(&map->extensions, ext_map_idx); + name = ext->name; + } + i_assert(name != NULL); + + if (!ctx->internal_update && + strcmp(name, MAIL_INDEX_EXT_KEYWORDS) == 0) { + /* Keyword extension is handled internally by the keyword + code. Any attempt to modify them directly could cause + assert-crashes later, so prevent them immediately. */ + mail_index_sync_set_corrupted(ctx, + "Extension introduction for keywords"); + return -1; + } + + i_zero(&ext_hdr); + ext_hdr.name_size = strlen(name); + ext_hdr.reset_id = u->reset_id; + ext_hdr.hdr_size = u->hdr_size; + ext_hdr.record_size = u->record_size; + ext_hdr.record_align = u->record_align; + no_shrink = (u->flags & MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK) != 0; + + /* make sure the header looks valid before doing anything with it */ + if (mail_index_map_ext_hdr_check(&map->hdr, &ext_hdr, + name, &error) < 0) { + mail_index_sync_set_corrupted(ctx, + "Broken extension introduction: %s", error); + return -1; + } + + ctx->cur_ext_record_size = u->record_size; + if (ext != NULL) { + /* exists already */ + if (u->reset_id == ext->reset_id) { + /* check if we need to resize anything */ + sync_ext_resize(u, ext_map_idx, ctx, no_shrink); + ctx->cur_ext_ignore = FALSE; + } else { + /* extension was reset and this transaction hadn't + yet seen it. ignore this update (except for + resets). */ + ctx->cur_ext_ignore = TRUE; + } + + ctx->cur_ext_map_idx = ext_map_idx; + return 1; + } + + mail_index_sync_ext_init_new(ctx, name, &ext_hdr, &ext_map_idx); + + ctx->cur_ext_ignore = FALSE; + ctx->cur_ext_map_idx = ctx->internal_update ? + (uint32_t)-1 : ext_map_idx; + return 1; +} + +static void mail_index_sync_ext_clear(struct mail_index_view *view, + struct mail_index_map *map, + struct mail_index_ext *ext) +{ + struct mail_index_record *rec; + uint32_t seq; + + memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset, + ext->hdr_size), 0, ext->hdr_size); + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); + + for (seq = 1; seq <= view->map->rec_map->records_count; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + memset(PTR_OFFSET(rec, ext->record_offset), 0, + ext->record_size); + } +} + +int mail_index_sync_ext_reset(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_reset *u) +{ + struct mail_index_map *map; + struct mail_index_ext_header *ext_hdr; + struct mail_index_ext *ext; + + if (ctx->cur_ext_map_idx == (uint32_t)-1) { + mail_index_sync_set_corrupted(ctx, + "Extension reset without intro prefix"); + return -1; + } + if (ctx->cur_ext_map_idx == (uint32_t)-2 && ctx->cur_ext_ignore) { + /* previous extension intro was broken */ + return -1; + } + /* since we're resetting the extension, don't check cur_ext_ignore */ + + /* a new index file will be created, so the old data won't be + accidentally used by other processes. */ + map = mail_index_sync_get_atomic_map(ctx); + + ext = array_idx_modifiable(&map->extensions, ctx->cur_ext_map_idx); + ext->reset_id = u->new_reset_id; + + if (u->preserve_data == 0) + mail_index_sync_ext_clear(ctx->view, map, ext); + + ext_hdr = get_ext_header(map, ext); + ext_hdr->reset_id = u->new_reset_id; + return 1; +} + +int mail_index_sync_ext_hdr_update(struct mail_index_sync_map_ctx *ctx, + uint32_t offset, uint32_t size, + const void *data) +{ + struct mail_index_map *map = ctx->view->map; + const struct mail_index_ext *ext; + + if (ctx->cur_ext_map_idx == (uint32_t)-1) { + mail_index_sync_set_corrupted(ctx, + "Extension header update without intro prefix"); + return -1; + } + if (ctx->cur_ext_ignore) + return 1; + + ext = array_idx(&map->extensions, ctx->cur_ext_map_idx); + if (offset + size > ext->hdr_size) { + mail_index_sync_set_corrupted(ctx, + "Extension header update points outside header size"); + return -1; + } + + buffer_write(map->hdr_copy_buf, ext->hdr_offset + offset, data, size); + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); + + if (ext->index_idx == ctx->view->index->modseq_ext_id) + mail_index_modseq_hdr_update(ctx->modseq_ctx); + return 1; +} + +int +mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_rec_update *u) +{ + struct mail_index_view *view = ctx->view; + struct mail_index_record *rec; + const struct mail_index_ext *ext; + void *old_data; + uint32_t seq; + + i_assert(ctx->cur_ext_map_idx != (uint32_t)-1); + i_assert(!ctx->cur_ext_ignore); + + if (u->uid == 0 || u->uid >= view->map->hdr.next_uid) { + mail_index_sync_set_corrupted(ctx, + "Extension record update for invalid uid=%u", u->uid); + return -1; + } + + if (!mail_index_lookup_seq(view, u->uid, &seq)) + return 1; + + ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx); + i_assert(ext->record_offset + ctx->cur_ext_record_size <= + view->map->hdr.record_size); + + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + old_data = PTR_OFFSET(rec, ext->record_offset); + + /* @UNSAFE */ + memcpy(old_data, u + 1, ctx->cur_ext_record_size); + if (ctx->cur_ext_record_size < ext->record_size) { + memset(PTR_OFFSET(old_data, ctx->cur_ext_record_size), 0, + ext->record_size - ctx->cur_ext_record_size); + } + return 1; +} + +int +mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_atomic_inc *u) +{ + struct mail_index_view *view = ctx->view; + struct mail_index_record *rec; + const struct mail_index_ext *ext; + void *data; + uint32_t seq; + uint64_t min_value, max_value, orig_num; + + i_assert(ctx->cur_ext_map_idx != (uint32_t)-1); + i_assert(!ctx->cur_ext_ignore); + + if (u->uid == 0 || u->uid >= view->map->hdr.next_uid) { + mail_index_sync_set_corrupted(ctx, + "Extension record inc for invalid uid=%u", u->uid); + return -1; + } + + if (!mail_index_lookup_seq(view, u->uid, &seq)) + return 1; + + ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx); + i_assert(ext->record_offset + ctx->cur_ext_record_size <= + view->map->hdr.record_size); + + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + data = PTR_OFFSET(rec, ext->record_offset); + + min_value = u->diff >= 0 ? 0 : (uint64_t)(-(int64_t)u->diff); + + max_value = ctx->cur_ext_record_size == 8 ? (uint64_t)-1 : + ((uint64_t)1 << (ctx->cur_ext_record_size*8)) - 1; + if (u->diff <= 0) { + /* skip */ + } else if (max_value >= (uint32_t)u->diff) { + max_value -= u->diff; + } else { + mail_index_sync_set_corrupted(ctx, + "Extension record inc diff=%d larger than max value=%u " + "(uid=%u)", u->diff, (unsigned int)max_value, u->uid); + return -1; + } + + switch (ctx->cur_ext_record_size) { + case 1: { + uint8_t *num = data; + + orig_num = *num; + if (orig_num >= min_value && orig_num <= max_value) + *num += u->diff; + break; + } + case 2: { + uint16_t *num = data; + orig_num = *num; + if (orig_num >= min_value && orig_num <= max_value) + *num += u->diff; + break; + } + case 4: { + uint32_t *num = data; + orig_num = *num; + if (orig_num >= min_value && orig_num <= max_value) + *num += u->diff; + break; + } + case 8: { + uint64_t *num = data; + orig_num = *num; + if (orig_num >= min_value && orig_num <= max_value) + *num += u->diff; + break; + } + default: + mail_index_sync_set_corrupted(ctx, + "Extension record inc with invalid size=%u", + ctx->cur_ext_record_size); + return -1; + } + if (orig_num < min_value) { + mail_index_sync_set_corrupted(ctx, + "Extension record inc drops number below zero " + "(uid=%u, diff=%d, orig=%"PRIu64")", + u->uid, u->diff, orig_num); + return -1; + } else if (orig_num > max_value) { + mail_index_sync_set_corrupted(ctx, + "Extension record inc overflows number " + "(uid=%u, diff=%d, orig=%"PRIu64")", + u->uid, u->diff, orig_num); + return -1; + } + return 1; +} diff --git a/src/lib-index/mail-index-sync-keywords.c b/src/lib-index/mail-index-sync-keywords.c new file mode 100644 index 0000000..2c45156 --- /dev/null +++ b/src/lib-index/mail-index-sync-keywords.c @@ -0,0 +1,347 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "mail-index-modseq.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-transaction-log.h" + +static bool +keyword_lookup(struct mail_index_sync_map_ctx *ctx, + const char *keyword_name, unsigned int *idx_r) +{ + struct mail_index_map *map = ctx->view->map; + const unsigned int *idx_map; + unsigned int i, count, keyword_idx; + + if (array_is_created(&map->keyword_idx_map) && + mail_index_keyword_lookup(ctx->view->index, keyword_name, + &keyword_idx)) { + /* FIXME: slow. maybe create index -> file mapping as well */ + idx_map = array_get(&map->keyword_idx_map, &count); + for (i = 0; i < count; i++) { + if (idx_map[i] == keyword_idx) { + *idx_r = i; + return TRUE; + } + } + } + return FALSE; +} + +static buffer_t * +keywords_get_header_buf(struct mail_index_map *map, + const struct mail_index_ext *ext, + unsigned int new_count, unsigned int *keywords_count_r, + size_t *rec_offset_r, size_t *name_offset_root_r, + size_t *name_offset_r) +{ + buffer_t *buf; + const struct mail_index_keyword_header *kw_hdr; + const struct mail_index_keyword_header_rec *kw_rec; + const char *name; + struct mail_index_keyword_header new_kw_hdr; + uint32_t offset; + + kw_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset); + kw_rec = (const void *)(kw_hdr + 1); + name = (const char *)(kw_rec + kw_hdr->keywords_count); + + if (kw_hdr->keywords_count == 0) + return NULL; + + i_assert((size_t)(name - (const char *)kw_hdr) < ext->hdr_size); + + new_kw_hdr = *kw_hdr; + new_kw_hdr.keywords_count += new_count; + *keywords_count_r = new_kw_hdr.keywords_count; + + offset = kw_rec[kw_hdr->keywords_count-1].name_offset; + offset += strlen(name + offset) + 1; + + buf = t_buffer_create(512); + buffer_append(buf, &new_kw_hdr, sizeof(new_kw_hdr)); + buffer_append(buf, kw_rec, sizeof(*kw_rec) * kw_hdr->keywords_count); + *rec_offset_r = buf->used; + buffer_write(buf, buf->used + sizeof(*kw_rec) * new_count, + name, offset); + *name_offset_root_r = buf->used; + *name_offset_r = offset; + return buf; +} + +static void keywords_ext_register(struct mail_index_sync_map_ctx *ctx, + uint32_t ext_map_idx, uint32_t reset_id, + uint32_t hdr_size, uint32_t keywords_count) +{ + buffer_t ext_intro_buf; + struct mail_transaction_ext_intro *u; + unsigned char ext_intro_data[sizeof(*u) + + sizeof(MAIL_INDEX_EXT_KEYWORDS)-1]; + + i_assert(keywords_count > 0); + + buffer_create_from_data(&ext_intro_buf, ext_intro_data, + sizeof(ext_intro_data)); + + u = buffer_append_space_unsafe(&ext_intro_buf, sizeof(*u)); + u->ext_id = ext_map_idx; + u->reset_id = reset_id; + u->hdr_size = hdr_size; + u->record_size = (keywords_count + CHAR_BIT - 1) / CHAR_BIT; + if ((u->record_size % 4) != 0) { + /* since we aren't properly aligned anyway, + reserve one extra byte for future */ + u->record_size++; + } + u->record_align = 1; + + if (ext_map_idx == (uint32_t)-1) { + u->name_size = strlen(MAIL_INDEX_EXT_KEYWORDS); + buffer_append(&ext_intro_buf, MAIL_INDEX_EXT_KEYWORDS, + u->name_size); + } + + ctx->internal_update = TRUE; + if (mail_index_sync_ext_intro(ctx, u) < 0) + i_panic("Keyword extension growing failed"); + ctx->internal_update = FALSE; +} + +static void +keywords_header_add(struct mail_index_sync_map_ctx *ctx, + const char *keyword_name, unsigned int *keyword_idx_r) +{ + struct mail_index_map *map; + const struct mail_index_ext *ext = NULL; + struct mail_index_keyword_header *kw_hdr; + struct mail_index_keyword_header_rec kw_rec; + uint32_t ext_map_idx; + buffer_t *buf = NULL; + size_t keyword_len, rec_offset, name_offset, name_offset_root; + unsigned int keywords_count; + + /* if we crash in the middle of writing the header, the + keywords are more or less corrupted. avoid that by + making sure the header is updated atomically. */ + map = mail_index_sync_get_atomic_map(ctx); + + if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, + &ext_map_idx)) + ext_map_idx = (uint32_t)-1; + else { + /* update existing header */ + ext = array_idx(&map->extensions, ext_map_idx); + buf = keywords_get_header_buf(map, ext, 1, &keywords_count, + &rec_offset, &name_offset_root, + &name_offset); + } + + if (buf == NULL) { + /* create new / replace broken header */ + const unsigned int initial_keywords_count = 1; + + buf = t_buffer_create(512); + kw_hdr = buffer_append_space_unsafe(buf, sizeof(*kw_hdr)); + kw_hdr->keywords_count = initial_keywords_count; + + keywords_count = kw_hdr->keywords_count; + rec_offset = buf->used; + name_offset_root = rec_offset + + initial_keywords_count * sizeof(kw_rec); + name_offset = 0; + } + + /* add the keyword */ + i_zero(&kw_rec); + kw_rec.name_offset = name_offset; + + keyword_len = strlen(keyword_name) + 1; + buffer_write(buf, rec_offset, &kw_rec, sizeof(kw_rec)); + buffer_write(buf, name_offset_root, keyword_name, keyword_len); + + rec_offset += sizeof(kw_rec); + kw_rec.name_offset += keyword_len; + name_offset_root += keyword_len; + + if ((buf->used % 4) != 0) + buffer_append_zero(buf, 4 - (buf->used % 4)); + + if (ext == NULL || buf->used > ext->hdr_size || + (uint32_t)ext->record_size * CHAR_BIT < keywords_count) { + /* if we need to grow the buffer, add some padding */ + buffer_append_zero(buf, 128); + keywords_ext_register(ctx, ext_map_idx, + ext == NULL ? 0 : ext->reset_id, + buf->used, keywords_count); + + /* map may have changed */ + map = ctx->view->map; + + if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, + &ext_map_idx)) + i_unreached(); + ext = array_idx(&map->extensions, ext_map_idx); + + i_assert(ext->hdr_size == buf->used); + } + + buffer_copy(map->hdr_copy_buf, ext->hdr_offset, buf, 0, buf->used); + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); + + if (mail_index_map_parse_keywords(map) < 0) + i_panic("Keyword update corrupted keywords header"); + + *keyword_idx_r = keywords_count - 1; + i_assert(*keyword_idx_r / CHAR_BIT < ext->record_size); +} + +static int +keywords_update_records(struct mail_index_sync_map_ctx *ctx, + const struct mail_index_ext *ext, + unsigned int keyword_idx, enum modify_type type, + uint32_t uid1, uint32_t uid2) +{ + struct mail_index_view *view = ctx->view; + struct mail_index_record *rec; + unsigned char *data, data_mask; + unsigned int data_offset; + uint32_t seq1, seq2; + + i_assert(keyword_idx != UINT_MAX); + + if (!mail_index_lookup_seq_range(view, uid1, uid2, &seq1, &seq2)) + return 1; + + mail_index_modseq_update_keyword(ctx->modseq_ctx, keyword_idx, + seq1, seq2); + + data_offset = keyword_idx / CHAR_BIT; + data_mask = 1 << (keyword_idx % CHAR_BIT); + + i_assert(data_offset < ext->record_size); + data_offset += ext->record_offset; + + i_assert(data_offset >= MAIL_INDEX_RECORD_MIN_SIZE); + + switch (type) { + case MODIFY_ADD: + for (; seq1 <= seq2; seq1++) { + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq1); + data = PTR_OFFSET(rec, data_offset); + *data |= data_mask; + } + break; + case MODIFY_REMOVE: + data_mask = (unsigned char)~data_mask; + for (; seq1 <= seq2; seq1++) { + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq1); + data = PTR_OFFSET(rec, data_offset); + *data &= data_mask; + } + break; + default: + i_unreached(); + } + return 1; +} + +int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_header *hdr, + const struct mail_transaction_keyword_update *rec) +{ + struct mail_index_view *view = ctx->view; + const char *keyword_name; + const struct mail_index_ext *ext; + const uint32_t *uid, *end; + uint32_t seqset_offset, ext_map_idx; + unsigned int keyword_idx; + int ret; + + i_assert(rec->name_size > 0); + + seqset_offset = sizeof(*rec) + rec->name_size; + if ((seqset_offset % 4) != 0) + seqset_offset += 4 - (seqset_offset % 4); + i_assert(seqset_offset < hdr->size); + + uid = CONST_PTR_OFFSET(rec, seqset_offset); + end = CONST_PTR_OFFSET(rec, hdr->size); + + keyword_name = t_strndup(rec + 1, rec->name_size); + if (!keyword_lookup(ctx, keyword_name, &keyword_idx)) + keywords_header_add(ctx, keyword_name, &keyword_idx); + + /* if the keyword wasn't found, the "keywords" extension was created. + if it was found, the record size should already be correct, but + in case it isn't just fix it ourself. */ + if (!mail_index_map_lookup_ext(view->map, MAIL_INDEX_EXT_KEYWORDS, + &ext_map_idx)) + i_unreached(); + + ext = array_idx(&view->map->extensions, ext_map_idx); + if (keyword_idx / CHAR_BIT >= ext->record_size) { + if (rec->modify_type == MODIFY_REMOVE) { + /* nothing to do */ + return 1; + } + + /* grow the record size */ + keywords_ext_register(ctx, ext_map_idx, ext->reset_id, + ext->hdr_size, + array_count(&view->map->keyword_idx_map)); + if (!mail_index_map_lookup_ext(view->map, + MAIL_INDEX_EXT_KEYWORDS, + &ext_map_idx)) + i_unreached(); + ext = array_idx(&view->map->extensions, ext_map_idx); + } + + while (uid+2 <= end) { + ret = keywords_update_records(ctx, ext, keyword_idx, + rec->modify_type, + uid[0], uid[1]); + if (ret <= 0) + return ret; + + uid += 2; + } + + return 1; +} + +int +mail_index_sync_keywords_reset(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_header *hdr, + const struct mail_transaction_keyword_reset *r) +{ + struct mail_index_map *map = ctx->view->map; + struct mail_index_record *rec; + const struct mail_index_ext *ext; + const struct mail_transaction_keyword_reset *end; + uint32_t ext_map_idx, seq1, seq2; + + if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, + &ext_map_idx)) { + /* nothing to do */ + return 1; + } + + ext = array_idx(&map->extensions, ext_map_idx); + end = CONST_PTR_OFFSET(r, hdr->size); + for (; r != end; r++) { + if (!mail_index_lookup_seq_range(ctx->view, r->uid1, r->uid2, + &seq1, &seq2)) + continue; + + mail_index_modseq_reset_keywords(ctx->modseq_ctx, seq1, seq2); + for (; seq1 <= seq2; seq1++) { + rec = MAIL_INDEX_REC_AT_SEQ(map, seq1); + memset(PTR_OFFSET(rec, ext->record_offset), + 0, ext->record_size); + } + } + return 1; +} diff --git a/src/lib-index/mail-index-sync-private.h b/src/lib-index/mail-index-sync-private.h new file mode 100644 index 0000000..094c83d --- /dev/null +++ b/src/lib-index/mail-index-sync-private.h @@ -0,0 +1,104 @@ +#ifndef MAIL_INDEX_SYNC_PRIVATE_H +#define MAIL_INDEX_SYNC_PRIVATE_H + +#include "mail-index-private.h" +#include "mail-transaction-log.h" + +struct uid_range { + uint32_t uid1, uid2; +}; +ARRAY_DEFINE_TYPE(uid_range, struct uid_range); + +struct mail_index_sync_list { + const ARRAY_TYPE(uid_range) *array; + unsigned int idx; + unsigned int keyword_idx:31; + bool keyword_remove:1; +}; + +struct mail_index_expunge_handler { + mail_index_expunge_handler_t *handler; + void *context; + void **sync_context; + uint32_t record_offset; +}; + +struct mail_index_sync_map_ctx { + struct mail_index_view *view; + struct mail_index_modseq_sync *modseq_ctx; + uint32_t cur_ext_map_idx; + uint32_t cur_ext_record_size; + + uint32_t ext_intro_seq; + uoff_t ext_intro_offset, ext_intro_end_offset; + + ARRAY(struct mail_index_expunge_handler) expunge_handlers; + ARRAY(void *) extra_contexts; + buffer_t *unknown_extensions; + + enum mail_index_sync_handler_type type; + + bool sync_handlers_initialized:1; + bool expunge_handlers_set:1; + bool expunge_handlers_used:1; + bool cur_ext_ignore:1; + bool internal_update:1; /* used by keywords for ext_intro */ + bool errors:1; +}; + +extern struct mail_transaction_map_functions mail_index_map_sync_funcs; + +void mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx, + struct mail_index_view *view, + enum mail_index_sync_handler_type type); +void mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx); +bool mail_index_sync_map_want_index_reopen(struct mail_index_map *map, + enum mail_index_sync_handler_type type); +int mail_index_sync_map(struct mail_index_map **_map, + enum mail_index_sync_handler_type type, + const char **reason_r); + +int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_header *hdr, + const void *data); + +struct mail_index_map * +mail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx); + +void mail_index_sync_init_expunge_handlers(struct mail_index_sync_map_ctx *ctx); +void +mail_index_sync_deinit_expunge_handlers(struct mail_index_sync_map_ctx *ctx); +void mail_index_sync_init_handlers(struct mail_index_sync_map_ctx *ctx); +void mail_index_sync_deinit_handlers(struct mail_index_sync_map_ctx *ctx); + +int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_intro *u); +int mail_index_sync_ext_reset(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_reset *u); +int mail_index_sync_ext_hdr_update(struct mail_index_sync_map_ctx *ctx, + uint32_t offset, uint32_t size, + const void *data); +int +mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_rec_update *u); +int +mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_ext_atomic_inc *u); + +int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_header *hdr, + const struct mail_transaction_keyword_update *rec); +int +mail_index_sync_keywords_reset(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_header *hdr, + const struct mail_transaction_keyword_reset *r); + +void mail_index_sync_set_corrupted(struct mail_index_sync_map_ctx *ctx, + const char *fmt, ...) + ATTR_FORMAT(2, 3) ATTR_COLD; + +#ifdef DEBUG +void mail_index_map_check(struct mail_index_map *map); +#endif + +#endif diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c new file mode 100644 index 0000000..0c37995 --- /dev/null +++ b/src/lib-index/mail-index-sync-update.c @@ -0,0 +1,1087 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "mmap-util.h" +#include "mail-index-modseq.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-transaction-log.h" +#include "mail-transaction-log-private.h" + +/* If we have less than this many bytes to sync from log file, don't bother + reading the main index */ +#define MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE 2048 + +static void +mail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx, + struct mail_index_map *map, bool eol) +{ + uint32_t prev_seq; + uoff_t prev_offset; + + mail_transaction_log_view_get_prev_pos(ctx->view->log_view, + &prev_seq, &prev_offset); + if (prev_seq == 0) { + /* handling lost changes in view syncing */ + return; + } + + if (!eol) { + if (prev_offset == ctx->ext_intro_end_offset && + prev_seq == ctx->ext_intro_seq) { + /* previous transaction was an extension introduction. + we probably came here from + mail_index_sync_ext_reset(). if there are any more + views which want to continue syncing it needs the + intro. so back up a bit more. + + don't do this in case the last transaction in the + log is the extension intro, so we don't keep trying + to sync it over and over again. */ + prev_offset = ctx->ext_intro_offset; + } + map->hdr.log_file_seq = prev_seq; + } else { + i_assert(ctx->view->index->log->head->hdr.file_seq == prev_seq); + if (map->hdr.log_file_seq != prev_seq) { + map->hdr.log_file_seq = prev_seq; + map->hdr.log_file_tail_offset = 0; + } + } + map->hdr.log_file_head_offset = prev_offset; +} + +static void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx, + struct mail_index_map *map) +{ + struct mail_index_view *view = ctx->view; + + i_assert(view->map != map); + + mail_index_sync_update_log_offset(ctx, view->map, FALSE); + mail_index_unmap(&view->map); + view->map = map; + + if (ctx->type != MAIL_INDEX_SYNC_HANDLER_VIEW) + view->index->map = map; + + mail_index_modseq_sync_map_replaced(ctx->modseq_ctx); +} + +static struct mail_index_map * +mail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx) +{ + struct mail_index_map *map = ctx->view->map; + + if (map->refcount > 1) { + /* Multiple views point to this map. Make a copy of the map + (but not rec_map). */ + map = mail_index_map_clone(map); + mail_index_sync_replace_map(ctx, map); + i_assert(ctx->view->map == map); + } + + if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map)) { + /* map points to mmap()ed area, copy it into memory. */ + mail_index_map_move_to_memory(ctx->view->map); + mail_index_modseq_sync_map_replaced(ctx->modseq_ctx); + } + return map; +} + +struct mail_index_map * +mail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx) +{ + /* First make sure we have a private map with rec_map pointing to + memory. */ + (void)mail_index_sync_move_to_private_memory(ctx); + /* Next make sure the rec_map is also private to us. */ + mail_index_record_map_move_to_private(ctx->view->map); + mail_index_modseq_sync_map_replaced(ctx->modseq_ctx); + return ctx->view->map; +} + +static int +mail_index_header_update_counts(struct mail_index_header *hdr, + uint8_t old_flags, uint8_t new_flags, + const char **error_r) +{ + if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) { + /* different seen-flag */ + if ((old_flags & MAIL_SEEN) != 0) { + if (hdr->seen_messages_count == 0) { + *error_r = "Seen counter wrong"; + return -1; + } + hdr->seen_messages_count--; + } else { + if (hdr->seen_messages_count >= hdr->messages_count) { + *error_r = "Seen counter wrong"; + return -1; + } + + if (++hdr->seen_messages_count == hdr->messages_count) + hdr->first_unseen_uid_lowwater = hdr->next_uid; + } + } + + if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) { + /* different deleted-flag */ + if ((old_flags & MAIL_DELETED) == 0) { + hdr->deleted_messages_count++; + if (hdr->deleted_messages_count > hdr->messages_count) { + *error_r = "Deleted counter wrong"; + return -1; + } + } else { + if (hdr->deleted_messages_count == 0 || + hdr->deleted_messages_count > hdr->messages_count) { + *error_r = "Deleted counter wrong"; + return -1; + } + + if (--hdr->deleted_messages_count == 0) + hdr->first_deleted_uid_lowwater = hdr->next_uid; + } + } + return 0; +} + +static void +mail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx, + uint32_t uid, + uint8_t old_flags, uint8_t new_flags) +{ + struct mail_index_map *const *maps; + const char *error; + unsigned int i, count; + + maps = array_get(&ctx->view->map->rec_map->maps, &count); + for (i = 0; i < count; i++) { + if (uid >= maps[i]->hdr.next_uid) + continue; + + if (mail_index_header_update_counts(&maps[i]->hdr, + old_flags, new_flags, + &error) < 0) + mail_index_sync_set_corrupted(ctx, "%s", error); + } +} + +static void +mail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx, + uint32_t uid, uint8_t old_flags, + uint8_t new_flags) +{ + const char *error; + + if (uid >= ctx->view->map->hdr.next_uid) { + mail_index_sync_set_corrupted(ctx, "uid %u >= next_uid %u", + uid, ctx->view->map->hdr.next_uid); + } else { + if (mail_index_header_update_counts(&ctx->view->map->hdr, + old_flags, new_flags, + &error) < 0) + mail_index_sync_set_corrupted(ctx, "%s", error); + } +} + +static void +mail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx, + uint32_t uid, enum mail_flags flags) +{ + struct mail_index_map *const *maps; + unsigned int i, count; + + maps = array_get(&ctx->view->map->rec_map->maps, &count); + for (i = 0; i < count; i++) { + if ((flags & MAIL_SEEN) == 0 && + uid < maps[i]->hdr.first_unseen_uid_lowwater) + maps[i]->hdr.first_unseen_uid_lowwater = uid; + if ((flags & MAIL_DELETED) != 0 && + uid < maps[i]->hdr.first_deleted_uid_lowwater) + maps[i]->hdr.first_deleted_uid_lowwater = uid; + } +} + +static void +sync_expunge_call_handlers(struct mail_index_sync_map_ctx *ctx, + uint32_t seq1, uint32_t seq2) +{ + const struct mail_index_expunge_handler *eh; + struct mail_index_record *rec; + uint32_t seq; + + array_foreach(&ctx->expunge_handlers, eh) { + for (seq = seq1; seq <= seq2; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq); + eh->handler(ctx, PTR_OFFSET(rec, eh->record_offset), + eh->sync_context); + } + } +} + +static bool +sync_expunge_handlers_init(struct mail_index_sync_map_ctx *ctx) +{ + /* call expunge handlers only when syncing index file */ + if (ctx->type != MAIL_INDEX_SYNC_HANDLER_FILE) + return FALSE; + + if (!ctx->expunge_handlers_set) + mail_index_sync_init_expunge_handlers(ctx); + + if (!array_is_created(&ctx->expunge_handlers)) + return FALSE; + return TRUE; +} + +static void +sync_expunge_range(struct mail_index_sync_map_ctx *ctx, const ARRAY_TYPE(seq_range) *seqs) +{ + struct mail_index_map *map; + const struct seq_range *range; + unsigned int i, count; + uint32_t dest_seq1, prev_seq2, orig_rec_count; + + range = array_get(seqs, &count); + if (count == 0) + return; + + /* Get a private in-memory rec_map, which we can modify. */ + map = mail_index_sync_get_atomic_map(ctx); + + /* call the expunge handlers first */ + if (sync_expunge_handlers_init(ctx)) { + for (i = 0; i < count; i++) { + sync_expunge_call_handlers(ctx, + range[i].seq1, range[i].seq2); + } + } + + prev_seq2 = 0; + dest_seq1 = 1; + orig_rec_count = map->rec_map->records_count; + for (i = 0; i < count; i++) { + uint32_t seq1 = range[i].seq1; + uint32_t seq2 = range[i].seq2; + struct mail_index_record *rec; + uint32_t seq_count, seq; + + i_assert(seq1 > prev_seq2); + + for (seq = seq1; seq <= seq2; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(map, seq); + mail_index_sync_header_update_counts(ctx, rec->uid, rec->flags, 0); + } + + if (prev_seq2+1 <= seq1-1) { + /* @UNSAFE: move (prev_seq2+1) .. (seq1-1) to its + final location in the map if necessary */ + uint32_t move_count = (seq1-1) - (prev_seq2+1) + 1; + if (prev_seq2+1-1 != dest_seq1-1) + memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1), + MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1), + move_count * map->hdr.record_size); + dest_seq1 += move_count; + } + seq_count = seq2 - seq1 + 1; + map->rec_map->records_count -= seq_count; + map->hdr.messages_count -= seq_count; + mail_index_modseq_expunge(ctx->modseq_ctx, seq1, seq2); + prev_seq2 = seq2; + } + /* Final stragglers */ + if (orig_rec_count > prev_seq2) { + uint32_t final_move_count = orig_rec_count - prev_seq2; + memmove(MAIL_INDEX_REC_AT_SEQ(map, dest_seq1), + MAIL_INDEX_REC_AT_SEQ(map, prev_seq2+1), + final_move_count * map->hdr.record_size); + } +} + +static void *sync_append_record(struct mail_index_map *map) +{ + size_t append_pos; + void *ret; + + append_pos = map->rec_map->records_count * map->hdr.record_size; + ret = buffer_get_space_unsafe(map->rec_map->buffer, append_pos, + map->hdr.record_size); + map->rec_map->records = + buffer_get_modifiable_data(map->rec_map->buffer, NULL); + return ret; +} + +static bool sync_update_ignored_change(struct mail_index_sync_map_ctx *ctx) +{ + struct mail_index_transaction_commit_result *result = + ctx->view->index->sync_commit_result; + uint32_t prev_log_seq; + uoff_t prev_log_offset, trans_start_offset, trans_end_offset; + + if (result == NULL) + return FALSE; + + /* we'll return TRUE if this modseq change was written within the + transaction that was just committed */ + mail_transaction_log_view_get_prev_pos(ctx->view->log_view, + &prev_log_seq, &prev_log_offset); + if (prev_log_seq != result->log_file_seq) + return FALSE; + + trans_end_offset = result->log_file_offset; + trans_start_offset = trans_end_offset - result->commit_size; + if (prev_log_offset < trans_start_offset || + prev_log_offset >= trans_end_offset) + return FALSE; + + return TRUE; +} + +static int +sync_modseq_update(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_modseq_update *u, + unsigned int size) +{ + struct mail_index_view *view = ctx->view; + const struct mail_transaction_modseq_update *end; + uint32_t seq; + uint64_t min_modseq; + int ret; + + end = CONST_PTR_OFFSET(u, size); + for (; u < end; u++) { + if (u->uid == 0) + seq = 0; + else if (!mail_index_lookup_seq(view, u->uid, &seq)) + continue; + + min_modseq = ((uint64_t)u->modseq_high32 << 32) | + u->modseq_low32; + + ret = seq == 0 ? 1 : + mail_index_modseq_set(view, seq, min_modseq); + if (ret < 0) { + mail_index_sync_set_corrupted(ctx, + "modseqs updated before they were enabled"); + return -1; + } + if (ret == 0 && sync_update_ignored_change(ctx)) + view->index->sync_commit_result->ignored_modseq_changes++; + } + return 1; +} + +static int sync_append(const struct mail_index_record *rec, + struct mail_index_sync_map_ctx *ctx) +{ + struct mail_index_view *view = ctx->view; + struct mail_index_map *map = view->map; + const struct mail_index_record *old_rec; + enum mail_flags new_flags; + void *dest; + + if (rec->uid < map->hdr.next_uid) { + mail_index_sync_set_corrupted(ctx, + "Append with UID %u, but next_uid = %u", + rec->uid, map->hdr.next_uid); + return -1; + } + + /* We'll need to append a new record. If map currently points to + mmap()ed index, it first needs to be moved to memory since we can't + write past the mmap()ed memory area. */ + map = mail_index_sync_move_to_private_memory(ctx); + + if (rec->uid <= map->rec_map->last_appended_uid) { + i_assert(map->hdr.messages_count < map->rec_map->records_count); + /* the flags may have changed since it was added to map. + use the updated flags already, so flag counters won't get + broken. */ + old_rec = MAIL_INDEX_MAP_IDX(map, map->hdr.messages_count); + i_assert(old_rec->uid == rec->uid); + new_flags = old_rec->flags; + } else { + /* don't rely on buffer->used being at the correct position. + at least expunges can move it */ + dest = sync_append_record(map); + memcpy(dest, rec, sizeof(*rec)); + memset(PTR_OFFSET(dest, sizeof(*rec)), 0, + map->hdr.record_size - sizeof(*rec)); + map->rec_map->records_count++; + map->rec_map->last_appended_uid = rec->uid; + new_flags = rec->flags; + + mail_index_modseq_append(ctx->modseq_ctx, + map->rec_map->records_count); + } + + map->hdr.messages_count++; + map->hdr.next_uid = rec->uid+1; + + if ((new_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0 && + (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0) + map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + + mail_index_header_update_lowwaters(ctx, rec->uid, new_flags); + mail_index_sync_header_update_counts(ctx, rec->uid, 0, new_flags); + return 1; +} + +static int sync_flag_update(const struct mail_transaction_flag_update *u, + struct mail_index_sync_map_ctx *ctx) +{ + struct mail_index_view *view = ctx->view; + struct mail_index_record *rec; + uint8_t flag_mask, old_flags; + uint32_t seq, seq1, seq2; + + if (!mail_index_lookup_seq_range(view, u->uid1, u->uid2, &seq1, &seq2)) + return 1; + + if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u)) { + mail_index_modseq_update_flags(ctx->modseq_ctx, + u->add_flags | u->remove_flags, + seq1, seq2); + } + + if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0 && + (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0) + view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + + flag_mask = (unsigned char)~u->remove_flags; + + if (((u->add_flags | u->remove_flags) & + (MAIL_SEEN | MAIL_DELETED)) == 0) { + /* we're not modifying any counted/lowwatered flags */ + for (seq = seq1; seq <= seq2; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + rec->flags = (rec->flags & flag_mask) | u->add_flags; + } + } else { + for (seq = seq1; seq <= seq2; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + + old_flags = rec->flags; + rec->flags = (rec->flags & flag_mask) | u->add_flags; + + mail_index_header_update_lowwaters(ctx, rec->uid, + rec->flags); + mail_index_sync_header_update_counts_all(ctx, rec->uid, + old_flags, + rec->flags); + } + } + return 1; +} + +static int sync_header_update(const struct mail_transaction_header_update *u, + struct mail_index_sync_map_ctx *ctx) +{ +#define MAIL_INDEX_HEADER_UPDATE_FIELD_IN_RANGE(u, field) \ + ((u)->offset <= offsetof(struct mail_index_header, field) && \ + (u)->offset + (u)->size > offsetof(struct mail_index_header, field)) + struct mail_index_map *map = ctx->view->map; + uint32_t orig_log_file_tail_offset = map->hdr.log_file_tail_offset; + uint32_t orig_next_uid = map->hdr.next_uid; + + if (u->offset >= map->hdr.base_header_size || + u->offset + u->size > map->hdr.base_header_size) { + mail_index_sync_set_corrupted(ctx, + "Header update outside range: %u + %u > %u", + u->offset, u->size, map->hdr.base_header_size); + return -1; + } + + buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size); + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); + + /* @UNSAFE */ + if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) { + memcpy(PTR_OFFSET(&map->hdr, u->offset), + u + 1, u->size); + } else if (u->offset < sizeof(map->hdr)) { + memcpy(PTR_OFFSET(&map->hdr, u->offset), + u + 1, sizeof(map->hdr) - u->offset); + } + + if (map->hdr.next_uid < orig_next_uid) { + /* next_uid update tried to shrink its value. this can happen + in some race conditions with e.g. with dsync, so just + silently ignore it. */ + map->hdr.next_uid = orig_next_uid; + } + + /* the tail offset updates are intended for internal transaction + log handling. we'll update the offset in the header only when + the sync is finished. */ + map->hdr.log_file_tail_offset = orig_log_file_tail_offset; + return 1; +} + +static int +mail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_header *hdr, + const void *data) +{ + int ret = 0; + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_APPEND: { + const struct mail_index_record *rec, *end; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (rec = data; rec < end; rec++) { + ret = sync_append(rec, ctx); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_EXPUNGE: + case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: { + const struct mail_transaction_expunge *rec = data, *end; + ARRAY_TYPE(seq_range) seqs; + uint32_t seq1, seq2; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + break; + } + t_array_init(&seqs, 64); + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (mail_index_lookup_seq_range(ctx->view, + rec->uid1, rec->uid2, &seq1, &seq2)) + seq_range_array_add_range(&seqs, seq1, seq2); + } + sync_expunge_range(ctx, &seqs); + break; + } + case MAIL_TRANSACTION_EXPUNGE_GUID: + case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: { + const struct mail_transaction_expunge_guid *rec = data, *end; + ARRAY_TYPE(seq_range) seqs; + uint32_t seq; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + break; + } + t_array_init(&seqs, 64); + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + i_assert(rec->uid != 0); + + if (mail_index_lookup_seq(ctx->view, rec->uid, &seq)) + seq_range_array_add(&seqs, seq); + } + + sync_expunge_range(ctx, &seqs); + break; + } + case MAIL_TRANSACTION_FLAG_UPDATE: { + const struct mail_transaction_flag_update *rec, *end; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (rec = data; rec < end; rec++) { + ret = sync_flag_update(rec, ctx); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_HEADER_UPDATE: { + const struct mail_transaction_header_update *rec; + unsigned int i; + + for (i = 0; i < hdr->size; ) { + rec = CONST_PTR_OFFSET(data, i); + ret = sync_header_update(rec, ctx); + if (ret <= 0) + break; + + i += sizeof(*rec) + rec->size; + if ((i % 4) != 0) + i += 4 - (i % 4); + } + break; + } + case MAIL_TRANSACTION_EXT_INTRO: { + const struct mail_transaction_ext_intro *rec = data; + unsigned int i; + uint32_t prev_seq; + uoff_t prev_offset; + + mail_transaction_log_view_get_prev_pos(ctx->view->log_view, + &prev_seq, &prev_offset); + ctx->ext_intro_seq = prev_seq; + ctx->ext_intro_offset = prev_offset; + ctx->ext_intro_end_offset = + prev_offset + hdr->size + sizeof(*hdr); + + for (i = 0; i < hdr->size; ) { + if (i + sizeof(*rec) > hdr->size) { + /* should be just extra padding */ + break; + } + + rec = CONST_PTR_OFFSET(data, i); + /* name_size checked by _log_view_next() */ + i_assert(i + sizeof(*rec) + rec->name_size <= hdr->size); + + ret = mail_index_sync_ext_intro(ctx, rec); + if (ret <= 0) + break; + + i += sizeof(*rec) + rec->name_size; + if ((i % 4) != 0) + i += 4 - (i % 4); + } + break; + } + case MAIL_TRANSACTION_EXT_RESET: { + struct mail_transaction_ext_reset rec; + + /* old versions have only new_reset_id */ + if (hdr->size < sizeof(uint32_t)) { + mail_index_sync_set_corrupted(ctx, + "ext reset: invalid record size"); + ret = -1; + break; + } + i_zero(&rec); + memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec))); + ret = mail_index_sync_ext_reset(ctx, &rec); + break; + } + case MAIL_TRANSACTION_EXT_HDR_UPDATE: { + const struct mail_transaction_ext_hdr_update *rec; + unsigned int i; + + for (i = 0; i < hdr->size; ) { + rec = CONST_PTR_OFFSET(data, i); + + if (i + sizeof(*rec) > hdr->size || + i + sizeof(*rec) + rec->size > hdr->size) { + mail_index_sync_set_corrupted(ctx, + "ext hdr update: invalid record size"); + ret = -1; + break; + } + + ret = mail_index_sync_ext_hdr_update(ctx, rec->offset, + rec->size, rec + 1); + if (ret <= 0) + break; + + i += sizeof(*rec) + rec->size; + if ((i % 4) != 0) + i += 4 - (i % 4); + } + break; + } + case MAIL_TRANSACTION_EXT_HDR_UPDATE32: { + const struct mail_transaction_ext_hdr_update32 *rec; + unsigned int i; + + for (i = 0; i < hdr->size; ) { + rec = CONST_PTR_OFFSET(data, i); + + if (i + sizeof(*rec) > hdr->size || + i + sizeof(*rec) + rec->size > hdr->size) { + mail_index_sync_set_corrupted(ctx, + "ext hdr update: invalid record size"); + ret = -1; + break; + } + + ret = mail_index_sync_ext_hdr_update(ctx, rec->offset, + rec->size, rec + 1); + if (ret <= 0) + break; + + i += sizeof(*rec) + rec->size; + if ((i % 4) != 0) + i += 4 - (i % 4); + } + break; + } + case MAIL_TRANSACTION_EXT_REC_UPDATE: { + const struct mail_transaction_ext_rec_update *rec; + unsigned int i, record_size; + + if (ctx->cur_ext_map_idx == (uint32_t)-1) { + mail_index_sync_set_corrupted(ctx, + "Extension record updated " + "without intro prefix"); + ret = -1; + break; + } + + if (ctx->cur_ext_ignore) { + ret = 1; + break; + } + + /* the record is padded to 32bits in the transaction log */ + record_size = (sizeof(*rec) + ctx->cur_ext_record_size + 3) & ~3U; + + for (i = 0; i < hdr->size; i += record_size) { + rec = CONST_PTR_OFFSET(data, i); + + if (i + record_size > hdr->size) { + mail_index_sync_set_corrupted(ctx, + "ext rec update: invalid record size"); + ret = -1; + break; + } + + ret = mail_index_sync_ext_rec_update(ctx, rec); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_EXT_ATOMIC_INC: { + const struct mail_transaction_ext_atomic_inc *rec, *end; + + if (ctx->cur_ext_map_idx == (uint32_t)-1) { + mail_index_sync_set_corrupted(ctx, + "Extension record updated " + "without intro prefix"); + ret = -1; + break; + } + + if (ctx->cur_ext_ignore) { + ret = 1; + break; + } + + end = CONST_PTR_OFFSET(data, hdr->size); + for (rec = data; rec < end; rec++) { + ret = mail_index_sync_ext_atomic_inc(ctx, rec); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_KEYWORD_UPDATE: { + const struct mail_transaction_keyword_update *rec = data; + + ret = mail_index_sync_keywords(ctx, hdr, rec); + break; + } + case MAIL_TRANSACTION_KEYWORD_RESET: { + const struct mail_transaction_keyword_reset *rec = data; + + ret = mail_index_sync_keywords_reset(ctx, hdr, rec); + break; + } + case MAIL_TRANSACTION_MODSEQ_UPDATE: { + const struct mail_transaction_modseq_update *rec = data; + + ret = sync_modseq_update(ctx, rec, hdr->size); + break; + } + case MAIL_TRANSACTION_INDEX_DELETED: + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* next sync finishes the deletion */ + ctx->view->index->index_delete_requested = TRUE; + } else { + /* transaction log reading handles this */ + } + break; + case MAIL_TRANSACTION_INDEX_UNDELETED: + ctx->view->index->index_delete_requested = FALSE; + break; + case MAIL_TRANSACTION_BOUNDARY: + break; + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: + break; + default: + mail_index_sync_set_corrupted(ctx, + "Unknown transaction record type 0x%x", + (hdr->type & MAIL_TRANSACTION_TYPE_MASK)); + ret = -1; + break; + } + return ret; +} + +int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx, + const struct mail_transaction_header *hdr, + const void *data) +{ + int ret; + + T_BEGIN { + ret = mail_index_sync_record_real(ctx, hdr, data); + } T_END; + return ret; +} + +void mail_index_sync_map_init(struct mail_index_sync_map_ctx *sync_map_ctx, + struct mail_index_view *view, + enum mail_index_sync_handler_type type) +{ + i_zero(sync_map_ctx); + sync_map_ctx->view = view; + sync_map_ctx->cur_ext_map_idx = (uint32_t)-1; + sync_map_ctx->type = type; + sync_map_ctx->modseq_ctx = mail_index_modseq_sync_begin(sync_map_ctx); + + mail_index_sync_init_handlers(sync_map_ctx); +} + +void mail_index_sync_map_deinit(struct mail_index_sync_map_ctx *sync_map_ctx) +{ + i_assert(sync_map_ctx->modseq_ctx == NULL); + + buffer_free(&sync_map_ctx->unknown_extensions); + if (sync_map_ctx->expunge_handlers_used) + mail_index_sync_deinit_expunge_handlers(sync_map_ctx); + mail_index_sync_deinit_handlers(sync_map_ctx); +} + +static void mail_index_sync_update_hdr_dirty_flag(struct mail_index_map *map) +{ + const struct mail_index_record *rec; + uint32_t seq; + + if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 || + (map->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) != 0) + return; + + /* do we have dirty flags anymore? */ + for (seq = 1; seq <= map->rec_map->records_count; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(map, seq); + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + break; + } + } +} + +#ifdef DEBUG +void mail_index_map_check(struct mail_index_map *map) +{ + const struct mail_index_header *hdr = &map->hdr; + unsigned int del = 0, seen = 0; + uint32_t seq, prev_uid = 0; + + if (getenv("DEBUG_IGNORE_INDEX_CORRUPTION") != NULL) + return; + + i_assert(hdr->messages_count <= map->rec_map->records_count); + for (seq = 1; seq <= hdr->messages_count; seq++) { + const struct mail_index_record *rec; + + rec = MAIL_INDEX_REC_AT_SEQ(map, seq); + i_assert(rec->uid > prev_uid); + prev_uid = rec->uid; + + if ((rec->flags & MAIL_DELETED) != 0) { + i_assert(rec->uid >= hdr->first_deleted_uid_lowwater); + del++; + } + if ((rec->flags & MAIL_SEEN) != 0) + seen++; + else + i_assert(rec->uid >= hdr->first_unseen_uid_lowwater); + } + i_assert(del == hdr->deleted_messages_count); + i_assert(seen == hdr->seen_messages_count); +} +#endif + +bool mail_index_sync_map_want_index_reopen(struct mail_index_map *map, + enum mail_index_sync_handler_type type) +{ + struct mail_index *index = map->index; + + if (index->log->head == NULL) + return TRUE; + + uoff_t start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ? + map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset; + /* don't check this if mmap is disabled, because reopening + index causes sync to get lost. */ + if ((index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) { + uoff_t log_size, index_size; + + if (index->fd == -1 && + index->log->head->hdr.prev_file_seq != 0) { + /* we don't know the index's size, so use the + smallest index size we're willing to read */ + index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE; + } else { + index_size = map->hdr.header_size + + map->rec_map->records_count * + map->hdr.record_size; + } + + /* this isn't necessary correct currently, but it should be + close enough */ + log_size = index->log->head->last_size; + if (log_size > start_offset && + log_size - start_offset > index_size) + return TRUE; + } + return FALSE; +} + +int mail_index_sync_map(struct mail_index_map **_map, + enum mail_index_sync_handler_type type, + const char **reason_r) +{ + struct mail_index_map *map = *_map; + struct mail_index *index = map->index; + struct mail_index_view *view; + struct mail_index_sync_map_ctx sync_map_ctx; + const struct mail_transaction_header *thdr; + const void *tdata; + uint32_t prev_seq; + uoff_t start_offset, prev_offset; + const char *reason, *error; + int ret; + bool had_dirty, reset; + + i_assert(index->log->head != NULL); + i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW); + + start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ? + map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset; + + view = mail_index_view_open_with_map(index, map); + ret = mail_transaction_log_view_set(view->log_view, + map->hdr.log_file_seq, start_offset, + (uint32_t)-1, UOFF_T_MAX, + &reset, &reason); + if (ret <= 0) { + mail_index_view_close(&view); + if (ret < 0) { + /* I/O failure */ + return -1; + } + /* the seq/offset is probably broken */ + *reason_r = t_strdup_printf( + "Lost log for seq=%u offset=%"PRIuUOFF_T": %s " + "(initial_mapped=%d)", + map->hdr.log_file_seq, start_offset, reason, + index->initial_mapped ? 1 : 0); + return 0; + } + + mail_transaction_log_get_head(index->log, &prev_seq, &prev_offset); + if (prev_seq != map->hdr.log_file_seq || + prev_offset - map->hdr.log_file_tail_offset > + index->optimization_set.index.rewrite_min_log_bytes) { + /* we're reading more from log than we would have preferred. + remember that we probably want to rewrite index soon. */ + index->index_min_write = TRUE; + } + + /* view referenced the map. avoid unnecessary map cloning by + unreferencing the map while view exists. */ + map->refcount--; + + had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0; + if (had_dirty) + map->hdr.flags &= ENUM_NEGATE(MAIL_INDEX_HDR_FLAG_HAVE_DIRTY); + + mail_transaction_log_view_get_prev_pos(view->log_view, + &prev_seq, &prev_offset); + + mail_index_sync_map_init(&sync_map_ctx, view, type); + if (reset) { + /* Reset the entire index. Leave only indexid and + log_file_seq. */ + mail_transaction_log_view_get_prev_pos(view->log_view, + &prev_seq, &prev_offset); + map = mail_index_map_alloc(index); + if ((index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0) + map->hdr.flags |= MAIL_INDEX_HDR_FLAG_FSCKD; + map->hdr.log_file_seq = prev_seq; + map->hdr.log_file_tail_offset = 0; + mail_index_sync_replace_map(&sync_map_ctx, map); + } + map = NULL; + + /* FIXME: when transaction sync lock is removed, we'll need to handle + the case when a transaction is committed while mailbox is being + synced ([synced transactions][new transaction][ext transaction]). + this means int_offset contains [synced] and ext_offset contains + all */ + while ((ret = mail_transaction_log_view_next(view->log_view, &thdr, + &tdata)) > 0) { + mail_transaction_log_view_get_prev_pos(view->log_view, + &prev_seq, &prev_offset); + + if (LOG_IS_BEFORE(prev_seq, prev_offset, + view->map->hdr.log_file_seq, + view->map->hdr.log_file_head_offset)) { + /* this has been synced already. */ + i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE); + continue; + } + + /* we'll just skip over broken entries */ + (void)mail_index_sync_record(&sync_map_ctx, thdr, tdata); + } + map = view->map; + + if (had_dirty) + mail_index_sync_update_hdr_dirty_flag(map); + mail_index_modseq_sync_end(&sync_map_ctx.modseq_ctx); + + mail_index_sync_update_log_offset(&sync_map_ctx, view->map, TRUE); + +#ifdef DEBUG + mail_index_map_check(map); +#endif + i_assert(map->hdr.indexid == index->indexid || map->hdr.indexid == 0); + + /* transaction log tracks internally the current tail offset. + besides using header updates, it also updates the offset to skip + over following external transactions to avoid extra unneeded log + reading. */ + i_assert(map->hdr.log_file_seq == index->log->head->hdr.file_seq); + if (map->hdr.log_file_tail_offset < index->log->head->max_tail_offset) { + map->hdr.log_file_tail_offset = + index->log->head->max_tail_offset; + } + + buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr)); + if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { + memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data, + map->hdr_copy_buf->used); + } + + /* restore refcount before closing the view. this is necessary also + if map got cloned, because view closing would otherwise destroy it */ + map->refcount++; + mail_index_sync_map_deinit(&sync_map_ctx); + mail_index_view_close(&view); + + i_assert(index->map == map || type == MAIL_INDEX_SYNC_HANDLER_VIEW); + + if (mail_index_map_check_header(map, &error) <= 0) { + mail_index_set_error(index, + "Synchronization corrupted index header %s: %s", + index->filepath, error); + (void)mail_index_fsck(index); + map = index->map; + } else if (sync_map_ctx.errors) { + /* make sure the index looks valid now */ + (void)mail_index_fsck(index); + map = index->map; + } + + *_map = map; + return ret < 0 ? -1 : 1; +} diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c new file mode 100644 index 0000000..6322ee1 --- /dev/null +++ b/src/lib-index/mail-index-sync.c @@ -0,0 +1,1062 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-index-transaction-private.h" +#include "mail-transaction-log-private.h" +#include "mail-cache-private.h" + +#include <stdio.h> + +struct mail_index_sync_ctx { + struct mail_index *index; + struct mail_index_view *view; + struct mail_index_transaction *sync_trans, *ext_trans; + struct mail_index_transaction_commit_result *sync_commit_result; + enum mail_index_sync_flags flags; + char *reason; + + const struct mail_transaction_header *hdr; + const void *data; + + ARRAY(struct mail_index_sync_list) sync_list; + uint32_t next_uid; + + bool no_warning:1; + bool seen_external_expunges:1; + bool seen_nonexternal_transactions:1; + bool fully_synced:1; +}; + +static void mail_index_sync_add_expunge(struct mail_index_sync_ctx *ctx) +{ + const struct mail_transaction_expunge *e = ctx->data; + size_t i, size = ctx->hdr->size / sizeof(*e); + uint32_t uid; + + for (i = 0; i < size; i++) { + for (uid = e[i].uid1; uid <= e[i].uid2; uid++) + mail_index_expunge(ctx->sync_trans, uid); + } +} + +static void mail_index_sync_add_expunge_guid(struct mail_index_sync_ctx *ctx) +{ + const struct mail_transaction_expunge_guid *e = ctx->data; + size_t i, size = ctx->hdr->size / sizeof(*e); + + for (i = 0; i < size; i++) { + mail_index_expunge_guid(ctx->sync_trans, e[i].uid, + e[i].guid_128); + } +} + +static void mail_index_sync_add_flag_update(struct mail_index_sync_ctx *ctx) +{ + const struct mail_transaction_flag_update *u = ctx->data; + size_t i, size = ctx->hdr->size / sizeof(*u); + + for (i = 0; i < size; i++) { + if (u[i].add_flags != 0) { + mail_index_update_flags_range(ctx->sync_trans, + u[i].uid1, u[i].uid2, + MODIFY_ADD, + u[i].add_flags); + } + if (u[i].remove_flags != 0) { + mail_index_update_flags_range(ctx->sync_trans, + u[i].uid1, u[i].uid2, + MODIFY_REMOVE, + u[i].remove_flags); + } + } +} + +static void mail_index_sync_add_keyword_update(struct mail_index_sync_ctx *ctx) +{ + const struct mail_transaction_keyword_update *u = ctx->data; + const char *keyword_names[2]; + struct mail_keywords *keywords; + const uint32_t *uids; + uint32_t uid; + size_t uidset_offset, i, size; + + i_assert(u->name_size > 0); + + uidset_offset = sizeof(*u) + u->name_size; + if ((uidset_offset % 4) != 0) + uidset_offset += 4 - (uidset_offset % 4); + uids = CONST_PTR_OFFSET(u, uidset_offset); + + keyword_names[0] = t_strndup(u + 1, u->name_size); + keyword_names[1] = NULL; + keywords = mail_index_keywords_create(ctx->index, keyword_names); + + size = (ctx->hdr->size - uidset_offset) / sizeof(uint32_t); + for (i = 0; i < size; i += 2) { + /* FIXME: mail_index_update_keywords_range() */ + for (uid = uids[i]; uid <= uids[i+1]; uid++) { + mail_index_update_keywords(ctx->sync_trans, uid, + u->modify_type, keywords); + } + } + + mail_index_keywords_unref(&keywords); +} + +static void mail_index_sync_add_keyword_reset(struct mail_index_sync_ctx *ctx) +{ + const struct mail_transaction_keyword_reset *u = ctx->data; + size_t i, size = ctx->hdr->size / sizeof(*u); + struct mail_keywords *keywords; + uint32_t uid; + + keywords = mail_index_keywords_create(ctx->index, NULL); + for (i = 0; i < size; i++) { + for (uid = u[i].uid1; uid <= u[i].uid2; uid++) { + mail_index_update_keywords(ctx->sync_trans, uid, + MODIFY_REPLACE, keywords); + } + } + mail_index_keywords_unref(&keywords); +} + +static bool mail_index_sync_add_transaction(struct mail_index_sync_ctx *ctx) +{ + switch (ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + mail_index_sync_add_expunge(ctx); + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: + mail_index_sync_add_expunge_guid(ctx); + break; + case MAIL_TRANSACTION_FLAG_UPDATE: + mail_index_sync_add_flag_update(ctx); + break; + case MAIL_TRANSACTION_KEYWORD_UPDATE: + mail_index_sync_add_keyword_update(ctx); + break; + case MAIL_TRANSACTION_KEYWORD_RESET: + mail_index_sync_add_keyword_reset(ctx); + break; + default: + return FALSE; + } + return TRUE; +} + +static void mail_index_sync_add_dirty_updates(struct mail_index_sync_ctx *ctx) +{ + struct mail_transaction_flag_update update; + const struct mail_index_record *rec; + uint32_t seq, messages_count; + + i_zero(&update); + + messages_count = mail_index_view_get_messages_count(ctx->view); + for (seq = 1; seq <= messages_count; seq++) { + rec = mail_index_lookup(ctx->view, seq); + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0) + continue; + + mail_index_update_flags(ctx->sync_trans, rec->uid, + MODIFY_REPLACE, rec->flags); + } +} + +static int +mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx) +{ + struct mail_index_transaction *sync_trans = ctx->sync_trans; + struct mail_index_sync_list *synclist; + const struct mail_index_transaction_keyword_update *keyword_updates; + unsigned int i, keyword_count; + int ret; + + if ((ctx->view->map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 && + (ctx->flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0 && + (ctx->view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0) { + /* show dirty flags as flag updates */ + mail_index_sync_add_dirty_updates(ctx); + } + + /* read all transactions from log into a transaction in memory. + skip the external ones, they're already synced to mailbox and + included in our view */ + while ((ret = mail_transaction_log_view_next(ctx->view->log_view, + &ctx->hdr, + &ctx->data)) > 0) { + if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) { + if ((ctx->hdr->type & (MAIL_TRANSACTION_EXPUNGE | + MAIL_TRANSACTION_EXPUNGE_GUID)) != 0) + ctx->seen_external_expunges = TRUE; + continue; + } + + T_BEGIN { + if (mail_index_sync_add_transaction(ctx)) { + /* update tail_offset if needed */ + ctx->seen_nonexternal_transactions = TRUE; + } else { + /* this is an internal change. we don't + necessarily need to update tail_offset, so + avoid the extra write caused by it. */ + } + } T_END; + } + + /* create an array containing all expunge, flag and keyword update + arrays so we can easily go through all of the changes. */ + keyword_count = !array_is_created(&sync_trans->keyword_updates) ? 0 : + array_count(&sync_trans->keyword_updates); + i_array_init(&ctx->sync_list, keyword_count + 2); + + if (array_is_created(&sync_trans->expunges)) { + mail_index_transaction_sort_expunges(sync_trans); + synclist = array_append_space(&ctx->sync_list); + synclist->array = (void *)&sync_trans->expunges; + } + + if (array_is_created(&sync_trans->updates)) { + synclist = array_append_space(&ctx->sync_list); + synclist->array = (void *)&sync_trans->updates; + } + + keyword_updates = keyword_count == 0 ? NULL : + array_front(&sync_trans->keyword_updates); + for (i = 0; i < keyword_count; i++) { + if (array_is_created(&keyword_updates[i].add_seq)) { + synclist = array_append_space(&ctx->sync_list); + synclist->array = + (const void *)&keyword_updates[i].add_seq; + synclist->keyword_idx = i; + } + if (array_is_created(&keyword_updates[i].remove_seq)) { + synclist = array_append_space(&ctx->sync_list); + synclist->array = + (const void *)&keyword_updates[i].remove_seq; + synclist->keyword_idx = i; + synclist->keyword_remove = TRUE; + } + } + + return ret; +} + +static bool +mail_index_need_sync(struct mail_index *index, enum mail_index_sync_flags flags, + uint32_t log_file_seq, uoff_t log_file_offset) +{ + const struct mail_index_header *hdr = &index->map->hdr; + if ((flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) == 0) + return TRUE; + + /* sync only if there's something to do */ + if (hdr->first_recent_uid < hdr->next_uid && + (flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0) + return TRUE; + + if ((hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 && + (flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0 && + (index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0) + return TRUE; + + if (log_file_seq == (uint32_t)-1) { + /* we want to sync up to transaction log's head */ + mail_transaction_log_get_head(index->log, + &log_file_seq, &log_file_offset); + } + if ((hdr->log_file_tail_offset < log_file_offset && + hdr->log_file_seq == log_file_seq) || + hdr->log_file_seq < log_file_seq) + return TRUE; + + if (index->need_recreate != NULL) + return TRUE; + + /* already synced */ + const char *reason; + return mail_cache_need_purge(index->cache, &reason); +} + +static int +mail_index_sync_set_log_view(struct mail_index_view *view, + uint32_t start_file_seq, uoff_t start_file_offset) +{ + uint32_t log_seq; + uoff_t log_offset; + const char *reason; + bool reset; + int ret; + + mail_transaction_log_get_head(view->index->log, &log_seq, &log_offset); + + ret = mail_transaction_log_view_set(view->log_view, + start_file_seq, start_file_offset, + log_seq, log_offset, &reset, &reason); + if (ret < 0) + return -1; + if (ret == 0) { + /* either corrupted or the file was deleted for + some reason. either way, we can't go forward */ + mail_index_set_error(view->index, + "Unexpected transaction log desync with index %s: %s", + view->index->filepath, reason); + return 0; + } + return 1; +} + +int mail_index_sync_begin(struct mail_index *index, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + enum mail_index_sync_flags flags) +{ + int ret; + + ret = mail_index_sync_begin_to(index, ctx_r, view_r, trans_r, + (uint32_t)-1, UOFF_T_MAX, flags); + i_assert(ret != 0 || + (flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) != 0); + return ret; +} + +static int +mail_index_sync_begin_init(struct mail_index *index, + enum mail_index_sync_flags flags, + uint32_t log_file_seq, uoff_t log_file_offset) +{ + const struct mail_index_header *hdr; + uint32_t seq; + uoff_t offset; + bool locked = FALSE; + int ret; + + /* if we require changes, don't lock transaction log yet. first check + if there's anything to sync. */ + if ((flags & MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES) == 0) { + if (mail_transaction_log_sync_lock(index->log, "syncing", + &seq, &offset) < 0) + return -1; + locked = TRUE; + } + + /* The view must contain what we expect the mailbox to look like + currently. That allows the backend to update external flag + changes (etc.) if the view doesn't match the mailbox. + + We'll update the view to contain everything that exist in the + transaction log except for expunges. They're synced in + mail_index_sync_commit(). */ + if ((ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD)) <= 0) { + if (ret == 0) { + if (locked) + mail_transaction_log_sync_unlock(index->log, "sync init failure"); + return -1; + } + + /* let's try again */ + if (mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0) { + if (locked) + mail_transaction_log_sync_unlock(index->log, "sync init failure"); + return -1; + } + } + + if (!mail_index_need_sync(index, flags, log_file_seq, log_file_offset) && + !index->index_deleted && index->need_recreate == NULL) { + if (locked) + mail_transaction_log_sync_unlock(index->log, "syncing determined unnecessary"); + return 0; + } + + if (!locked) { + /* it looks like we have something to sync. lock the file and + check again. */ + flags &= ENUM_NEGATE(MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES); + return mail_index_sync_begin_init(index, flags, log_file_seq, + log_file_offset); + } + + if (index->index_deleted && + (flags & MAIL_INDEX_SYNC_FLAG_DELETING_INDEX) == 0) { + /* index is already deleted. we can't sync. */ + if (locked) + mail_transaction_log_sync_unlock(index->log, "syncing detected deleted index"); + return -1; + } + + hdr = &index->map->hdr; + if (hdr->log_file_tail_offset > hdr->log_file_head_offset || + hdr->log_file_seq > seq || + (hdr->log_file_seq == seq && hdr->log_file_tail_offset > offset)) { + /* broken sync positions. fix them. */ + mail_index_set_error(index, + "broken sync positions in index file %s", + index->filepath); + mail_index_fsck_locked(index); + } + return 1; +} + +static int +mail_index_sync_begin_to2(struct mail_index *index, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + uint32_t log_file_seq, uoff_t log_file_offset, + enum mail_index_sync_flags flags, bool *retry_r) +{ + const struct mail_index_header *hdr; + struct mail_index_sync_ctx *ctx; + struct mail_index_view *sync_view; + enum mail_index_transaction_flags trans_flags; + int ret; + + i_assert(!index->syncing); + + *retry_r = FALSE; + + if (index->map != NULL && + (index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { + /* index is corrupted and need to be reopened */ + return -1; + } + + if (log_file_seq != (uint32_t)-1) + flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; + + ret = mail_index_sync_begin_init(index, flags, log_file_seq, + log_file_offset); + if (ret <= 0) + return ret; + + hdr = &index->map->hdr; + + ctx = i_new(struct mail_index_sync_ctx, 1); + ctx->index = index; + ctx->flags = flags; + + ctx->view = mail_index_view_open(index); + + sync_view = mail_index_dummy_view_open(index); + ctx->sync_trans = mail_index_transaction_begin(sync_view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_view_close(&sync_view); + + /* set before any rollbacks are called */ + index->syncing = TRUE; + + /* we wish to see all the changes from last mailbox sync position to + the end of the transaction log */ + ret = mail_index_sync_set_log_view(ctx->view, hdr->log_file_seq, + hdr->log_file_tail_offset); + if (ret < 0) { + mail_index_sync_rollback(&ctx); + return -1; + } + if (ret == 0) { + /* if a log file is missing, there's nothing we can do except + to skip over it. fix the problem with fsck and try again. */ + mail_index_fsck_locked(index); + mail_index_sync_rollback(&ctx); + *retry_r = TRUE; + return 0; + } + + /* we need to have all the transactions sorted to optimize + caller's mailbox access patterns */ + if (mail_index_sync_read_and_sort(ctx) < 0) { + mail_index_sync_rollback(&ctx); + return -1; + } + + /* create the transaction after the view has been updated with + external transactions and marked as sync view */ + trans_flags = MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL; + if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES) != 0) + trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES; + if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_FSYNC) != 0) + trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_FSYNC; + ctx->ext_trans = mail_index_transaction_begin(ctx->view, trans_flags); + ctx->ext_trans->sync_transaction = TRUE; + ctx->ext_trans->commit_deleted_index = + (flags & (MAIL_INDEX_SYNC_FLAG_DELETING_INDEX | + MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX)) != 0; + + *ctx_r = ctx; + *view_r = ctx->view; + *trans_r = ctx->ext_trans; + return 1; +} + +int mail_index_sync_begin_to(struct mail_index *index, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + uint32_t log_file_seq, uoff_t log_file_offset, + enum mail_index_sync_flags flags) +{ + bool retry; + int ret; + + i_assert(index->open_count > 0); + + ret = mail_index_sync_begin_to2(index, ctx_r, view_r, trans_r, + log_file_seq, log_file_offset, + flags, &retry); + if (retry) { + ret = mail_index_sync_begin_to2(index, ctx_r, view_r, trans_r, + log_file_seq, log_file_offset, + flags, &retry); + } + return ret; +} + +bool mail_index_sync_has_expunges(struct mail_index_sync_ctx *ctx) +{ + return array_is_created(&ctx->sync_trans->expunges) && + array_count(&ctx->sync_trans->expunges) > 0; +} + +static bool mail_index_sync_view_have_any(struct mail_index_view *view, + enum mail_index_sync_flags flags, + bool expunges_only) +{ + const struct mail_transaction_header *hdr; + const void *data; + uint32_t log_seq; + uoff_t log_offset; + const char *reason; + bool reset; + int ret; + + if (view->map->hdr.first_recent_uid < view->map->hdr.next_uid && + (flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0) + return TRUE; + + if ((view->map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 && + (flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0 && + (view->index->flags & MAIL_INDEX_OPEN_FLAG_NO_DIRTY) == 0) + return TRUE; + + mail_transaction_log_get_head(view->index->log, &log_seq, &log_offset); + if (mail_transaction_log_view_set(view->log_view, + view->map->hdr.log_file_seq, + view->map->hdr.log_file_tail_offset, + log_seq, log_offset, + &reset, &reason) <= 0) { + /* let the actual syncing handle the error */ + return TRUE; + } + + while ((ret = mail_transaction_log_view_next(view->log_view, + &hdr, &data)) > 0) { + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) + continue; + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + case MAIL_TRANSACTION_EXPUNGE_GUID: + return TRUE; + case MAIL_TRANSACTION_EXT_REC_UPDATE: + case MAIL_TRANSACTION_EXT_ATOMIC_INC: + /* extension record updates aren't exactly needed + to be synced, but cache syncing relies on tail + offsets being updated. */ + case MAIL_TRANSACTION_FLAG_UPDATE: + case MAIL_TRANSACTION_KEYWORD_UPDATE: + case MAIL_TRANSACTION_KEYWORD_RESET: + case MAIL_TRANSACTION_INDEX_DELETED: + case MAIL_TRANSACTION_INDEX_UNDELETED: + if (!expunges_only) + return TRUE; + break; + default: + break; + } + } + return ret < 0; +} + +bool mail_index_sync_have_any(struct mail_index *index, + enum mail_index_sync_flags flags) +{ + struct mail_index_view *view; + bool ret; + + view = mail_index_view_open(index); + ret = mail_index_sync_view_have_any(view, flags, FALSE); + mail_index_view_close(&view); + return ret; +} + +bool mail_index_sync_have_any_expunges(struct mail_index *index) +{ + struct mail_index_view *view; + bool ret; + + view = mail_index_view_open(index); + ret = mail_index_sync_view_have_any(view, 0, TRUE); + mail_index_view_close(&view); + return ret; +} + +void mail_index_sync_get_offsets(struct mail_index_sync_ctx *ctx, + uint32_t *seq1_r, uoff_t *offset1_r, + uint32_t *seq2_r, uoff_t *offset2_r) +{ + *seq1_r = ctx->view->map->hdr.log_file_seq; + *offset1_r = ctx->view->map->hdr.log_file_tail_offset != 0 ? + ctx->view->map->hdr.log_file_tail_offset : + ctx->view->index->log->head->hdr.hdr_size; + mail_transaction_log_get_head(ctx->view->index->log, seq2_r, offset2_r); +} + +static void +mail_index_sync_get_expunge(struct mail_index_sync_rec *rec, + const struct mail_transaction_expunge_guid *exp) +{ + rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE; + rec->uid1 = exp->uid; + rec->uid2 = exp->uid; + memcpy(rec->guid_128, exp->guid_128, sizeof(rec->guid_128)); +} + +static void +mail_index_sync_get_update(struct mail_index_sync_rec *rec, + const struct mail_index_flag_update *update) +{ + rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS; + rec->uid1 = update->uid1; + rec->uid2 = update->uid2; + + rec->add_flags = update->add_flags; + rec->remove_flags = update->remove_flags; +} + +static void +mail_index_sync_get_keyword_update(struct mail_index_sync_rec *rec, + const struct uid_range *range, + struct mail_index_sync_list *sync_list) +{ + rec->type = !sync_list->keyword_remove ? + MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD : + MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE; + rec->uid1 = range->uid1; + rec->uid2 = range->uid2; + rec->keyword_idx = sync_list->keyword_idx; +} + +bool mail_index_sync_next(struct mail_index_sync_ctx *ctx, + struct mail_index_sync_rec *sync_rec) +{ + struct mail_index_transaction *sync_trans = ctx->sync_trans; + struct mail_index_sync_list *sync_list; + const struct uid_range *uid_range = NULL; + unsigned int i, count, next_i; + uint32_t next_found_uid; + + next_i = UINT_MAX; + next_found_uid = (uint32_t)-1; + + /* FIXME: replace with a priority queue so we don't have to go + through the whole list constantly. and remember to make sure that + keyword resets are sent before adds! */ + /* FIXME: pretty ugly to do this for expunges, which isn't even a + seq_range. */ + sync_list = array_get_modifiable(&ctx->sync_list, &count); + for (i = 0; i < count; i++) { + if (!array_is_created(sync_list[i].array) || + sync_list[i].idx == array_count(sync_list[i].array)) + continue; + + uid_range = array_idx(sync_list[i].array, sync_list[i].idx); + if (uid_range->uid1 == ctx->next_uid) { + /* use this one. */ + break; + } + if (uid_range->uid1 < next_found_uid) { + next_i = i; + next_found_uid = uid_range->uid1; + } + } + + if (i == count) { + if (next_i == UINT_MAX) { + /* nothing left in sync_list */ + ctx->fully_synced = TRUE; + return FALSE; + } + ctx->next_uid = next_found_uid; + i = next_i; + uid_range = array_idx(sync_list[i].array, sync_list[i].idx); + } + + if (sync_list[i].array == (void *)&sync_trans->expunges) { + mail_index_sync_get_expunge(sync_rec, + (const struct mail_transaction_expunge_guid *)uid_range); + } else if (sync_list[i].array == (void *)&sync_trans->updates) { + mail_index_sync_get_update(sync_rec, + (const struct mail_index_flag_update *)uid_range); + } else { + mail_index_sync_get_keyword_update(sync_rec, uid_range, + &sync_list[i]); + } + sync_list[i].idx++; + return TRUE; +} + +bool mail_index_sync_have_more(struct mail_index_sync_ctx *ctx) +{ + const struct mail_index_sync_list *sync_list; + + array_foreach(&ctx->sync_list, sync_list) { + if (array_is_created(sync_list->array) && + sync_list->idx != array_count(sync_list->array)) + return TRUE; + } + return FALSE; +} + +void mail_index_sync_set_commit_result(struct mail_index_sync_ctx *ctx, + struct mail_index_transaction_commit_result *result) +{ + ctx->sync_commit_result = result; +} + +void mail_index_sync_reset(struct mail_index_sync_ctx *ctx) +{ + struct mail_index_sync_list *sync_list; + + ctx->next_uid = 0; + array_foreach_modifiable(&ctx->sync_list, sync_list) + sync_list->idx = 0; +} + +void mail_index_sync_no_warning(struct mail_index_sync_ctx *ctx) +{ + ctx->no_warning = TRUE; +} + +void mail_index_sync_set_reason(struct mail_index_sync_ctx *ctx, + const char *reason) +{ + i_free(ctx->reason); + ctx->reason = i_strdup(reason); +} + +static void mail_index_sync_end(struct mail_index_sync_ctx **_ctx) +{ + struct mail_index_sync_ctx *ctx = *_ctx; + const char *lock_reason; + + i_assert(ctx->index->syncing); + + *_ctx = NULL; + + ctx->index->syncing = FALSE; + if (ctx->no_warning) + lock_reason = NULL; + else if (ctx->reason != NULL) + lock_reason = ctx->reason; + else + lock_reason = "Mailbox was synchronized"; + mail_transaction_log_sync_unlock(ctx->index->log, lock_reason); + + mail_index_view_close(&ctx->view); + mail_index_transaction_rollback(&ctx->sync_trans); + if (array_is_created(&ctx->sync_list)) + array_free(&ctx->sync_list); + i_free(ctx->reason); + i_free(ctx); +} + +static void +mail_index_sync_update_mailbox_offset(struct mail_index_sync_ctx *ctx) +{ + const struct mail_index_header *hdr = &ctx->index->map->hdr; + uint32_t seq; + uoff_t offset; + + if (!ctx->fully_synced) { + /* Everything wasn't synced. This usually means that syncing + was used for locking and nothing was synced. Don't update + tail offset. */ + return; + } + /* All changes were synced. During the syncing other transactions may + have been created and committed as well. They're expected to be + external transactions. These could be at least: + - mdbox finishing expunges + - mdbox writing to dovecot.map.index (requires tail offset updates) + - sdbox appending messages + + If any expunges were committed, tail_offset must not be updated + before mail_index_map(MAIL_INDEX_SYNC_HANDLER_FILE) is called. + Otherwise expunge handlers won't be called for them. + + We'll require MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET flag for the + few places that actually require tail_offset to include the + externally committed transactions. Otherwise tail_offset is updated + only up to what was just synced. */ + if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET) != 0) + mail_transaction_log_get_head(ctx->index->log, &seq, &offset); + else { + mail_transaction_log_view_get_prev_pos(ctx->view->log_view, + &seq, &offset); + } + mail_transaction_log_set_mailbox_sync_pos(ctx->index->log, seq, offset); + + /* If tail offset has changed, make sure it gets written to + transaction log. do this only if we're required to make changes. + + avoid writing a new tail offset if all the transactions were + external, because that wouldn't change effective the tail offset. + except e.g. mdbox map requires this to happen, so do it + optionally. Also update the tail if we've been calling any expunge + handlers, so they won't be called multiple times. That could cause + at least cache file's [deleted_]record_count to shrink too much. */ + if ((hdr->log_file_seq != seq || hdr->log_file_tail_offset < offset) && + (ctx->seen_external_expunges || + ctx->seen_nonexternal_transactions || + (ctx->flags & MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET) != 0)) { + ctx->ext_trans->log_updates = TRUE; + ctx->ext_trans->tail_offset_changed = TRUE; + } +} + +static bool mail_index_sync_want_index_write(struct mail_index *index, const char **reason_r) +{ + uint32_t log_diff; + + if (index->main_index_hdr_log_file_seq != 0 && + index->main_index_hdr_log_file_seq != index->map->hdr.log_file_seq) { + /* dovecot.index points to an old .log file. we were supposed + to rewrite the dovecot.index when rotating the log, so + we shouldn't usually get here. */ + *reason_r = "points to old .log file"; + return TRUE; + } + + log_diff = index->map->hdr.log_file_tail_offset - + index->main_index_hdr_log_file_tail_offset; + if (log_diff > index->optimization_set.index.rewrite_max_log_bytes) { + *reason_r = t_strdup_printf( + ".log read %u..%u > rewrite_max_log_bytes %"PRIuUOFF_T, + index->map->hdr.log_file_tail_offset, + index->main_index_hdr_log_file_tail_offset, + index->optimization_set.index.rewrite_max_log_bytes); + return TRUE; + } + if (index->index_min_write && + log_diff > index->optimization_set.index.rewrite_min_log_bytes) { + *reason_r = t_strdup_printf( + ".log read %u..%u > rewrite_min_log_bytes %"PRIuUOFF_T, + index->map->hdr.log_file_tail_offset, + index->main_index_hdr_log_file_tail_offset, + index->optimization_set.index.rewrite_min_log_bytes); + return TRUE; + } + + if (index->need_recreate != NULL) { + *reason_r = t_strdup_printf("Need to recreate index: %s", + index->need_recreate); + return TRUE; + } + return FALSE; +} + +int mail_index_sync_commit(struct mail_index_sync_ctx **_ctx) +{ + struct mail_index_sync_ctx *ctx = *_ctx; + struct mail_index *index = ctx->index; + const char *reason = NULL; + uint32_t next_uid; + bool want_rotate, index_undeleted, delete_index; + int ret = 0, ret2; + + index_undeleted = ctx->ext_trans->index_undeleted; + delete_index = index->index_delete_requested && !index_undeleted && + (ctx->flags & (MAIL_INDEX_SYNC_FLAG_DELETING_INDEX | + MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX)) != 0; + if (delete_index) { + /* finish this sync by marking the index deleted */ + mail_index_set_deleted(ctx->ext_trans); + } else if (index->index_deleted && !index_undeleted && + (ctx->flags & MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX) == 0) { + /* another process just marked the index deleted. + finish the sync, but return error. */ + mail_index_set_error_nolog(index, "Index is marked deleted"); + ret = -1; + } + + mail_index_sync_update_mailbox_offset(ctx); + + if ((ctx->flags & MAIL_INDEX_SYNC_FLAG_DROP_RECENT) != 0) { + next_uid = mail_index_transaction_get_next_uid(ctx->ext_trans); + if (index->map->hdr.first_recent_uid < next_uid) { + mail_index_update_header(ctx->ext_trans, + offsetof(struct mail_index_header, + first_recent_uid), + &next_uid, sizeof(next_uid), FALSE); + } + } + if (index->hdr_log2_rotate_time_delayed_update != 0) { + /* We checked whether .log.2 should be deleted in this same + sync. It resulted in wanting to change the log2_rotate_time + in the header. Do it here as part of the other changes. */ + uint32_t log2_rotate_time = + index->hdr_log2_rotate_time_delayed_update; + + mail_index_update_header(ctx->ext_trans, + offsetof(struct mail_index_header, log2_rotate_time), + &log2_rotate_time, sizeof(log2_rotate_time), TRUE); + index->hdr_log2_rotate_time_delayed_update = 0; + } + + ret2 = mail_index_transaction_commit(&ctx->ext_trans); + if (ret2 < 0) { + mail_index_sync_end(&ctx); + return -1; + } + + if (delete_index) + index->index_deleted = TRUE; + else if (index_undeleted) { + index->index_deleted = FALSE; + index->index_delete_requested = FALSE; + } + + /* refresh the mapping with newly committed external transactions + and the synced expunges. sync using file handler here so that the + expunge handlers get called. */ + index->sync_commit_result = ctx->sync_commit_result; + if (mail_index_map(ctx->index, MAIL_INDEX_SYNC_HANDLER_FILE) <= 0) + ret = -1; + index->sync_commit_result = NULL; + + /* The previously called expunged handlers will update cache's + record_count and deleted_record_count. That also has a side effect + of updating whether cache needs to be purged. */ + if (ret == 0 && mail_cache_need_purge(index->cache, &reason) && + !mail_cache_transactions_have_changes(index->cache)) { + if (mail_cache_purge(index->cache, + index->cache->need_purge_file_seq, + reason) < 0) { + /* can't really do anything if it fails */ + } + /* Make sure the newly committed cache record offsets are + updated to the current index. This is important if the + dovecot.index gets recreated below, because rotation of + dovecot.index.log also re-maps the index to make sure + everything is up-to-date. But if it wasn't, + mail_index_write() will just assert-crash because + log_file_head_offset changed. */ + if (mail_index_map(ctx->index, MAIL_INDEX_SYNC_HANDLER_FILE) <= 0) + ret = -1; + } + + /* Log rotation is allowed only if everything was synced. Note that + tail_offset might not equal head_offset here, because + mail_index_sync_update_mailbox_offset() doesn't always update + tail_offset to skip over other committed external transactions. + However, it's still safe to do the rotation because external + transactions don't require syncing. */ + want_rotate = ctx->fully_synced && + mail_transaction_log_want_rotate(index->log, &reason); + if (ret == 0 && + (want_rotate || mail_index_sync_want_index_write(index, &reason))) { + i_free(index->need_recreate); + index->index_min_write = FALSE; + mail_index_write(index, want_rotate, reason); + } + mail_index_sync_end(_ctx); + return ret; +} + +void mail_index_sync_rollback(struct mail_index_sync_ctx **ctx) +{ + if ((*ctx)->ext_trans != NULL) + mail_index_transaction_rollback(&(*ctx)->ext_trans); + mail_index_sync_end(ctx); +} + +void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec, + uint8_t *flags) +{ + i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); + + *flags = (*flags & ENUM_NEGATE(sync_rec->remove_flags)) | sync_rec->add_flags; +} + +bool mail_index_sync_keywords_apply(const struct mail_index_sync_rec *sync_rec, + ARRAY_TYPE(keyword_indexes) *keywords) +{ + const unsigned int *keyword_indexes; + unsigned int idx = sync_rec->keyword_idx; + unsigned int i, count; + + keyword_indexes = array_get(keywords, &count); + switch (sync_rec->type) { + case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + for (i = 0; i < count; i++) { + if (keyword_indexes[i] == idx) + return FALSE; + } + + array_push_back(keywords, &idx); + return TRUE; + case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + for (i = 0; i < count; i++) { + if (keyword_indexes[i] == idx) { + array_delete(keywords, i, 1); + return TRUE; + } + } + return FALSE; + default: + i_unreached(); + return FALSE; + } +} + +void mail_index_sync_set_corrupted(struct mail_index_sync_map_ctx *ctx, + const char *fmt, ...) +{ + va_list va; + uint32_t seq; + uoff_t offset; + char *reason, *reason_free = NULL; + + va_start(va, fmt); + reason = reason_free = i_strdup_vprintf(fmt, va); + va_end(va); + + ctx->errors = TRUE; + /* make sure we don't get to this same error again by updating the + dovecot.index */ + if (ctx->view->index->need_recreate == NULL) { + ctx->view->index->need_recreate = reason; + reason_free = NULL; + } + + mail_transaction_log_view_get_prev_pos(ctx->view->log_view, + &seq, &offset); + + if (seq < ctx->view->index->fsck_log_head_file_seq || + (seq == ctx->view->index->fsck_log_head_file_seq && + offset < ctx->view->index->fsck_log_head_file_offset)) { + /* be silent */ + } else { + mail_index_set_error(ctx->view->index, + "Log synchronization error at " + "seq=%u,offset=%"PRIuUOFF_T" for %s: %s", + seq, offset, ctx->view->index->filepath, + reason); + } + i_free(reason_free); +} diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c new file mode 100644 index 0000000..2aced67 --- /dev/null +++ b/src/lib-index/mail-index-transaction-export.c @@ -0,0 +1,533 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log-private.h" +#include "mail-index-transaction-private.h" + +struct mail_index_export_context { + struct mail_index_transaction *trans; + struct mail_transaction_log_append_ctx *append_ctx; +}; + +static void +log_append_buffer(struct mail_index_export_context *ctx, + const buffer_t *buf, enum mail_transaction_type type) +{ + mail_transaction_log_append_add(ctx->append_ctx, type, + buf->data, buf->used); +} + +static void log_append_flag_updates(struct mail_index_export_context *ctx, + struct mail_index_transaction *t) +{ + ARRAY(struct mail_transaction_flag_update) log_updates; + const struct mail_index_flag_update *updates; + struct mail_transaction_flag_update *log_update; + unsigned int i, count; + + updates = array_get(&t->updates, &count); + if (count == 0) + return; + + i_array_init(&log_updates, count); + + for (i = 0; i < count; i++) { + log_update = array_append_space(&log_updates); + log_update->uid1 = updates[i].uid1; + log_update->uid2 = updates[i].uid2; + log_update->add_flags = updates[i].add_flags & 0xff; + log_update->remove_flags = updates[i].remove_flags & 0xff; + if ((updates[i].add_flags & MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ) != 0) + log_update->modseq_inc_flag = 1; + } + log_append_buffer(ctx, log_updates.arr.buffer, + MAIL_TRANSACTION_FLAG_UPDATE); + array_free(&log_updates); +} + +static const buffer_t * +log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend) +{ + buffer_t *buf; + const unsigned char *data, *mask; + struct mail_transaction_header_update u; + uint16_t offset; + int state = 0; + + i_zero(&u); + + data = prepend ? t->pre_hdr_change : t->post_hdr_change; + mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask; + + buf = t_buffer_create(256); + for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) { + if (offset < sizeof(t->pre_hdr_change) && mask[offset] != 0) { + if (state == 0) { + u.offset = offset; + state++; + } + } else { + if (state > 0) { + u.size = offset - u.offset; + buffer_append(buf, &u, sizeof(u)); + buffer_append(buf, data + u.offset, u.size); + state = 0; + } + } + } + return buf; +} + +static unsigned int +ext_hdr_update_get_size(const struct mail_index_transaction_ext_hdr_update *hu) +{ + unsigned int i; + + for (i = hu->alloc_size; i > 0; i--) { + if (hu->mask[i-1] != 0) + return i; + } + return 0; +} + +static void log_append_ext_intro(struct mail_index_export_context *ctx, + uint32_t ext_id, uint32_t reset_id, + unsigned int *hdr_size_r) +{ + struct mail_index_transaction *t = ctx->trans; + const struct mail_index_registered_ext *rext; + const struct mail_index_ext *ext; + struct mail_transaction_ext_intro *intro, *resizes; + buffer_t *buf; + uint32_t idx; + unsigned int count; + + i_assert(ext_id != (uint32_t)-1); + + if (t->reset || + !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) { + /* new extension */ + idx = (uint32_t)-1; + } + + rext = array_idx(&t->view->index->extensions, ext_id); + if (!array_is_created(&t->ext_resizes)) { + resizes = NULL; + count = 0; + } else { + resizes = array_get_modifiable(&t->ext_resizes, &count); + } + + buf = t_buffer_create(128); + if (ext_id < count && resizes[ext_id].name_size != 0) { + /* we're resizing the extension. use the resize struct. */ + intro = &resizes[ext_id]; + + if (idx != (uint32_t)-1) { + intro->ext_id = idx; + intro->name_size = 0; + } else { + intro->ext_id = (uint32_t)-1; + intro->name_size = strlen(rext->name); + } + buffer_append(buf, intro, sizeof(*intro)); + } else { + /* generate a new intro structure */ + intro = buffer_append_space_unsafe(buf, sizeof(*intro)); + intro->ext_id = idx; + intro->record_size = rext->record_size; + intro->record_align = rext->record_align; + if (idx == (uint32_t)-1) { + intro->hdr_size = rext->hdr_size; + intro->name_size = strlen(rext->name); + } else { + ext = array_idx(&t->view->index->map->extensions, idx); + intro->hdr_size = ext->hdr_size; + intro->name_size = 0; + } + intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK; + + /* handle increasing header size automatically */ + if (array_is_created(&t->ext_hdr_updates) && + ext_id < array_count(&t->ext_hdr_updates)) { + const struct mail_index_transaction_ext_hdr_update *hu; + unsigned int hdr_update_size; + + hu = array_idx(&t->ext_hdr_updates, ext_id); + hdr_update_size = ext_hdr_update_get_size(hu); + if (intro->hdr_size < hdr_update_size) + intro->hdr_size = hdr_update_size; + } + } + i_assert(intro->record_size != 0 || intro->hdr_size != 0); + if (reset_id != 0) { + /* we're going to reset this extension in this transaction */ + intro->reset_id = reset_id; + } else if (idx != (uint32_t)-1) { + /* use the existing reset_id */ + const struct mail_index_ext *map_ext = + array_idx(&t->view->index->map->extensions, idx); + intro->reset_id = map_ext->reset_id; + } else { + /* new extension, reset_id defaults to 0 */ + } + buffer_append(buf, rext->name, intro->name_size); + if ((buf->used % 4) != 0) + buffer_append_zero(buf, 4 - (buf->used % 4)); + + if (ctx->append_ctx->new_highest_modseq == 0 && + strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) { + /* modseq tracking started */ + ctx->append_ctx->new_highest_modseq = 1; + } + + log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO); + *hdr_size_r = intro->hdr_size; +} + +static void +log_append_ext_hdr_update(struct mail_index_export_context *ctx, + const struct mail_index_transaction_ext_hdr_update *hdr, + unsigned int ext_hdr_size) +{ + buffer_t *buf; + const unsigned char *data, *mask; + struct mail_transaction_ext_hdr_update u; + struct mail_transaction_ext_hdr_update32 u32; + size_t offset; + bool started = FALSE, use_32 = hdr->alloc_size >= 65536; + + i_zero(&u); + i_zero(&u32); + + data = hdr->data; + mask = hdr->mask; + + buf = buffer_create_dynamic(default_pool, 256); + for (offset = 0; offset <= hdr->alloc_size; offset++) { + if (offset < hdr->alloc_size && mask[offset] != 0) { + if (!started) { + u32.offset = offset; + started = TRUE; + } + } else { + if (started) { + u32.size = offset - u32.offset; + if (use_32) + buffer_append(buf, &u32, sizeof(u32)); + else { + u.offset = u32.offset; + u.size = u32.size; + buffer_append(buf, &u, sizeof(u)); + } + i_assert(u32.offset + u32.size <= ext_hdr_size); + buffer_append(buf, data + u32.offset, u32.size); + started = FALSE; + } + } + } + if (buf->used % 4 != 0) + buffer_append_zero(buf, 4 - buf->used % 4); + log_append_buffer(ctx, buf, use_32 ? MAIL_TRANSACTION_EXT_HDR_UPDATE32 : + MAIL_TRANSACTION_EXT_HDR_UPDATE); + buffer_free(&buf); +} + +static void +mail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx) +{ + struct mail_index_transaction *t = ctx->trans; + const struct mail_transaction_ext_intro *resize; + const struct mail_index_transaction_ext_hdr_update *hdrs; + struct mail_transaction_ext_reset ext_reset; + unsigned int resize_count, ext_count = 0; + unsigned int hdrs_count, reset_id_count, reset_count, hdr_size; + uint32_t ext_id, reset_id; + const struct mail_transaction_ext_reset *reset; + const uint32_t *reset_ids; + buffer_t reset_buf; + + if (!array_is_created(&t->ext_resizes)) { + resize = NULL; + resize_count = 0; + } else { + resize = array_get(&t->ext_resizes, &resize_count); + if (ext_count < resize_count) + ext_count = resize_count; + } + + if (!array_is_created(&t->ext_reset_ids)) { + reset_ids = NULL; + reset_id_count = 0; + } else { + reset_ids = array_get(&t->ext_reset_ids, &reset_id_count); + } + + if (!array_is_created(&t->ext_resets)) { + reset = NULL; + reset_count = 0; + } else { + reset = array_get(&t->ext_resets, &reset_count); + if (ext_count < reset_count) + ext_count = reset_count; + } + + if (!array_is_created(&t->ext_hdr_updates)) { + hdrs = NULL; + hdrs_count = 0; + } else { + hdrs = array_get(&t->ext_hdr_updates, &hdrs_count); + if (ext_count < hdrs_count) + ext_count = hdrs_count; + } + + i_zero(&ext_reset); + buffer_create_from_data(&reset_buf, &ext_reset, sizeof(ext_reset)); + buffer_set_used_size(&reset_buf, sizeof(ext_reset)); + + for (ext_id = 0; ext_id < ext_count; ext_id++) { + if (ext_id < reset_count) + ext_reset = reset[ext_id]; + else + ext_reset.new_reset_id = 0; + if ((ext_id < resize_count && resize[ext_id].name_size > 0) || + ext_reset.new_reset_id != 0 || + (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) { + if (ext_reset.new_reset_id != 0) { + /* we're going to reset this extension + immediately after the intro */ + reset_id = 0; + } else { + reset_id = ext_id < reset_id_count ? + reset_ids[ext_id] : 0; + } + log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size); + } else { + hdr_size = 0; + } + if (ext_reset.new_reset_id != 0) { + i_assert(ext_id < reset_id_count && + ext_reset.new_reset_id == reset_ids[ext_id]); + log_append_buffer(ctx, &reset_buf, + MAIL_TRANSACTION_EXT_RESET); + } + if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) { + T_BEGIN { + log_append_ext_hdr_update(ctx, &hdrs[ext_id], + hdr_size); + } T_END; + } + } +} + +static void log_append_ext_recs(struct mail_index_export_context *ctx, + const ARRAY_TYPE(seq_array_array) *arr, + enum mail_transaction_type type) +{ + struct mail_index_transaction *t = ctx->trans; + const ARRAY_TYPE(seq_array) *updates; + const uint32_t *reset_ids; + unsigned int ext_id, count, reset_id_count, hdr_size; + uint32_t reset_id; + + if (!array_is_created(&t->ext_reset_ids)) { + reset_ids = NULL; + reset_id_count = 0; + } else { + reset_ids = array_get_modifiable(&t->ext_reset_ids, + &reset_id_count); + } + + updates = array_get(arr, &count); + for (ext_id = 0; ext_id < count; ext_id++) { + if (!array_is_created(&updates[ext_id])) + continue; + + reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0; + log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size); + + log_append_buffer(ctx, updates[ext_id].arr.buffer, type); + } +} + +static void +log_append_keyword_update(struct mail_index_export_context *ctx, + buffer_t *tmp_buf, enum modify_type modify_type, + const char *keyword, const buffer_t *uid_buffer) +{ + struct mail_transaction_keyword_update kt_hdr; + + i_assert(uid_buffer->used > 0); + + i_zero(&kt_hdr); + kt_hdr.modify_type = modify_type; + kt_hdr.name_size = strlen(keyword); + + buffer_set_used_size(tmp_buf, 0); + buffer_append(tmp_buf, &kt_hdr, sizeof(kt_hdr)); + buffer_append(tmp_buf, keyword, kt_hdr.name_size); + if ((tmp_buf->used % 4) != 0) + buffer_append_zero(tmp_buf, 4 - (tmp_buf->used % 4)); + buffer_append(tmp_buf, uid_buffer->data, uid_buffer->used); + + log_append_buffer(ctx, tmp_buf, MAIL_TRANSACTION_KEYWORD_UPDATE); +} + +static bool +log_append_keyword_updates(struct mail_index_export_context *ctx) +{ + const struct mail_index_transaction_keyword_update *updates; + const char *const *keywords; + buffer_t *tmp_buf; + unsigned int i, count, keywords_count; + bool changed = FALSE; + + tmp_buf = t_buffer_create(64); + + keywords = array_get_modifiable(&ctx->trans->view->index->keywords, + &keywords_count); + updates = array_get_modifiable(&ctx->trans->keyword_updates, &count); + i_assert(count <= keywords_count); + + for (i = 0; i < count; i++) { + if (array_is_created(&updates[i].add_seq) && + array_count(&updates[i].add_seq) > 0) { + changed = TRUE; + log_append_keyword_update(ctx, tmp_buf, + MODIFY_ADD, keywords[i], + updates[i].add_seq.arr.buffer); + } + if (array_is_created(&updates[i].remove_seq) && + array_count(&updates[i].remove_seq) > 0) { + changed = TRUE; + log_append_keyword_update(ctx, tmp_buf, + MODIFY_REMOVE, keywords[i], + updates[i].remove_seq.arr.buffer); + } + } + return changed; +} + +void mail_index_transaction_export(struct mail_index_transaction *t, + struct mail_transaction_log_append_ctx *append_ctx, + enum mail_index_transaction_change *changes_r) +{ + static uint8_t null4[4] = { 0, 0, 0, 0 }; + enum mail_index_fsync_mask change_mask = 0; + struct mail_index_export_context ctx; + + *changes_r = 0; + + i_zero(&ctx); + ctx.trans = t; + ctx.append_ctx = append_ctx; + + if (t->index_undeleted) { + i_assert(!t->index_deleted); + mail_transaction_log_append_add(ctx.append_ctx, + MAIL_TRANSACTION_INDEX_UNDELETED, &null4, 4); + } + + /* send all extension introductions and resizes before appends + to avoid resize overhead as much as possible */ + mail_transaction_log_append_ext_intros(&ctx); + + if (t->pre_hdr_changed) { + log_append_buffer(&ctx, log_get_hdr_update_buffer(t, TRUE), + MAIL_TRANSACTION_HEADER_UPDATE); + } + + if (append_ctx->output->used > 0) + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + + if (t->attribute_updates != NULL) { + buffer_append_c(t->attribute_updates, '\0'); + /* need to have 32bit alignment */ + if (t->attribute_updates->used % 4 != 0) { + buffer_append_zero(t->attribute_updates, + 4 - t->attribute_updates->used % 4); + } + /* append the timestamp and value lengths */ + buffer_append(t->attribute_updates, + t->attribute_updates_suffix->data, + t->attribute_updates_suffix->used); + i_assert(t->attribute_updates->used % 4 == 0); + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_ATTRIBUTE; + log_append_buffer(&ctx, t->attribute_updates, + MAIL_TRANSACTION_ATTRIBUTE_UPDATE); + } + if (array_is_created(&t->appends)) { + change_mask |= MAIL_INDEX_FSYNC_MASK_APPENDS; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_APPEND; + log_append_buffer(&ctx, t->appends.arr.buffer, + MAIL_TRANSACTION_APPEND); + } + + if (array_is_created(&t->updates)) { + change_mask |= MAIL_INDEX_FSYNC_MASK_FLAGS; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_FLAGS; + log_append_flag_updates(&ctx, t); + } + + if (array_is_created(&t->ext_rec_updates)) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + log_append_ext_recs(&ctx, &t->ext_rec_updates, + MAIL_TRANSACTION_EXT_REC_UPDATE); + } + if (array_is_created(&t->ext_rec_atomics)) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + log_append_ext_recs(&ctx, &t->ext_rec_atomics, + MAIL_TRANSACTION_EXT_ATOMIC_INC); + } + + if (array_is_created(&t->keyword_updates)) { + if (log_append_keyword_updates(&ctx)) { + change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_KEYWORDS; + } + } + /* keep modseq updates almost last */ + if (array_is_created(&t->modseq_updates)) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_MODSEQ; + log_append_buffer(&ctx, t->modseq_updates.arr.buffer, + MAIL_TRANSACTION_MODSEQ_UPDATE); + } + + if (array_is_created(&t->expunges)) { + /* non-external expunges are only requests, ignore them when + checking fsync_mask */ + if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0) { + change_mask |= MAIL_INDEX_FSYNC_MASK_EXPUNGES; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_EXPUNGE; + } else { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + } + log_append_buffer(&ctx, t->expunges.arr.buffer, + MAIL_TRANSACTION_EXPUNGE_GUID); + } + + if (t->post_hdr_changed) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE), + MAIL_TRANSACTION_HEADER_UPDATE); + } + + if (t->index_deleted) { + i_assert(!t->index_undeleted); + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + mail_transaction_log_append_add(ctx.append_ctx, + MAIL_TRANSACTION_INDEX_DELETED, + &null4, 4); + } + + i_assert((append_ctx->output->used > 0) == (*changes_r != 0)); + + append_ctx->index_sync_transaction = t->sync_transaction; + append_ctx->tail_offset_changed = t->tail_offset_changed; + append_ctx->want_fsync = + (t->view->index->set.fsync_mask & change_mask) != 0 || + (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0; +} diff --git a/src/lib-index/mail-index-transaction-finish.c b/src/lib-index/mail-index-transaction-finish.c new file mode 100644 index 0000000..360e597 --- /dev/null +++ b/src/lib-index/mail-index-transaction-finish.c @@ -0,0 +1,350 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-index-transaction-private.h" + +int mail_transaction_expunge_guid_cmp(const struct mail_transaction_expunge_guid *e1, + const struct mail_transaction_expunge_guid *e2) +{ + if (e1->uid < e2->uid) + return -1; + else if (e1->uid > e2->uid) + return 1; + else + return 0; +} + +void mail_index_transaction_sort_expunges(struct mail_index_transaction *t) +{ + if (!t->expunges_nonsorted) + return; + + array_sort(&t->expunges, mail_transaction_expunge_guid_cmp); + t->expunges_nonsorted = FALSE; +} + +static void +ext_reset_update_atomic(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t expected_reset_id) +{ + const struct mail_index_ext *map_ext; + struct mail_transaction_ext_reset *reset; + uint32_t idx, reset_id; + + if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) { + /* new extension */ + reset_id = 1; + } else { + map_ext = array_idx(&t->view->index->map->extensions, idx); + reset_id = map_ext->reset_id + 1; + } + if (reset_id != expected_reset_id) { + /* ignore this extension update */ + mail_index_ext_set_reset_id(t, ext_id, 0); + return; + } + + if (reset_id == 0) + reset_id++; + + array_idx_set(&t->ext_reset_ids, ext_id, &reset_id); + + /* reseting existing data is optional */ + if (array_is_created(&t->ext_resets)) { + reset = array_idx_modifiable(&t->ext_resets, ext_id); + if (reset->new_reset_id == (uint32_t)-1) + reset->new_reset_id = reset_id; + } +} + +static void +transaction_update_atomic_reset_ids(struct mail_index_transaction *t) +{ + const uint32_t *expected_reset_ids; + unsigned int ext_id, count; + + if (!array_is_created(&t->ext_reset_atomic)) + return; + + expected_reset_ids = array_get(&t->ext_reset_atomic, &count); + for (ext_id = 0; ext_id < count; ext_id++) { + if (expected_reset_ids[ext_id] != 0) { + ext_reset_update_atomic(t, ext_id, + expected_reset_ids[ext_id]); + } + } +} + +static unsigned int +mail_transaction_drop_range(struct mail_index_transaction *t, + struct mail_index_flag_update update, + unsigned int update_idx, + ARRAY_TYPE(seq_range) *keeps) +{ + const struct seq_range *keep_range; + unsigned int i, keep_count; + + keep_range = array_get(keeps, &keep_count); + if (keep_count == 1 && + update.uid1 == keep_range[0].seq1 && + update.uid2 == keep_range[0].seq2) { + /* everything is kept */ + return update_idx + 1; + } + + array_delete(&t->updates, update_idx, 1); + + /* add back all the updates we want to keep */ + for (i = 0; i < keep_count; i++, update_idx++) { + update.uid1 = keep_range[i].seq1; + update.uid2 = keep_range[i].seq2; + array_insert(&t->updates, update_idx, &update, 1); + } + return update_idx; +} + +static void +mail_index_transaction_finish_flag_updates(struct mail_index_transaction *t) +{ + const struct mail_index_flag_update *updates, *u; + const struct mail_index_record *rec; + unsigned int i, count; + ARRAY_TYPE(seq_range) keeps; + uint32_t seq; + + if (!t->drop_unnecessary_flag_updates || !array_is_created(&t->updates)) + return; + + t_array_init(&keeps, 64); + updates = array_get(&t->updates, &count); + for (i = 0; i < count; ) { + /* first get the list of changes to drop */ + u = &updates[i]; + array_clear(&keeps); + for (seq = u->uid1; seq <= u->uid2; seq++) { + rec = mail_index_lookup(t->view, seq); + if ((rec->flags & u->add_flags) != u->add_flags || + (rec->flags & u->remove_flags) != 0) { + /* keep this change */ + seq_range_array_add(&keeps, seq); + } + } + i = mail_transaction_drop_range(t, updates[i], i, &keeps); + updates = array_get(&t->updates, &count); + } + + if (array_count(&t->updates) == 0) + array_free(&t->updates); +} + +static void +mail_index_transaction_check_conflicts(struct mail_index_transaction *t) +{ + uint32_t seq; + bool ret1, ret2; + + i_assert(t->max_modseq != 0); + i_assert(t->conflict_seqs != NULL); + + if (t->max_modseq == mail_index_modseq_get_highest(t->view)) { + /* no conflicts possible */ + return; + } + if (t->min_flagupdate_seq == 0) { + /* no flag updates */ + return; + } + + for (seq = t->min_flagupdate_seq; seq <= t->max_flagupdate_seq; seq++) { + if (mail_index_modseq_lookup(t->view, seq) > t->max_modseq) { + ret1 = mail_index_cancel_flag_updates(t, seq); + ret2 = mail_index_cancel_keyword_updates(t, seq); + if (ret1 || ret2) { + seq_range_array_add_with_init(t->conflict_seqs, + 16, seq); + } + } + } + mail_index_transaction_set_log_updates(t); +} + +static uint32_t +mail_index_transaction_get_uid(struct mail_index_transaction *t, uint32_t seq) +{ + const struct mail_index_record *rec; + + i_assert(seq > 0); + + if (seq >= t->first_new_seq) + rec = mail_index_transaction_lookup(t, seq); + else { + i_assert(seq <= t->view->map->hdr.messages_count); + rec = MAIL_INDEX_REC_AT_SEQ(t->view->map, seq); + } + i_assert(rec->uid != 0); + return rec->uid; +} + +static void +mail_index_convert_to_uids(struct mail_index_transaction *t, + ARRAY_TYPE(seq_array) *array) +{ + uint32_t *seq; + unsigned int i, count; + + if (!array_is_created(array)) + return; + + count = array_count(array); + for (i = 0; i < count; i++) { + seq = array_idx_modifiable(array, i); + *seq = mail_index_transaction_get_uid(t, *seq); + } +} + +static uint32_t +get_nonexpunged_uid2(struct mail_index_transaction *t, + uint32_t uid1, uint32_t seq1) +{ + seq1++; + + while (mail_index_transaction_get_uid(t, seq1) == uid1 + 1) { + seq1++; + uid1++; + } + return uid1; +} + +void mail_index_transaction_seq_range_to_uid(struct mail_index_transaction *t, + ARRAY_TYPE(seq_range) *array) +{ + struct seq_range *range, *new_range; + unsigned int i, count; + uint32_t uid1, uid2, prev_uid = 0; + + if (!array_is_created(array)) + return; + + count = array_count(array); + for (i = 0; i < count; i++) { + range = array_idx_modifiable(array, i); + + uid1 = mail_index_transaction_get_uid(t, range->seq1); + uid2 = mail_index_transaction_get_uid(t, range->seq2); + i_assert(uid1 > prev_uid); + if (uid2 - uid1 == range->seq2 - range->seq1) { + /* simple conversion */ + range->seq1 = uid1; + range->seq2 = uid2; + prev_uid = uid2; + } else { + /* remove expunged UIDs */ + new_range = array_insert_space(array, i); + range = array_idx_modifiable(array, i + 1); + count++; + + memcpy(new_range, range, array->arr.element_size); + new_range->seq1 = uid1; + new_range->seq2 = get_nonexpunged_uid2(t, uid1, + range->seq1); + i_assert(new_range->seq2 < uid2); + + /* continue the range without the inserted seqs */ + range->seq1 += new_range->seq2 - new_range->seq1 + 1; + prev_uid = new_range->seq2; + } + } +} + +static void keyword_updates_convert_to_uids(struct mail_index_transaction *t) +{ + struct mail_index_transaction_keyword_update *update; + + if (!array_is_created(&t->keyword_updates)) + return; + + array_foreach_modifiable(&t->keyword_updates, update) { + mail_index_transaction_seq_range_to_uid(t, &update->add_seq); + mail_index_transaction_seq_range_to_uid(t, &update->remove_seq); + } +} + +static void expunges_convert_to_uids(struct mail_index_transaction *t) +{ + struct mail_transaction_expunge_guid *expunges; + unsigned int src, dest, count; + + if (!array_is_created(&t->expunges)) + return; + + mail_index_transaction_sort_expunges(t); + + expunges = array_get_modifiable(&t->expunges, &count); + if (count == 0) + return; + + /* convert uids and drop duplicates */ + expunges[0].uid = mail_index_transaction_get_uid(t, expunges[0].uid); + for (src = dest = 1; src < count; src++) { + expunges[dest].uid = + mail_index_transaction_get_uid(t, expunges[src].uid); + if (expunges[dest-1].uid != expunges[dest].uid) { + if (dest != src) { + memcpy(expunges[dest].guid_128, expunges[src].guid_128, + sizeof(expunges[dest].guid_128)); + } + dest++; + } + } + array_delete(&t->expunges, dest, count-dest); +} + +static void +mail_index_transaction_convert_to_uids(struct mail_index_transaction *t) +{ + ARRAY_TYPE(seq_array) *update; + + if (array_is_created(&t->ext_rec_updates)) { + array_foreach_modifiable(&t->ext_rec_updates, update) + mail_index_convert_to_uids(t, update); + } + if (array_is_created(&t->ext_rec_atomics)) { + array_foreach_modifiable(&t->ext_rec_atomics, update) + mail_index_convert_to_uids(t, update); + } + + keyword_updates_convert_to_uids(t); + expunges_convert_to_uids(t); + mail_index_convert_to_uids(t, (void *)&t->modseq_updates); + mail_index_transaction_seq_range_to_uid(t, (void *)&t->updates); +} + +void mail_index_transaction_finish_so_far(struct mail_index_transaction *t) +{ + if (array_is_created(&t->appends)) + mail_index_transaction_sort_appends(t); + mail_index_transaction_finish_flag_updates(t); + if (t->max_modseq != 0) + mail_index_transaction_check_conflicts(t); +} + +void mail_index_transaction_finish(struct mail_index_transaction *t) +{ + mail_index_transaction_finish_so_far(t); + + if (array_is_created(&t->appends)) + mail_index_update_day_headers(t, ioloop_time); + if (array_is_created(&t->ext_reset_atomic)) + transaction_update_atomic_reset_ids(t); + /* finally convert all sequences to UIDs before we write them, + but after we've checked and removed conflicts */ + mail_index_transaction_convert_to_uids(t); + + /* and kind of ugly way to update highest modseq */ + if (t->min_highest_modseq != 0) + mail_index_update_modseq(t, 0, t->min_highest_modseq); +} diff --git a/src/lib-index/mail-index-transaction-private.h b/src/lib-index/mail-index-transaction-private.h new file mode 100644 index 0000000..eb438b2 --- /dev/null +++ b/src/lib-index/mail-index-transaction-private.h @@ -0,0 +1,165 @@ +#ifndef MAIL_INDEX_TRANSACTION_PRIVATE_H +#define MAIL_INDEX_TRANSACTION_PRIVATE_H + +#include "seq-range-array.h" +#include "mail-transaction-log.h" + +ARRAY_DEFINE_TYPE(seq_array_array, ARRAY_TYPE(seq_array)); + +struct mail_index_transaction_keyword_update { + ARRAY_TYPE(seq_range) add_seq; + ARRAY_TYPE(seq_range) remove_seq; +}; + +struct mail_index_transaction_ext_hdr_update { + size_t alloc_size; + /* mask is in bytes, not bits */ + unsigned char *mask; + unsigned char *data; +}; + +struct mail_index_transaction_vfuncs { + void (*reset)(struct mail_index_transaction *t); + int (*commit)(struct mail_index_transaction *t, + struct mail_index_transaction_commit_result *result_r); + void (*rollback)(struct mail_index_transaction *t); +}; + +union mail_index_transaction_module_context { + struct mail_index_transaction_vfuncs super; + struct mail_index_module_register *reg; +}; + +struct mail_index_flag_update { + uint32_t uid1, uid2; + uint16_t add_flags; + uint16_t remove_flags; +}; + +struct mail_index_transaction { + struct mail_index_transaction *prev, *next; + int refcount; + + enum mail_index_transaction_flags flags; + struct mail_index_transaction_vfuncs v, *vlast; + struct mail_index_view *view; + struct mail_index_view *latest_view; + + /* NOTE: If you add anything new, remember to update + mail_index_transaction_reset_v() to reset it. */ + ARRAY(struct mail_index_record) appends; + uint32_t first_new_seq, last_new_seq; + uint32_t highest_append_uid; + /* lowest/highest sequence that updates flags/keywords */ + uint32_t min_flagupdate_seq, max_flagupdate_seq; + + ARRAY(struct mail_transaction_modseq_update) modseq_updates; + ARRAY(struct mail_transaction_expunge_guid) expunges; + ARRAY(struct mail_index_flag_update) updates; + size_t last_update_idx; + + unsigned char pre_hdr_change[sizeof(struct mail_index_header)]; + unsigned char pre_hdr_mask[sizeof(struct mail_index_header)]; + unsigned char post_hdr_change[sizeof(struct mail_index_header)]; + unsigned char post_hdr_mask[sizeof(struct mail_index_header)]; + + ARRAY(struct mail_index_transaction_ext_hdr_update) ext_hdr_updates; + ARRAY_TYPE(seq_array_array) ext_rec_updates; + ARRAY_TYPE(seq_array_array) ext_rec_atomics; + ARRAY(struct mail_transaction_ext_intro) ext_resizes; + ARRAY(struct mail_transaction_ext_reset) ext_resets; + ARRAY(uint32_t) ext_reset_ids; + ARRAY(uint32_t) ext_reset_atomic; + + ARRAY(struct mail_index_transaction_keyword_update) keyword_updates; + buffer_t *attribute_updates; /* [+-][ps]key\0.. */ + buffer_t *attribute_updates_suffix; /* <timestamp>[<value len>].. */ + + uint64_t min_highest_modseq; + uint64_t max_modseq; + ARRAY_TYPE(seq_range) *conflict_seqs; + + /* Module-specific contexts. */ + ARRAY(union mail_index_transaction_module_context *) module_contexts; + + bool no_appends:1; + + bool sync_transaction:1; + bool appends_nonsorted:1; + bool expunges_nonsorted:1; + bool drop_unnecessary_flag_updates:1; + bool pre_hdr_changed:1; + bool post_hdr_changed:1; + bool reset:1; + bool index_deleted:1; + bool index_undeleted:1; + bool commit_deleted_index:1; + bool tail_offset_changed:1; + /* non-extension updates. flag updates don't change this because + they may be added and removed, so be sure to check that the updates + array is non-empty also. */ + bool log_updates:1; + /* extension updates */ + bool log_ext_updates:1; +}; + +#define MAIL_INDEX_TRANSACTION_HAS_CHANGES(t) \ + ((t)->log_updates || (t)->log_ext_updates || \ + (array_is_created(&(t)->updates) && array_count(&(t)->updates) > 0) || \ + (t)->index_deleted || (t)->index_undeleted) + +typedef void hook_mail_index_transaction_created_t(struct mail_index_transaction *t); + +void mail_index_transaction_hook_register(hook_mail_index_transaction_created_t *hook); +void mail_index_transaction_hook_unregister(hook_mail_index_transaction_created_t *hook); + +struct mail_index_record * +mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq); + +void mail_index_transaction_ref(struct mail_index_transaction *t); +void mail_index_transaction_unref(struct mail_index_transaction **t); +void mail_index_transaction_reset_v(struct mail_index_transaction *t); + +void mail_index_transaction_sort_appends(struct mail_index_transaction *t); +void mail_index_transaction_sort_expunges(struct mail_index_transaction *t); +uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t); +void mail_index_transaction_set_log_updates(struct mail_index_transaction *t); +void mail_index_update_day_headers(struct mail_index_transaction *t, time_t day_stamp); + +unsigned int +mail_index_transaction_get_flag_update_pos(struct mail_index_transaction *t, + unsigned int left_idx, + unsigned int right_idx, + uint32_t seq); +void mail_index_transaction_lookup_latest_keywords(struct mail_index_transaction *t, + uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keywords); + +bool mail_index_cancel_flag_updates(struct mail_index_transaction *t, + uint32_t seq); +bool mail_index_cancel_keyword_updates(struct mail_index_transaction *t, + uint32_t seq); + +/* As input the array's each element starts with struct seq_range where + uid1..uid2 are actually sequences within the transaction view. This function + changes the sequences into UIDs. If the transaction has any appends, they + must have already been assigned UIDs. */ +void mail_index_transaction_seq_range_to_uid(struct mail_index_transaction *t, + ARRAY_TYPE(seq_range) *array); +void mail_index_transaction_finish_so_far(struct mail_index_transaction *t); +void mail_index_transaction_finish(struct mail_index_transaction *t); +void mail_index_transaction_export(struct mail_index_transaction *t, + struct mail_transaction_log_append_ctx *append_ctx, + enum mail_index_transaction_change *changes_r); +int mail_transaction_expunge_guid_cmp(const struct mail_transaction_expunge_guid *e1, + const struct mail_transaction_expunge_guid *e2); +unsigned int +mail_index_transaction_get_flag_update_pos(struct mail_index_transaction *t, + unsigned int left_idx, + unsigned int right_idx, + uint32_t seq); + +void mail_index_ext_using_reset_id(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t reset_id); + +#endif diff --git a/src/lib-index/mail-index-transaction-sort-appends.c b/src/lib-index/mail-index-transaction-sort-appends.c new file mode 100644 index 0000000..c2a29d3 --- /dev/null +++ b/src/lib-index/mail-index-transaction-sort-appends.c @@ -0,0 +1,184 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "seq-range-array.h" +#include "mail-index-private.h" +#include "mail-index-transaction-private.h" + + +struct uid_map { + uint32_t idx; + uint32_t uid; +}; + +static int uid_map_cmp(const void *p1, const void *p2) +{ + const struct uid_map *m1 = p1, *m2 = p2; + + return m1->uid < m2->uid ? -1 : + (m1->uid > m2->uid ? 1 : 0); +} + +static void +mail_index_transaction_sort_appends_ext(ARRAY_TYPE(seq_array_array) *updates, + uint32_t first_new_seq, + const uint32_t *old_to_newseq_map) +{ + ARRAY_TYPE(seq_array) *ext_rec_arrays; + ARRAY_TYPE(seq_array) *old_array; + ARRAY_TYPE(seq_array) new_array; + unsigned int ext_count; + const uint32_t *ext_rec; + uint32_t seq; + unsigned int i, j, count; + + if (!array_is_created(updates)) + return; + + ext_rec_arrays = array_get_modifiable(updates, &count); + for (j = 0; j < count; j++) { + old_array = &ext_rec_arrays[j]; + if (!array_is_created(old_array)) + continue; + + ext_count = array_count(old_array); + array_create(&new_array, default_pool, + old_array->arr.element_size, ext_count); + for (i = 0; i < ext_count; i++) { + ext_rec = array_idx(old_array, i); + + seq = *ext_rec < first_new_seq ? *ext_rec : + old_to_newseq_map[*ext_rec - first_new_seq]; + (void)mail_index_seq_array_add(&new_array, seq, ext_rec+1, + old_array->arr.element_size - + sizeof(*ext_rec), NULL); + } + array_free(old_array); + ext_rec_arrays[j] = new_array; + } +} + +static void +sort_appends_seq_range(ARRAY_TYPE(seq_range) *array, uint32_t first_new_seq, + const uint32_t *old_to_newseq_map) +{ + struct seq_range *range, temp_range; + ARRAY_TYPE(seq_range) old_seqs; + uint32_t idx, idx1, idx2; + unsigned int i, count; + + range = array_get_modifiable(array, &count); + for (i = 0; i < count; i++) { + if (range[i].seq2 >= first_new_seq) + break; + } + if (i == count) { + /* nothing to do */ + return; + } + + i_array_init(&old_seqs, count - i); + if (range[i].seq1 < first_new_seq) { + temp_range.seq1 = first_new_seq; + temp_range.seq2 = range[i].seq2; + array_push_back(&old_seqs, &temp_range); + range[i].seq2 = first_new_seq - 1; + i++; + } + array_append(&old_seqs, &range[i], count - i); + array_delete(array, i, count - i); + + range = array_get_modifiable(&old_seqs, &count); + for (i = 0; i < count; i++) { + idx1 = range[i].seq1 - first_new_seq; + idx2 = range[i].seq2 - first_new_seq; + for (idx = idx1; idx <= idx2; idx++) + seq_range_array_add(array, old_to_newseq_map[idx]); + } + array_free(&old_seqs); +} + +static void +mail_index_transaction_sort_appends_keywords(struct mail_index_transaction *t, + const uint32_t *old_to_newseq_map) +{ + struct mail_index_transaction_keyword_update *update; + + if (!array_is_created(&t->keyword_updates)) + return; + + array_foreach_modifiable(&t->keyword_updates, update) { + if (array_is_created(&update->add_seq)) { + sort_appends_seq_range(&update->add_seq, + t->first_new_seq, + old_to_newseq_map); + } + if (array_is_created(&update->remove_seq)) { + sort_appends_seq_range(&update->remove_seq, + t->first_new_seq, + old_to_newseq_map); + } + } +} + +void mail_index_transaction_sort_appends(struct mail_index_transaction *t) +{ + struct mail_index_record *recs, *sorted_recs; + struct uid_map *new_uid_map; + uint32_t *old_to_newseq_map; + unsigned int i, count; + + if (!array_is_created(&t->appends)) + return; + recs = array_get_modifiable(&t->appends, &count); + i_assert(count > 0); + + if (!t->appends_nonsorted) { + i_assert(recs[0].uid != 0); +#ifdef DEBUG + for (i = 1; i < count; i++) + i_assert(recs[i-1].uid < recs[i].uid); +#endif + return; + } + + /* first make a copy of the UIDs and map them to sequences */ + new_uid_map = i_new(struct uid_map, count); + for (i = 0; i < count; i++) { + i_assert(recs[i].uid != 0); + new_uid_map[i].idx = i; + new_uid_map[i].uid = recs[i].uid; + } + + /* now sort the UID map */ + qsort(new_uid_map, count, sizeof(*new_uid_map), uid_map_cmp); + + /* sort mail records */ + sorted_recs = i_new(struct mail_index_record, count); + sorted_recs[0] = recs[new_uid_map[0].idx]; + for (i = 1; i < count; i++) { + sorted_recs[i] = recs[new_uid_map[i].idx]; + if (sorted_recs[i].uid == sorted_recs[i-1].uid) + i_panic("Duplicate UIDs added in transaction"); + } + buffer_write(t->appends.arr.buffer, 0, sorted_recs, + sizeof(*sorted_recs) * count); + i_free(sorted_recs); + + old_to_newseq_map = i_new(uint32_t, count); + for (i = 0; i < count; i++) + old_to_newseq_map[new_uid_map[i].idx] = i + t->first_new_seq; + i_free(new_uid_map); + + mail_index_transaction_sort_appends_ext(&t->ext_rec_updates, + t->first_new_seq, + old_to_newseq_map); + mail_index_transaction_sort_appends_ext(&t->ext_rec_atomics, + t->first_new_seq, + old_to_newseq_map); + mail_index_transaction_sort_appends_keywords(t, old_to_newseq_map); + i_free(old_to_newseq_map); + + t->appends_nonsorted = FALSE; +} diff --git a/src/lib-index/mail-index-transaction-update.c b/src/lib-index/mail-index-transaction-update.c new file mode 100644 index 0000000..c7bcbd9 --- /dev/null +++ b/src/lib-index/mail-index-transaction-update.c @@ -0,0 +1,1367 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +/* Inside transaction we keep messages stored in sequences in uid fields. + Before they're written to transaction log the sequences are changed to + UIDs. */ + +#include "lib.h" +#include "array.h" +#include "time-util.h" +#include "mail-index-private.h" +#include "mail-index-transaction-private.h" + +static bool +mail_index_transaction_has_ext_changes(struct mail_index_transaction *t); + +struct mail_index_record * +mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq) +{ + i_assert(seq >= t->first_new_seq && seq <= t->last_new_seq); + + return array_idx_modifiable(&t->appends, seq - t->first_new_seq); +} + +void mail_index_transaction_reset_v(struct mail_index_transaction *t) +{ + ARRAY_TYPE(seq_array) *rec; + struct mail_index_transaction_ext_hdr_update *ext_hdr; + + if (array_is_created(&t->ext_rec_updates)) { + array_foreach_modifiable(&t->ext_rec_updates, rec) { + if (array_is_created(rec)) + array_free(rec); + } + array_free(&t->ext_rec_updates); + } + if (array_is_created(&t->ext_rec_atomics)) { + array_foreach_modifiable(&t->ext_rec_atomics, rec) { + if (array_is_created(rec)) + array_free(rec); + } + array_free(&t->ext_rec_atomics); + } + if (array_is_created(&t->ext_hdr_updates)) { + array_foreach_modifiable(&t->ext_hdr_updates, ext_hdr) { + i_free(ext_hdr->data); + i_free(ext_hdr->mask); + } + array_free(&t->ext_hdr_updates); + } + + if (array_is_created(&t->keyword_updates)) { + struct mail_index_transaction_keyword_update *u; + + array_foreach_modifiable(&t->keyword_updates, u) { + if (array_is_created(&u->add_seq)) + array_free(&u->add_seq); + if (array_is_created(&u->remove_seq)) + array_free(&u->remove_seq); + } + array_free(&t->keyword_updates); + } + + if (array_is_created(&t->appends)) + array_free(&t->appends); + if (array_is_created(&t->modseq_updates)) + array_free(&t->modseq_updates); + if (array_is_created(&t->expunges)) + array_free(&t->expunges); + if (array_is_created(&t->updates)) + array_free(&t->updates); + if (array_is_created(&t->ext_resizes)) + array_free(&t->ext_resizes); + if (array_is_created(&t->ext_resets)) + array_free(&t->ext_resets); + if (array_is_created(&t->ext_reset_ids)) + array_free(&t->ext_reset_ids); + if (array_is_created(&t->ext_reset_atomic)) + array_free(&t->ext_reset_atomic); + buffer_free(&t->attribute_updates); + buffer_free(&t->attribute_updates_suffix); + + t->first_new_seq = mail_index_view_get_messages_count(t->view)+1; + t->last_new_seq = 0; + t->last_update_idx = 0; + t->min_flagupdate_seq = 0; + t->max_flagupdate_seq = 0; + t->min_highest_modseq = 0; + + memset(t->pre_hdr_mask, 0, sizeof(t->pre_hdr_mask)); + memset(t->post_hdr_mask, 0, sizeof(t->post_hdr_mask)); + + t->appends_nonsorted = FALSE; + t->expunges_nonsorted = FALSE; + t->drop_unnecessary_flag_updates = FALSE; + t->pre_hdr_changed = FALSE; + t->post_hdr_changed = FALSE; + t->reset = FALSE; + t->index_deleted = FALSE; + t->index_undeleted = FALSE; + t->log_updates = FALSE; + t->log_ext_updates = FALSE; + t->tail_offset_changed = FALSE; +} + +void mail_index_transaction_set_log_updates(struct mail_index_transaction *t) +{ + /* flag updates aren't included in log_updates */ + t->log_updates = array_is_created(&t->appends) || + array_is_created(&t->modseq_updates) || + array_is_created(&t->expunges) || + array_is_created(&t->keyword_updates) || + t->attribute_updates != NULL || + t->pre_hdr_changed || t->post_hdr_changed || + t->min_highest_modseq != 0; +} + +void mail_index_update_day_headers(struct mail_index_transaction *t, + time_t day_stamp) +{ + struct mail_index_header hdr; + const struct mail_index_record *rec; + const int max_days = N_ELEMENTS(hdr.day_first_uid); + time_t stamp; + int i, days; + + hdr = *mail_index_get_header(t->view); + rec = array_front(&t->appends); + + stamp = time_to_local_day_start(day_stamp); + if ((time_t)hdr.day_stamp >= stamp) + return; + + /* get number of days since last message */ + days = (stamp - hdr.day_stamp) / (3600*24); + if (days > max_days) + days = max_days; + + /* @UNSAFE: move days forward and fill the missing days with old + day_first_uid[0]. */ + if (days > 0 && days < max_days) + memmove(hdr.day_first_uid + days, hdr.day_first_uid, + (max_days - days) * sizeof(hdr.day_first_uid[0])); + for (i = 1; i < days; i++) + hdr.day_first_uid[i] = hdr.day_first_uid[0]; + + hdr.day_stamp = stamp; + hdr.day_first_uid[0] = rec->uid; + + mail_index_update_header(t, + offsetof(struct mail_index_header, day_stamp), + &hdr.day_stamp, sizeof(hdr.day_stamp), FALSE); + mail_index_update_header(t, + offsetof(struct mail_index_header, day_first_uid), + hdr.day_first_uid, sizeof(hdr.day_first_uid), FALSE); +} + +void mail_index_append(struct mail_index_transaction *t, uint32_t uid, + uint32_t *seq_r) +{ + struct mail_index_record *rec; + + i_assert(!t->no_appends); + + t->log_updates = TRUE; + + if (!array_is_created(&t->appends)) + i_array_init(&t->appends, 32); + + /* sequence number is visible only inside given view, + so let it generate it */ + if (t->last_new_seq != 0) + *seq_r = ++t->last_new_seq; + else + *seq_r = t->last_new_seq = t->first_new_seq; + + rec = array_append_space(&t->appends); + if (uid != 0) { + rec->uid = uid; + if (!t->appends_nonsorted && + t->last_new_seq != t->first_new_seq) { + /* if previous record's UID is larger than this one, + we'll have to sort the appends later */ + rec = mail_index_transaction_lookup(t, *seq_r - 1); + if (rec->uid > uid) + t->appends_nonsorted = TRUE; + else if (rec->uid == uid) + i_panic("Duplicate UIDs added in transaction"); + } + if (t->highest_append_uid < uid) + t->highest_append_uid = uid; + } +} + +void mail_index_append_finish_uids(struct mail_index_transaction *t, + uint32_t first_uid, + ARRAY_TYPE(seq_range) *uids_r) +{ + return mail_index_append_finish_uids_full(t, first_uid, first_uid, uids_r); +} + +void mail_index_append_finish_uids_full(struct mail_index_transaction *t, + uint32_t min_allowed_uid, + uint32_t first_new_uid, + ARRAY_TYPE(seq_range) *uids_r) +{ + struct mail_index_record *recs; + unsigned int i, count; + struct seq_range *range; + uint32_t next_uid; + + if (!array_is_created(&t->appends)) + return; + + i_assert(min_allowed_uid <= first_new_uid); + i_assert(first_new_uid < (uint32_t)-1); + + /* first find the highest assigned uid */ + recs = array_get_modifiable(&t->appends, &count); + i_assert(count > 0); + + next_uid = first_new_uid; + for (i = 0; i < count; i++) { + if (next_uid <= recs[i].uid) + next_uid = recs[i].uid + 1; + } + i_assert(next_uid > 0 && next_uid < (uint32_t)-1); + + /* assign missing uids */ + for (i = 0; i < count; i++) { + if (recs[i].uid == 0 || recs[i].uid < min_allowed_uid) { + i_assert(next_uid < (uint32_t)-1); + recs[i].uid = next_uid++; + if (t->highest_append_uid < recs[i].uid) + t->highest_append_uid = recs[i].uid; + } else { + t->appends_nonsorted = TRUE; + } + } + + /* write the saved uids range */ + array_clear(uids_r); + range = array_append_space(uids_r); + range->seq1 = range->seq2 = recs[0].uid; + for (i = 1; i < count; i++) { + if (range->seq2 + 1 == recs[i].uid) + range->seq2++; + else { + range = array_append_space(uids_r); + range->seq1 = range->seq2 = recs[i].uid; + } + } +} + +void mail_index_update_modseq(struct mail_index_transaction *t, uint32_t seq, + uint64_t min_modseq) +{ + struct mail_transaction_modseq_update *u; + + /* modseq=1 is the minimum always and it's only for mails that were + created/modified before modseqs were enabled. */ + if (min_modseq <= 1) + return; + + if (!array_is_created(&t->modseq_updates)) + i_array_init(&t->modseq_updates, 32); + + u = array_append_space(&t->modseq_updates); + u->uid = seq; + u->modseq_low32 = min_modseq & 0xffffffff; + u->modseq_high32 = min_modseq >> 32; + + t->log_updates = TRUE; +} + +void mail_index_update_highest_modseq(struct mail_index_transaction *t, + uint64_t min_modseq) +{ + /* modseq=1 is the minimum always and it's only for mails that were + created/modified before modseqs were enabled. */ + if (min_modseq <= 1) + return; + + if (t->min_highest_modseq < min_modseq) + t->min_highest_modseq = min_modseq; + + t->log_updates = TRUE; +} + +static void +mail_index_revert_ext(ARRAY_TYPE(seq_array_array) *ext_updates, + uint32_t seq) +{ + ARRAY_TYPE(seq_array) *seqs; + unsigned int idx; + + if (!array_is_created(ext_updates)) + return; + + array_foreach_modifiable(ext_updates, seqs) { + if (array_is_created(seqs) && + mail_index_seq_array_lookup(seqs, seq, &idx)) + array_delete(seqs, idx, 1); + } +} + +static void +mail_index_revert_changes_common(struct mail_index_transaction *t, uint32_t seq) +{ + struct mail_index_transaction_keyword_update *kw_update; + unsigned int i; + + /* remove extension updates */ + mail_index_revert_ext(&t->ext_rec_updates, seq); + mail_index_revert_ext(&t->ext_rec_atomics, seq); + t->log_ext_updates = mail_index_transaction_has_ext_changes(t); + + /* remove keywords */ + if (array_is_created(&t->keyword_updates)) { + array_foreach_modifiable(&t->keyword_updates, kw_update) { + if (array_is_created(&kw_update->add_seq)) { + seq_range_array_remove(&kw_update->add_seq, + seq); + } + if (array_is_created(&kw_update->remove_seq)) { + seq_range_array_remove(&kw_update->remove_seq, + seq); + } + } + } + /* remove modseqs */ + if (array_is_created(&t->modseq_updates) && + mail_index_seq_array_lookup((void *)&t->modseq_updates, seq, &i)) + array_delete(&t->modseq_updates, i, 1); +} + +void mail_index_revert_changes(struct mail_index_transaction *t, uint32_t seq) +{ + mail_index_revert_changes_common(t, seq); + (void)mail_index_cancel_flag_updates(t, seq); +} + +static void +mail_index_expunge_last_append(struct mail_index_transaction *t, uint32_t seq) +{ + i_assert(seq == t->last_new_seq); + + mail_index_revert_changes_common(t, seq); + + /* and finally remove the append itself */ + array_delete(&t->appends, seq - t->first_new_seq, 1); + t->last_new_seq--; + if (t->first_new_seq > t->last_new_seq) { + t->last_new_seq = 0; + t->appends_nonsorted = FALSE; + array_free(&t->appends); + } + mail_index_transaction_set_log_updates(t); +} + +void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq) +{ + static guid_128_t null_guid = + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + mail_index_expunge_guid(t, seq, null_guid); +} + +void mail_index_expunge_guid(struct mail_index_transaction *t, uint32_t seq, + const guid_128_t guid_128) +{ + const struct mail_transaction_expunge_guid *expunges; + struct mail_transaction_expunge_guid *expunge; + unsigned int count; + + i_assert(seq > 0); + if (seq >= t->first_new_seq) { + /* we can handle only the last append. otherwise we'd have to + renumber sequences and that gets tricky. for now this is + enough, since we typically want to expunge all the + appends. */ + mail_index_expunge_last_append(t, seq); + } else { + t->log_updates = TRUE; + + /* ignore duplicates here. drop them when committing. */ + if (!array_is_created(&t->expunges)) + i_array_init(&t->expunges, 64); + else if (!t->expunges_nonsorted) { + /* usually expunges are added in increasing order. */ + expunges = array_get(&t->expunges, &count); + if (count > 0 && seq < expunges[count-1].uid) + t->expunges_nonsorted = TRUE; + } + expunge = array_append_space(&t->expunges); + expunge->uid = seq; + memcpy(expunge->guid_128, guid_128, sizeof(expunge->guid_128)); + } +} + +static void update_minmax_flagupdate_seq(struct mail_index_transaction *t, + uint32_t seq1, uint32_t seq2) +{ + if (t->min_flagupdate_seq == 0) { + t->min_flagupdate_seq = seq1; + t->max_flagupdate_seq = seq2; + } else { + if (t->min_flagupdate_seq > seq1) + t->min_flagupdate_seq = seq1; + if (t->max_flagupdate_seq < seq2) + t->max_flagupdate_seq = seq2; + } +} + +unsigned int +mail_index_transaction_get_flag_update_pos(struct mail_index_transaction *t, + unsigned int left_idx, + unsigned int right_idx, + uint32_t seq) +{ + const struct mail_index_flag_update *updates; + unsigned int idx, count; + + updates = array_get(&t->updates, &count); + i_assert(left_idx <= right_idx && right_idx <= count); + i_assert(count < INT_MAX); + + /* find the first update with either overlapping range, + or the update which will come after our insert */ + idx = left_idx; + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + + if (updates[idx].uid2 < seq) + left_idx = idx+1; + else if (updates[idx].uid1 > seq) + right_idx = idx; + else + break; + } + if (left_idx > idx) + idx++; + return idx; +} + +static void +mail_index_insert_flag_update(struct mail_index_transaction *t, + struct mail_index_flag_update u, + unsigned int idx) +{ + struct mail_index_flag_update *updates, tmp_update; + unsigned int count, first_idx, max; + + updates = array_get_modifiable(&t->updates, &count); + + /* overlapping ranges, split/merge them */ + i_assert(idx == 0 || updates[idx-1].uid2 < u.uid1); + i_assert(idx == count || updates[idx].uid2 >= u.uid1); + + /* first we'll just add the changes without trying to merge anything */ + first_idx = idx; + for (; idx < count && u.uid2 >= updates[idx].uid1; idx++) { + i_assert(u.uid1 <= updates[idx].uid2); + if (u.uid1 != updates[idx].uid1 && + (updates[idx].add_flags != u.add_flags || + updates[idx].remove_flags != u.remove_flags)) { + if (u.uid1 < updates[idx].uid1) { + /* insert new update */ + tmp_update = u; + tmp_update.uid2 = updates[idx].uid1 - 1; + } else { + /* split existing update from beginning */ + tmp_update = updates[idx]; + tmp_update.uid2 = u.uid1 - 1; + updates[idx].uid1 = u.uid1; + } + + i_assert(tmp_update.uid1 <= tmp_update.uid2); + i_assert(updates[idx].uid1 <= updates[idx].uid2); + + array_insert(&t->updates, idx, &tmp_update, 1); + updates = array_get_modifiable(&t->updates, &count); + idx++; + } else if (u.uid1 < updates[idx].uid1) { + updates[idx].uid1 = u.uid1; + } + + if (u.uid2 < updates[idx].uid2 && + (updates[idx].add_flags != u.add_flags || + updates[idx].remove_flags != u.remove_flags)) { + /* split existing update from end */ + tmp_update = updates[idx]; + tmp_update.uid2 = u.uid2; + updates[idx].uid1 = u.uid2 + 1; + + i_assert(tmp_update.uid1 <= tmp_update.uid2); + i_assert(updates[idx].uid1 <= updates[idx].uid2); + + array_insert(&t->updates, idx, &tmp_update, 1); + updates = array_get_modifiable(&t->updates, &count); + } + + updates[idx].add_flags = + (updates[idx].add_flags | u.add_flags) & + ENUM_NEGATE(u.remove_flags); + updates[idx].remove_flags = + (updates[idx].remove_flags | u.remove_flags) & + ENUM_NEGATE(u.add_flags); + u.uid1 = updates[idx].uid2 + 1; + + if (updates[idx].add_flags == 0 && + updates[idx].remove_flags == 0) { + /* we can remove this update completely */ + array_delete(&t->updates, idx, 1); + updates = array_get_modifiable(&t->updates, &count); + } + + if (u.uid1 > u.uid2) { + /* break here before idx++ so last_update_idx is set + correctly */ + break; + } + } + i_assert(idx <= count); + + if (u.uid1 <= u.uid2) { + i_assert(idx == 0 || updates[idx-1].uid2 < u.uid1); + i_assert(idx == count || updates[idx].uid1 > u.uid2); + array_insert(&t->updates, idx, &u, 1); + } + updates = array_get_modifiable(&t->updates, &count); + t->last_update_idx = idx == count ? count-1 : idx; + + /* merge everything */ + idx = first_idx == 0 ? 0 : first_idx - 1; + max = count == 0 ? 0 : I_MIN(t->last_update_idx + 1, count-1); + for (; idx < max; ) { + if (updates[idx].uid2 + 1 == updates[idx+1].uid1 && + updates[idx].add_flags == updates[idx+1].add_flags && + updates[idx].remove_flags == updates[idx+1].remove_flags) { + /* merge */ + updates[idx].uid2 = updates[idx+1].uid2; + array_delete(&t->updates, idx + 1, 1); + max--; + if (t->last_update_idx > idx) + t->last_update_idx--; + updates = array_get_modifiable(&t->updates, &count); + } else { + idx++; + } + } +} + +static void mail_index_record_modify_flags(struct mail_index_record *rec, + enum modify_type modify_type, + enum mail_flags flags) +{ + switch (modify_type) { + case MODIFY_REPLACE: + rec->flags = flags; + break; + case MODIFY_ADD: + rec->flags |= flags; + break; + case MODIFY_REMOVE: + rec->flags &= ENUM_NEGATE(flags); + break; + } +} + +void mail_index_update_flags_range(struct mail_index_transaction *t, + uint32_t seq1, uint32_t seq2, + enum modify_type modify_type, + enum mail_flags flags) +{ + struct mail_index_record *rec; + struct mail_index_flag_update u, *last_update; + unsigned int idx, first_idx, count; + + update_minmax_flagupdate_seq(t, seq1, seq2); + if (seq2 >= t->first_new_seq) { + /* updates for appended messages, modify them directly */ + uint32_t seq; + + for (seq = I_MAX(t->first_new_seq, seq1); seq <= seq2; seq++) { + rec = mail_index_transaction_lookup(t, seq); + mail_index_record_modify_flags(rec, modify_type, flags); + } + if (seq1 >= t->first_new_seq) + return; + + /* range contains also existing messages. update them next. */ + seq2 = t->first_new_seq - 1; + } + + i_assert(seq1 <= seq2 && seq1 > 0); + i_assert(seq2 <= mail_index_view_get_messages_count(t->view)); + + if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES) != 0) + t->drop_unnecessary_flag_updates = TRUE; + + i_zero(&u); + u.uid1 = seq1; + u.uid2 = seq2; + + switch (modify_type) { + case MODIFY_REPLACE: + u.add_flags = flags; + u.remove_flags = ENUM_NEGATE(flags) & MAIL_INDEX_FLAGS_MASK; + break; + case MODIFY_ADD: + if (flags == 0) + return; + u.add_flags = flags; + break; + case MODIFY_REMOVE: + if (flags == 0) + return; + u.remove_flags = flags; + break; + } + + if (!array_is_created(&t->updates)) { + i_array_init(&t->updates, 256); + array_push_back(&t->updates, &u); + return; + } + + last_update = array_get_modifiable(&t->updates, &count); + if (t->last_update_idx < count) { + /* fast path - hopefully we're updating the next message, + or a message that is to be appended as last update */ + last_update += t->last_update_idx; + if (seq1 - 1 == last_update->uid2) { + if (u.add_flags == last_update->add_flags && + u.remove_flags == last_update->remove_flags && + (t->last_update_idx + 1 == count || + last_update[1].uid1 > seq2)) { + /* we can just update the UID range */ + last_update->uid2 = seq2; + return; + } + } else if (seq1 > last_update->uid2) { + /* hopefully we can just append it */ + t->last_update_idx++; + last_update++; + } + } + + if (t->last_update_idx == count) + array_push_back(&t->updates, &u); + else { + i_assert(t->last_update_idx < count); + + /* slow path */ + if (seq1 > last_update->uid2) { + /* added after this */ + first_idx = t->last_update_idx + 1; + } else { + /* added before this or on top of this */ + first_idx = 0; + count = t->last_update_idx + 1; + } + idx = mail_index_transaction_get_flag_update_pos(t, first_idx, + count, u.uid1); + mail_index_insert_flag_update(t, u, idx); + } +} + +void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq, + enum modify_type modify_type, + enum mail_flags flags) +{ + mail_index_update_flags_range(t, seq, seq, modify_type, flags); +} + +static void +mail_index_attribute_set_full(struct mail_index_transaction *t, + const char *key, bool pvt, char prefix, + time_t timestamp, uint32_t value_len) +{ + uint32_t ts = timestamp; + + if (t->attribute_updates == NULL) { + t->attribute_updates = buffer_create_dynamic(default_pool, 64); + t->attribute_updates_suffix = buffer_create_dynamic(default_pool, 64); + } + buffer_append_c(t->attribute_updates, prefix); + buffer_append_c(t->attribute_updates, pvt ? 'p' : 's'); + buffer_append(t->attribute_updates, key, strlen(key)+1); + + buffer_append(t->attribute_updates_suffix, &ts, sizeof(ts)); + if (prefix == '+') { + buffer_append(t->attribute_updates_suffix, + &value_len, sizeof(value_len)); + } + t->log_updates = TRUE; +} + +void mail_index_attribute_set(struct mail_index_transaction *t, + bool pvt, const char *key, + time_t timestamp, uint32_t value_len) +{ + mail_index_attribute_set_full(t, key, pvt, '+', timestamp, value_len); +} + +void mail_index_attribute_unset(struct mail_index_transaction *t, + bool pvt, const char *key, + time_t timestamp) +{ + mail_index_attribute_set_full(t, key, pvt, '-', timestamp, 0); +} + +void mail_index_update_header(struct mail_index_transaction *t, + size_t offset, const void *data, size_t size, + bool prepend) +{ + i_assert(offset < sizeof(t->pre_hdr_change)); + i_assert(size <= sizeof(t->pre_hdr_change) - offset); + + t->log_updates = TRUE; + + if (prepend) { + t->pre_hdr_changed = TRUE; + memcpy(t->pre_hdr_change + offset, data, size); + for (; size > 0; size--) + t->pre_hdr_mask[offset++] = 1; + } else { + t->post_hdr_changed = TRUE; + memcpy(t->post_hdr_change + offset, data, size); + for (; size > 0; size--) + t->post_hdr_mask[offset++] = 1; + } +} + +static void +mail_index_ext_rec_updates_resize(struct mail_index_transaction *t, + uint32_t ext_id, uint16_t new_record_size) +{ + ARRAY_TYPE(seq_array) *array, old_array; + unsigned int i; + + if (!array_is_created(&t->ext_rec_updates)) + return; + array = array_idx_modifiable(&t->ext_rec_updates, ext_id); + if (!array_is_created(array)) + return; + + old_array = *array; + i_zero(array); + mail_index_seq_array_alloc(array, new_record_size); + + /* copy the records' beginnings. leave the end zero-filled. */ + for (i = 0; i < array_count(&old_array); i++) { + const void *old_record = array_idx(&old_array, i); + + memcpy(array_append_space(array), old_record, + old_array.arr.element_size); + } + array_free(&old_array); +} + +void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t hdr_size, uint16_t record_size, + uint16_t record_align) +{ + const struct mail_index_registered_ext *rext; + const struct mail_transaction_ext_intro *resizes; + unsigned int resizes_count; + struct mail_transaction_ext_intro intro; + uint32_t old_record_size = 0, old_record_align, old_header_size; + + i_zero(&intro); + rext = array_idx(&t->view->index->extensions, ext_id); + + /* get ext_id from transaction's map if it's there */ + if (!mail_index_map_get_ext_idx(t->view->map, ext_id, &intro.ext_id)) { + /* have to create it */ + intro.ext_id = (uint32_t)-1; + old_record_align = rext->record_align; + old_header_size = rext->hdr_size; + } else { + const struct mail_index_ext *ext; + + ext = array_idx(&t->view->map->extensions, intro.ext_id); + old_record_align = ext->record_align; + old_header_size = ext->hdr_size; + } + + /* get the record size. if there are any existing record updates, + they're using the registered size, not the map's existing + record_size. */ + if (array_is_created(&t->ext_resizes)) + resizes = array_get(&t->ext_resizes, &resizes_count); + else { + resizes = NULL; + resizes_count = 0; + } + if (ext_id < resizes_count && resizes[ext_id].name_size != 0) { + /* already resized once. use the resized value. */ + old_record_size = resizes[ext_id].record_size; + } else { + /* use the registered values. */ + old_record_size = rext->record_size; + } + + if (record_size != old_record_size && record_size != (uint16_t)-1) { + /* if record_size grows, we'll just resize the existing + ext_rec_updates array. it's not possible to shrink + record_size without data loss. */ + i_assert(record_size > old_record_size); + mail_index_ext_rec_updates_resize(t, ext_id, record_size); + } + + t->log_ext_updates = TRUE; + + if (!array_is_created(&t->ext_resizes)) + i_array_init(&t->ext_resizes, ext_id + 2); + + intro.hdr_size = hdr_size != (uint32_t)-1 ? hdr_size : old_header_size; + if (record_size != (uint16_t)-1) { + i_assert(record_align != (uint16_t)-1); + intro.record_size = record_size; + intro.record_align = record_align; + } else { + i_assert(record_align == (uint16_t)-1); + intro.record_size = old_record_size; + intro.record_align = old_record_align; + } + intro.name_size = 1; + array_idx_set(&t->ext_resizes, ext_id, &intro); +} + +void mail_index_ext_resize_hdr(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t hdr_size) +{ + mail_index_ext_resize(t, ext_id, hdr_size, (uint16_t)-1, (uint16_t)-1); +} + +void mail_index_ext_reset(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t reset_id, bool clear_data) +{ + struct mail_transaction_ext_reset reset; + + i_assert(reset_id != 0); + + i_zero(&reset); + reset.new_reset_id = reset_id; + reset.preserve_data = clear_data ? 0 : 1; + + mail_index_ext_set_reset_id(t, ext_id, reset_id); + + if (!array_is_created(&t->ext_resets)) + i_array_init(&t->ext_resets, ext_id + 2); + array_idx_set(&t->ext_resets, ext_id, &reset); + t->log_ext_updates = TRUE; +} + +void mail_index_ext_reset_inc(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t prev_reset_id, bool clear_data) +{ + uint32_t expected_reset_id = prev_reset_id + 1; + + mail_index_ext_reset(t, ext_id, (uint32_t)-1, clear_data); + + if (!array_is_created(&t->ext_reset_atomic)) + i_array_init(&t->ext_reset_atomic, ext_id + 2); + array_idx_set(&t->ext_reset_atomic, ext_id, &expected_reset_id); +} + +static bool +mail_index_transaction_has_ext_updates(const ARRAY_TYPE(seq_array_array) *arr) +{ + const ARRAY_TYPE(seq_array) *array; + + if (array_is_created(arr)) { + array_foreach(arr, array) { + if (array_is_created(array)) + return TRUE; + } + } + return FALSE; +} + +static bool +mail_index_transaction_has_ext_changes(struct mail_index_transaction *t) +{ + if (mail_index_transaction_has_ext_updates(&t->ext_rec_updates)) + return TRUE; + if (mail_index_transaction_has_ext_updates(&t->ext_rec_atomics)) + return TRUE; + + if (array_is_created(&t->ext_hdr_updates)) { + const struct mail_index_transaction_ext_hdr_update *hdr; + + array_foreach(&t->ext_hdr_updates, hdr) { + if (hdr->alloc_size > 0) + return TRUE; + } + } + if (array_is_created(&t->ext_resets)) { + const struct mail_transaction_ext_reset *reset; + + array_foreach(&t->ext_resets, reset) { + if (reset->new_reset_id != 0) + return TRUE; + } + } + if (array_is_created(&t->ext_resizes)) { + const struct mail_transaction_ext_intro *resize; + + array_foreach(&t->ext_resizes, resize) { + if (resize->name_size > 0) + return TRUE; + } + } + return FALSE; +} + +static void +mail_index_ext_update_reset(ARRAY_TYPE(seq_array_array) *arr, uint32_t ext_id) +{ + if (array_is_created(arr) && ext_id < array_count(arr)) { + /* if extension records have been updated, clear them */ + ARRAY_TYPE(seq_array) *array; + + array = array_idx_modifiable(arr, ext_id); + if (array_is_created(array)) + array_clear(array); + } +} + +static void +mail_index_ext_reset_changes(struct mail_index_transaction *t, uint32_t ext_id) +{ + mail_index_ext_update_reset(&t->ext_rec_updates, ext_id); + mail_index_ext_update_reset(&t->ext_rec_atomics, ext_id); + if (array_is_created(&t->ext_hdr_updates) && + ext_id < array_count(&t->ext_hdr_updates)) { + /* if extension headers have been updated, clear them */ + struct mail_index_transaction_ext_hdr_update *hdr; + + hdr = array_idx_modifiable(&t->ext_hdr_updates, ext_id); + if (hdr->alloc_size > 0) { + i_free_and_null(hdr->mask); + i_free_and_null(hdr->data); + } + hdr->alloc_size = 0; + } + if (array_is_created(&t->ext_resets) && + ext_id < array_count(&t->ext_resets)) { + /* clear resets */ + array_idx_clear(&t->ext_resets, ext_id); + } + if (array_is_created(&t->ext_resizes) && + ext_id < array_count(&t->ext_resizes)) { + /* clear resizes */ + array_idx_clear(&t->ext_resizes, ext_id); + } + + t->log_ext_updates = mail_index_transaction_has_ext_changes(t); +} + +void mail_index_ext_using_reset_id(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t reset_id) +{ + uint32_t *reset_id_p; + bool changed; + + if (!array_is_created(&t->ext_reset_ids)) + i_array_init(&t->ext_reset_ids, ext_id + 2); + reset_id_p = array_idx_get_space(&t->ext_reset_ids, ext_id); + changed = *reset_id_p != reset_id && *reset_id_p != 0; + *reset_id_p = reset_id; + if (changed) { + /* reset_id changed, clear existing changes */ + mail_index_ext_reset_changes(t, ext_id); + } +} + +void mail_index_ext_set_reset_id(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t reset_id) +{ + mail_index_ext_using_reset_id(t, ext_id, reset_id); + /* make sure the changes get reset, even if reset_id doesn't change */ + mail_index_ext_reset_changes(t, ext_id); +} + +void mail_index_update_header_ext(struct mail_index_transaction *t, + uint32_t ext_id, size_t offset, + const void *data, size_t size) +{ + struct mail_index_transaction_ext_hdr_update *hdr; + size_t new_size; + + i_assert(offset <= (uint32_t)-1 && size <= (uint32_t)-1 && + offset + size <= (uint32_t)-1); + + if (!array_is_created(&t->ext_hdr_updates)) + i_array_init(&t->ext_hdr_updates, ext_id + 2); + + hdr = array_idx_get_space(&t->ext_hdr_updates, ext_id); + if (hdr->alloc_size < offset || hdr->alloc_size - offset < size) { + i_assert(size < SIZE_MAX - offset); + new_size = nearest_power(offset + size); + hdr->mask = i_realloc(hdr->mask, hdr->alloc_size, new_size); + hdr->data = i_realloc(hdr->data, hdr->alloc_size, new_size); + hdr->alloc_size = new_size; + } + memset(hdr->mask + offset, 1, size); + memcpy(hdr->data + offset, data, size); + + t->log_ext_updates = TRUE; +} + +void mail_index_update_ext(struct mail_index_transaction *t, uint32_t seq, + uint32_t ext_id, const void *data, void *old_data_r) +{ + struct mail_index *index = t->view->index; + const struct mail_index_registered_ext *rext; + const struct mail_transaction_ext_intro *intro; + uint16_t record_size; + ARRAY_TYPE(seq_array) *array; + unsigned int count; + + i_assert(seq > 0 && + (seq <= mail_index_view_get_messages_count(t->view) || + seq <= t->last_new_seq)); + i_assert(ext_id < array_count(&index->extensions)); + + t->log_ext_updates = TRUE; + + if (!array_is_created(&t->ext_resizes)) { + intro = NULL; + count = 0; + } else { + intro = array_get(&t->ext_resizes, &count); + } + if (ext_id < count && intro[ext_id].name_size != 0) { + /* resized record */ + record_size = intro[ext_id].record_size; + } else { + rext = array_idx(&index->extensions, ext_id); + record_size = rext->record_size; + } + i_assert(record_size > 0); + + if (!array_is_created(&t->ext_rec_updates)) + i_array_init(&t->ext_rec_updates, ext_id + 2); + array = array_idx_get_space(&t->ext_rec_updates, ext_id); + + /* @UNSAFE */ + if (!mail_index_seq_array_add(array, seq, data, record_size, + old_data_r)) { + /* not found, clear old_data if it was given */ + if (old_data_r != NULL) + memset(old_data_r, 0, record_size); + } +} + +int mail_index_atomic_inc_ext(struct mail_index_transaction *t, + uint32_t seq, uint32_t ext_id, int diff) +{ + ARRAY_TYPE(seq_array) *array; + int32_t old_diff32, diff32 = diff; + + i_assert(seq > 0 && + (seq <= mail_index_view_get_messages_count(t->view) || + seq <= t->last_new_seq)); + i_assert(ext_id < array_count(&t->view->index->extensions)); + /* currently non-external transactions can be applied multiple times, + causing multiple increments. FIXME: we need this now and it doesn't + actually seem to be a real problem at least right now - why? */ + /*i_assert((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0);*/ + + t->log_ext_updates = TRUE; + if (!array_is_created(&t->ext_rec_atomics)) + i_array_init(&t->ext_rec_atomics, ext_id + 2); + array = array_idx_get_space(&t->ext_rec_atomics, ext_id); + if (mail_index_seq_array_add(array, seq, &diff32, sizeof(diff32), + &old_diff32)) { + /* already incremented this sequence in this transaction */ + diff32 += old_diff32; + (void)mail_index_seq_array_add(array, seq, &diff32, + sizeof(diff32), NULL); + } + return diff32; +} + +static bool +keyword_update_has_changes(struct mail_index_transaction *t, uint32_t seq, + enum modify_type modify_type, + struct mail_keywords *keywords) +{ + ARRAY_TYPE(keyword_indexes) existing; + const unsigned int *existing_idx; + unsigned int i, j, existing_count; + bool found; + + t_array_init(&existing, 32); + if (seq < t->first_new_seq) + mail_index_transaction_lookup_latest_keywords(t, seq, &existing); + existing_idx = array_get(&existing, &existing_count); + + if (modify_type == MODIFY_REPLACE && existing_count != keywords->count) + return TRUE; + + for (i = 0; i < keywords->count; i++) { + found = FALSE; + for (j = 0; j < existing_count; j++) { + if (existing_idx[j] == keywords->idx[i]) { + found = TRUE; + break; + } + } + switch (modify_type) { + case MODIFY_ADD: + case MODIFY_REPLACE: + if (!found) + return TRUE; + break; + case MODIFY_REMOVE: + if (found) + return TRUE; + break; + } + } + return FALSE; +} + +static struct mail_keywords * +keyword_update_remove_existing(struct mail_index_transaction *t, uint32_t seq) +{ + ARRAY_TYPE(keyword_indexes) keywords; + uint32_t i, keywords_count; + + t_array_init(&keywords, 32); + if (t->view->v.lookup_full == NULL) { + /* syncing is saving a list of changes into this transaction. + the seq is actual an uid, so we can't lookup the existing + keywords. we shouldn't get here unless we're reading + pre-v2.2 keyword-reset records from .log files. so we don't + really care about performance that much here, */ + keywords_count = array_count(&t->view->index->keywords); + for (i = 0; i < keywords_count; i++) + array_push_back(&keywords, &i); + } else { + mail_index_transaction_lookup_latest_keywords(t, seq, &keywords); + } + if (array_count(&keywords) == 0) + return NULL; + return mail_index_keywords_create_from_indexes(t->view->index, + &keywords); +} + +void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq, + enum modify_type modify_type, + struct mail_keywords *keywords) +{ + struct mail_index_transaction_keyword_update *u; + struct mail_keywords *add_keywords = NULL, *remove_keywords = NULL; + struct mail_keywords *unref_keywords = NULL; + unsigned int i; + bool changed; + + i_assert(seq > 0 && + (seq <= mail_index_view_get_messages_count(t->view) || + seq <= t->last_new_seq)); + i_assert(keywords->index == t->view->index); + + if (keywords->count == 0 && modify_type != MODIFY_REPLACE) + return; + + update_minmax_flagupdate_seq(t, seq, seq); + + if (!array_is_created(&t->keyword_updates)) { + uint32_t max_idx = keywords->count == 0 ? 3 : + keywords->idx[keywords->count-1]; + + i_array_init(&t->keyword_updates, max_idx + 1); + } + + if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES) != 0) { + T_BEGIN { + changed = keyword_update_has_changes(t, seq, + modify_type, + keywords); + } T_END; + if (!changed) + return; + } + + switch (modify_type) { + case MODIFY_REPLACE: + /* split this into add+remove. remove all existing keywords not + included in the keywords list */ + if (seq < t->first_new_seq) { + /* remove the ones currently in index */ + remove_keywords = keyword_update_remove_existing(t, seq); + unref_keywords = remove_keywords; + } + /* remove from all changes we've done in this transaction */ + array_foreach_modifiable(&t->keyword_updates, u) + seq_range_array_remove(&u->add_seq, seq); + add_keywords = keywords; + break; + case MODIFY_ADD: + add_keywords = keywords; + break; + case MODIFY_REMOVE: + remove_keywords = keywords; + break; + } + + /* Update add_seq and remove_seq arrays which describe the keyword + changes. First do the removes, since replace removes everything + first. */ + if (remove_keywords != NULL) { + for (i = 0; i < remove_keywords->count; i++) { + u = array_idx_get_space(&t->keyword_updates, + remove_keywords->idx[i]); + seq_range_array_remove(&u->add_seq, seq); + /* Don't bother updating remove_seq for new messages, + since their initial state is "no keyword" anyway */ + if (seq < t->first_new_seq) { + seq_range_array_add_with_init(&u->remove_seq, + 16, seq); + } + } + } + if (add_keywords != NULL) { + for (i = 0; i < add_keywords->count; i++) { + u = array_idx_get_space(&t->keyword_updates, + add_keywords->idx[i]); + seq_range_array_add_with_init(&u->add_seq, 16, seq); + seq_range_array_remove(&u->remove_seq, seq); + } + } + if (unref_keywords != NULL) + mail_index_keywords_unref(&unref_keywords); + + t->log_updates = TRUE; +} + +bool mail_index_cancel_flag_updates(struct mail_index_transaction *t, + uint32_t seq) +{ + struct mail_index_flag_update *updates, tmp_update; + unsigned int i, count; + + if (!array_is_created(&t->updates)) + return FALSE; + + updates = array_get_modifiable(&t->updates, &count); + i = mail_index_transaction_get_flag_update_pos(t, 0, count, seq); + if (i == count) + return FALSE; + else { + i_assert(seq <= updates[i].uid2); + if (seq < updates[i].uid1) + return FALSE; + } + + /* exists */ + if (updates[i].uid1 == seq) { + if (updates[i].uid2 != seq) + updates[i].uid1++; + else if (count > 1) + array_delete(&t->updates, i, 1); + else + array_free(&t->updates); + } else if (updates[i].uid2 == seq) { + updates[i].uid2--; + } else { + /* need to split it in two */ + tmp_update = updates[i]; + tmp_update.uid1 = seq+1; + updates[i].uid2 = seq-1; + array_insert(&t->updates, i + 1, &tmp_update, 1); + } + return TRUE; +} + +static bool mail_index_cancel_array(ARRAY_TYPE(seq_range) *array, uint32_t seq) +{ + if (array_is_created(array)) { + if (seq_range_array_remove(array, seq)) { + if (array_count(array) == 0) + array_free(array); + return TRUE; + } + } + return FALSE; +} + +bool mail_index_cancel_keyword_updates(struct mail_index_transaction *t, + uint32_t seq) +{ + struct mail_index_transaction_keyword_update *kw; + bool ret = FALSE, have_kw_changes = FALSE; + + if (!array_is_created(&t->keyword_updates)) + return FALSE; + + array_foreach_modifiable(&t->keyword_updates, kw) { + if (mail_index_cancel_array(&kw->add_seq, seq)) + ret = TRUE; + if (mail_index_cancel_array(&kw->remove_seq, seq)) + ret = TRUE; + if (array_is_created(&kw->add_seq) || + array_is_created(&kw->remove_seq)) + have_kw_changes = TRUE; + } + if (!have_kw_changes) + array_free(&t->keyword_updates); + return ret; +} + +void mail_index_transaction_reset(struct mail_index_transaction *t) +{ + t->v.reset(t); +} + +void mail_index_reset(struct mail_index_transaction *t) +{ + mail_index_transaction_reset(t); + + t->reset = TRUE; +} + +void mail_index_unset_fscked(struct mail_index_transaction *t) +{ + struct mail_index_header new_hdr = + *mail_index_get_header(t->view); + + i_assert(t->view->index->log_sync_locked); + + /* remove fsck'd-flag if it exists. */ + if ((new_hdr.flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0) { + new_hdr.flags &= ENUM_NEGATE(MAIL_INDEX_HDR_FLAG_FSCKD); + mail_index_update_header(t, + offsetof(struct mail_index_header, flags), + &new_hdr.flags, sizeof(new_hdr.flags), FALSE); + } +} + +void mail_index_set_deleted(struct mail_index_transaction *t) +{ + i_assert(!t->index_undeleted); + + t->index_deleted = TRUE; +} + +void mail_index_set_undeleted(struct mail_index_transaction *t) +{ + i_assert(!t->index_deleted); + + t->index_undeleted = TRUE; +} + +void mail_index_transaction_set_max_modseq(struct mail_index_transaction *t, + uint64_t max_modseq, + ARRAY_TYPE(seq_range) *seqs) +{ + i_assert(array_is_created(seqs)); + + t->max_modseq = max_modseq; + t->conflict_seqs = seqs; +} diff --git a/src/lib-index/mail-index-transaction-view.c b/src/lib-index/mail-index-transaction-view.c new file mode 100644 index 0000000..240c7fe --- /dev/null +++ b/src/lib-index/mail-index-transaction-view.c @@ -0,0 +1,534 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "seq-range-array.h" +#include "mail-index-private.h" +#include "mail-index-view-private.h" +#include "mail-index-transaction-private.h" + +struct mail_index_view_transaction { + struct mail_index_view view; + struct mail_index_view_vfuncs *super; + struct mail_index_transaction *t; + + struct mail_index_map *lookup_map; + struct mail_index_header hdr; + + buffer_t *lookup_return_data; + uint32_t lookup_prev_seq; + + unsigned int record_size; + unsigned int recs_count; + void *recs; + ARRAY(void *) all_recs; +}; + +static void tview_close(struct mail_index_view *view) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + struct mail_index_transaction *t = tview->t; + void **recs; + unsigned int i, count; + + if (tview->lookup_map != NULL) + mail_index_unmap(&tview->lookup_map); + buffer_free(&tview->lookup_return_data); + + if (array_is_created(&tview->all_recs)) { + recs = array_get_modifiable(&tview->all_recs, &count); + for (i = 0; i < count; i++) + i_free(recs[i]); + array_free(&tview->all_recs); + } + + tview->super->close(view); + mail_index_transaction_unref(&t); +} + +static uint32_t tview_get_message_count(struct mail_index_view *view) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + + return view->map->hdr.messages_count + + (tview->t->last_new_seq == 0 ? 0 : + tview->t->last_new_seq - tview->t->first_new_seq + 1); +} + +static const struct mail_index_header * +tview_get_header(struct mail_index_view *view) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + const struct mail_index_header *hdr; + uint32_t next_uid; + + /* FIXME: header counters may not be correct */ + hdr = tview->super->get_header(view); + + next_uid = mail_index_transaction_get_next_uid(tview->t); + if (next_uid != hdr->next_uid) { + tview->hdr = *hdr; + tview->hdr.next_uid = next_uid; + hdr = &tview->hdr; + } + return hdr; +} + +static const struct mail_index_record * +tview_apply_flag_updates(struct mail_index_view_transaction *tview, + struct mail_index_map *map, + const struct mail_index_record *rec, uint32_t seq) +{ + struct mail_index_transaction *t = tview->t; + const struct mail_index_flag_update *updates; + struct mail_index_record *trec; + unsigned int idx, count; + + /* see if there are any flag updates */ + if (seq < t->min_flagupdate_seq || seq > t->max_flagupdate_seq || + !array_is_created(&t->updates)) + return rec; + + updates = array_get(&t->updates, &count); + idx = mail_index_transaction_get_flag_update_pos(t, 0, count, seq); + if (seq < updates[idx].uid1 || seq > updates[idx].uid2) + return rec; + + /* yes, we have flag updates. since we can't modify rec directly and + we want to be able to handle multiple mail_index_lookup() calls + without the second one overriding the first one's data, we'll + create a records array and return data from there. + + it's also possible that the record size increases, so we potentially + have to create multiple arrays. they all get eventually freed when + the view gets freed. */ + if (map->hdr.record_size > tview->record_size) { + if (!array_is_created(&tview->all_recs)) + i_array_init(&tview->all_recs, 4); + tview->recs_count = t->first_new_seq; + tview->record_size = I_MAX(map->hdr.record_size, + tview->view.map->hdr.record_size); + tview->recs = i_malloc(MALLOC_MULTIPLY(tview->record_size, + tview->recs_count)); + array_push_back(&tview->all_recs, &tview->recs); + } + i_assert(tview->recs_count == t->first_new_seq); + i_assert(seq > 0 && seq <= tview->recs_count); + + trec = PTR_OFFSET(tview->recs, (seq-1) * tview->record_size); + memcpy(trec, rec, map->hdr.record_size); + trec->flags |= updates[idx].add_flags & 0xff; + trec->flags &= ENUM_NEGATE(updates[idx].remove_flags); + return trec; +} + +static const struct mail_index_record * +tview_lookup_full(struct mail_index_view *view, uint32_t seq, + struct mail_index_map **map_r, bool *expunged_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + const struct mail_index_record *rec; + + if (seq >= tview->t->first_new_seq) { + /* FIXME: is this right to return index map..? + it's not there yet. */ + *map_r = view->index->map; + if (expunged_r != NULL) + *expunged_r = FALSE; + return mail_index_transaction_lookup(tview->t, seq); + } + + rec = tview->super->lookup_full(view, seq, map_r, expunged_r); + rec = tview_apply_flag_updates(tview, *map_r, rec, seq); + + if (expunged_r != NULL && + mail_index_transaction_is_expunged(tview->t, seq)) + *expunged_r = TRUE; + return rec; +} + +static void +tview_lookup_uid(struct mail_index_view *view, uint32_t seq, uint32_t *uid_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + + if (seq >= tview->t->first_new_seq) + *uid_r = mail_index_transaction_lookup(tview->t, seq)->uid; + else + tview->super->lookup_uid(view, seq, uid_r); +} + +static void tview_lookup_seq_range(struct mail_index_view *view, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + const struct mail_index_record *rec; + uint32_t seq; + + if (!tview->t->reset) { + tview->super->lookup_seq_range(view, first_uid, last_uid, + first_seq_r, last_seq_r); + } else { + /* index is being reset. we never want to return old + sequences. */ + *first_seq_r = *last_seq_r = 0; + } + if (tview->t->last_new_seq == 0) { + /* no new messages, the results are final. */ + return; + } + + rec = mail_index_transaction_lookup(tview->t, tview->t->first_new_seq); + if (rec->uid == 0) { + /* new messages don't have UIDs */ + return; + } + if (last_uid < rec->uid) { + /* all wanted messages were existing */ + return; + } + + /* at least some of the wanted messages are newly created */ + if (*first_seq_r == 0) { + seq = tview->t->first_new_seq; + for (; seq <= tview->t->last_new_seq; seq++) { + rec = mail_index_transaction_lookup(tview->t, seq); + if (first_uid <= rec->uid) + break; + } + if (seq > tview->t->last_new_seq || rec->uid > last_uid) { + /* no messages in range */ + return; + } + *first_seq_r = seq; + + if (rec->uid == last_uid) { + /* one seq in range */ + *last_seq_r = seq; + return; + } + } + + seq = tview->t->last_new_seq; + for (; seq >= tview->t->first_new_seq; seq--) { + rec = mail_index_transaction_lookup(tview->t, seq); + if (rec->uid <= last_uid) { + *last_seq_r = seq; + break; + } + } + i_assert(seq >= tview->t->first_new_seq); +} + +static void tview_lookup_first(struct mail_index_view *view, + enum mail_flags flags, uint8_t flags_mask, + uint32_t *seq_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + const struct mail_index_record *rec; + unsigned int append_count; + uint32_t seq, message_count; + + if (!tview->t->reset) { + tview->super->lookup_first(view, flags, flags_mask, seq_r); + if (*seq_r != 0) + return; + } else { + *seq_r = 0; + } + + rec = array_get(&tview->t->appends, &append_count); + seq = tview->t->first_new_seq; + message_count = tview->t->last_new_seq; + i_assert(append_count == message_count - seq + 1); + + for (; seq <= message_count; seq++, rec++) { + if ((rec->flags & flags_mask) == (uint8_t)flags) { + *seq_r = seq; + break; + } + } +} + +static void keyword_index_add(ARRAY_TYPE(keyword_indexes) *keywords, + unsigned int idx) +{ + const unsigned int *indexes; + unsigned int i, count; + + indexes = array_get(keywords, &count); + for (i = 0; i < count; i++) { + if (indexes[i] == idx) + return; + } + array_push_back(keywords, &idx); +} + +static void keyword_index_remove(ARRAY_TYPE(keyword_indexes) *keywords, + unsigned int idx) +{ + const unsigned int *indexes; + unsigned int i, count; + + indexes = array_get(keywords, &count); + for (i = 0; i < count; i++) { + if (indexes[i] == idx) { + array_delete(keywords, i, 1); + break; + } + } +} + +static void tview_lookup_keywords(struct mail_index_view *view, uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keyword_idx) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + struct mail_index_transaction *t = tview->t; + const struct mail_index_transaction_keyword_update *updates; + unsigned int i, count; + + tview->super->lookup_keywords(view, seq, keyword_idx); + + if (seq < t->min_flagupdate_seq || seq > t->max_flagupdate_seq) { + /* no keyword updates for this sequence */ + return; + } + + if (array_is_created(&t->keyword_updates)) + updates = array_get(&t->keyword_updates, &count); + else { + updates = NULL; + count = 0; + } + for (i = 0; i < count; i++) { + if (array_is_created(&updates[i].add_seq) && + seq_range_exists(&updates[i].add_seq, seq)) + keyword_index_add(keyword_idx, i); + else if (array_is_created(&updates[i].remove_seq) && + seq_range_exists(&updates[i].remove_seq, seq)) + keyword_index_remove(keyword_idx, i); + } +} + +static const void * +tview_return_updated_ext(struct mail_index_view_transaction *tview, + uint32_t seq, const void *data, uint32_t ext_id) +{ + const struct mail_index_ext *ext; + const struct mail_index_registered_ext *rext; + const struct mail_transaction_ext_intro *intro; + unsigned int record_align, record_size; + uint32_t ext_idx; + size_t pos; + + /* data begins with a 32bit sequence, followed by the actual + extension data */ + data = CONST_PTR_OFFSET(data, sizeof(uint32_t)); + + if (!mail_index_map_get_ext_idx(tview->lookup_map, ext_id, &ext_idx)) { + /* we're adding the extension now. */ + rext = array_idx(&tview->view.index->extensions, ext_id); + record_align = rext->record_align; + record_size = rext->record_size; + } else { + ext = array_idx(&tview->lookup_map->extensions, ext_idx); + record_align = ext->record_align; + record_size = ext->record_size; + } + + /* see if the extension has been resized within this transaction */ + if (array_is_created(&tview->t->ext_resizes) && + ext_id < array_count(&tview->t->ext_resizes)) { + intro = array_idx(&tview->t->ext_resizes, ext_id); + if (intro[ext_id].name_size != 0) { + record_align = intro->record_align; + record_size = intro->record_size; + } + } + + if (record_align <= sizeof(uint32_t)) { + /* data is 32bit aligned already */ + return data; + } else { + /* assume we want 64bit alignment - copy the data to + temporary buffer and return it */ + if (tview->lookup_return_data == NULL) { + tview->lookup_return_data = + buffer_create_dynamic(default_pool, + record_size + 64); + } else if (seq != tview->lookup_prev_seq) { + /* clear the buffer between lookups for different + messages */ + buffer_set_used_size(tview->lookup_return_data, 0); + } + tview->lookup_prev_seq = seq; + pos = tview->lookup_return_data->used; + buffer_append(tview->lookup_return_data, data, record_size); + return CONST_PTR_OFFSET(tview->lookup_return_data->data, pos); + } +} + +static bool +tview_is_ext_reset(struct mail_index_view_transaction *tview, uint32_t ext_id) +{ + const struct mail_transaction_ext_reset *resets; + unsigned int count; + + if (!array_is_created(&tview->t->ext_resets)) + return FALSE; + + resets = array_get(&tview->t->ext_resets, &count); + return ext_id < count && resets[ext_id].new_reset_id != 0; +} + +static bool +tview_lookup_ext_update(struct mail_index_view_transaction *tview, uint32_t seq, + uint32_t ext_id, struct mail_index_map **map_r, + const void **data_r) +{ + const ARRAY_TYPE(seq_array) *ext_buf; + const void *data; + unsigned int idx; + uint32_t map_ext_idx; + + ext_buf = array_idx(&tview->t->ext_rec_updates, ext_id); + if (!array_is_created(ext_buf) || + !mail_index_seq_array_lookup(ext_buf, seq, &idx)) + return FALSE; + + if (tview->lookup_map == NULL) { + tview->lookup_map = + mail_index_map_clone(tview->view.index->map); + } + if (!mail_index_map_get_ext_idx(tview->lookup_map, ext_id, &map_ext_idx)) { + /* extension doesn't yet exist in the map. add it there with + the preliminary information (mainly its size) so if caller + looks it up, it's going to be found. */ + const struct mail_index_registered_ext *rext = + array_idx(&tview->view.index->extensions, ext_id); + struct mail_index_ext_header ext_hdr; + + i_zero(&ext_hdr); + ext_hdr.hdr_size = rext->hdr_size; + ext_hdr.record_size = ext_buf->arr.element_size - sizeof(uint32_t); + ext_hdr.record_align = rext->record_align; + + mail_index_map_register_ext(tview->lookup_map, rext->name, + (uint32_t)-1, &ext_hdr); + } + + data = array_idx(ext_buf, idx); + *map_r = tview->lookup_map; + *data_r = tview_return_updated_ext(tview, seq, data, ext_id); + return TRUE; +} + +static void +tview_lookup_ext_full(struct mail_index_view *view, uint32_t seq, + uint32_t ext_id, struct mail_index_map **map_r, + const void **data_r, bool *expunged_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + + i_assert(ext_id < array_count(&view->index->extensions)); + + if (expunged_r != NULL) + *expunged_r = FALSE; + + if (array_is_created(&tview->t->ext_rec_updates) && + ext_id < array_count(&tview->t->ext_rec_updates)) { + /* there are some ext updates in transaction. + see if there's any for this sequence. */ + if (tview_lookup_ext_update(tview, seq, ext_id, map_r, data_r)) + return; + } + + /* not updated, return the existing value, unless ext was + already reset */ + if (seq < tview->t->first_new_seq && + !tview_is_ext_reset(tview, ext_id)) { + tview->super->lookup_ext_full(view, seq, ext_id, + map_r, data_r, expunged_r); + } else { + *map_r = view->index->map; + *data_r = NULL; + } +} + +static void tview_get_header_ext(struct mail_index_view *view, + struct mail_index_map *map, uint32_t ext_id, + const void **data_r, size_t *data_size_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + + /* FIXME: check updates */ + tview->super->get_header_ext(view, map, ext_id, data_r, data_size_r); +} + +static bool tview_ext_get_reset_id(struct mail_index_view *view, + struct mail_index_map *map, + uint32_t ext_id, uint32_t *reset_id_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + const uint32_t *reset_id_p; + + if (array_is_created(&tview->t->ext_reset_ids) && + ext_id < array_count(&tview->t->ext_reset_ids) && + map == tview->lookup_map) { + reset_id_p = array_idx(&tview->t->ext_reset_ids, ext_id); + *reset_id_r = *reset_id_p; + return TRUE; + } + + return tview->super->ext_get_reset_id(view, map, ext_id, reset_id_r); +} + +static struct mail_index_view_vfuncs trans_view_vfuncs = { + tview_close, + tview_get_message_count, + tview_get_header, + tview_lookup_full, + tview_lookup_uid, + tview_lookup_seq_range, + tview_lookup_first, + tview_lookup_keywords, + tview_lookup_ext_full, + tview_get_header_ext, + tview_ext_get_reset_id +}; + +struct mail_index_view * +mail_index_transaction_open_updated_view(struct mail_index_transaction *t) +{ + struct mail_index_view_transaction *tview; + + if (t->view->syncing) { + /* transaction view is being synced. while it's done, it's not + possible to add new messages, but the view itself might + change. so we can't make a copy of the view. */ + mail_index_view_ref(t->view); + return t->view; + } + + tview = i_new(struct mail_index_view_transaction, 1); + mail_index_view_clone(&tview->view, t->view); + tview->view.v = trans_view_vfuncs; + tview->super = &t->view->v; + tview->t = t; + + mail_index_transaction_ref(t); + return &tview->view; +} diff --git a/src/lib-index/mail-index-transaction.c b/src/lib-index/mail-index-transaction.c new file mode 100644 index 0000000..0c61170 --- /dev/null +++ b/src/lib-index/mail-index-transaction.c @@ -0,0 +1,360 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "hook-build.h" +#include "bsearch-insert-pos.h" +#include "llist.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" +#include "mail-index-transaction-private.h" + +static ARRAY(hook_mail_index_transaction_created_t *) + hook_mail_index_transaction_created; + +void mail_index_transaction_hook_register(hook_mail_index_transaction_created_t *hook) +{ + if (!array_is_created(&hook_mail_index_transaction_created)) + i_array_init(&hook_mail_index_transaction_created, 8); + array_push_back(&hook_mail_index_transaction_created, &hook); +} + +void mail_index_transaction_hook_unregister(hook_mail_index_transaction_created_t *hook) +{ + unsigned int idx; + bool found = FALSE; + + i_assert(array_is_created(&hook_mail_index_transaction_created)); + for(idx = 0; idx < array_count(&hook_mail_index_transaction_created); idx++) { + hook_mail_index_transaction_created_t *arr_hook = + array_idx_elem(&hook_mail_index_transaction_created, idx); + if (arr_hook == hook) { + array_delete(&hook_mail_index_transaction_created, idx, 1); + found = TRUE; + break; + } + } + i_assert(found == TRUE); + if (array_count(&hook_mail_index_transaction_created) == 0) + array_free(&hook_mail_index_transaction_created); +} + + +struct mail_index_view * +mail_index_transaction_get_view(struct mail_index_transaction *t) +{ + return t->view; +} + +bool mail_index_transaction_is_expunged(struct mail_index_transaction *t, + uint32_t seq) +{ + struct mail_transaction_expunge_guid key; + + if (!array_is_created(&t->expunges)) + return FALSE; + + if (t->expunges_nonsorted) + mail_index_transaction_sort_expunges(t); + + key.uid = seq; + return array_bsearch(&t->expunges, &key, + mail_transaction_expunge_guid_cmp) != NULL; +} + +void mail_index_transaction_ref(struct mail_index_transaction *t) +{ + t->refcount++; +} + +void mail_index_transaction_unref(struct mail_index_transaction **_t) +{ + struct mail_index_transaction *t = *_t; + + *_t = NULL; + if (--t->refcount > 0) + return; + + mail_index_transaction_reset_v(t); + + DLLIST_REMOVE(&t->view->transactions_list, t); + array_free(&t->module_contexts); + if (t->latest_view != NULL) + mail_index_view_close(&t->latest_view); + mail_index_view_close(&t->view); + i_free(t); +} + +uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t) +{ + const struct mail_index_header *head_hdr, *hdr; + unsigned int offset; + uint32_t next_uid; + + head_hdr = &t->view->index->map->hdr; + hdr = &t->view->map->hdr; + next_uid = t->reset || head_hdr->uid_validity != hdr->uid_validity ? + 1 : hdr->next_uid; + if (array_is_created(&t->appends) && t->highest_append_uid != 0) { + /* get next_uid from appends if they have UIDs. it's possible + that some appends have too low UIDs, they'll be caught + later. */ + if (next_uid <= t->highest_append_uid) + next_uid = t->highest_append_uid + 1; + } + + /* see if it's been updated in pre/post header changes */ + offset = offsetof(struct mail_index_header, next_uid); + if (t->post_hdr_mask[offset] != 0) { + hdr = (const void *)t->post_hdr_change; + if (hdr->next_uid > next_uid) + next_uid = hdr->next_uid; + } + if (t->pre_hdr_mask[offset] != 0) { + hdr = (const void *)t->pre_hdr_change; + if (hdr->next_uid > next_uid) + next_uid = hdr->next_uid; + } + return next_uid; +} + +void mail_index_transaction_lookup_latest_keywords(struct mail_index_transaction *t, + uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keywords) +{ + uint32_t uid, latest_seq; + + /* seq points to the transaction's primary view */ + mail_index_lookup_uid(t->view, seq, &uid); + + /* get the latest keywords from the updated index, or fallback to the + primary view if the message is already expunged */ + if (t->latest_view == NULL) { + mail_index_refresh(t->view->index); + t->latest_view = mail_index_view_open(t->view->index); + } + if (mail_index_lookup_seq(t->latest_view, uid, &latest_seq)) + mail_index_lookup_keywords(t->latest_view, latest_seq, keywords); + else + mail_index_lookup_keywords(t->view, seq, keywords); +} + +static int +mail_transaction_log_file_refresh(struct mail_index_transaction *t, + struct mail_transaction_log_append_ctx *ctx) +{ + struct mail_transaction_log_file *file; + + if (t->reset) { + /* Reset the whole index, preserving only indexid. Begin by + rotating the log. We don't care if we skip some non-synced + transactions. */ + if (mail_transaction_log_rotate(t->view->index->log, TRUE) < 0) + return -1; + + if (!MAIL_INDEX_TRANSACTION_HAS_CHANGES(t)) { + /* we only wanted to reset */ + return 0; + } + } + file = t->view->index->log->head; + + /* make sure we have everything mapped */ + if (mail_index_map(t->view->index, MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0) + return -1; + + i_assert(file->sync_offset >= file->buffer_offset); + ctx->new_highest_modseq = file->sync_highest_modseq; + return 1; +} + +static int +mail_index_transaction_commit_real(struct mail_index_transaction *t, + uoff_t *commit_size_r, + enum mail_index_transaction_change *changes_r) +{ + struct mail_transaction_log *log = t->view->index->log; + struct mail_transaction_log_append_ctx *ctx; + enum mail_transaction_type trans_flags = 0; + uint32_t log_seq1, log_seq2; + uoff_t log_offset1, log_offset2; + int ret; + + *changes_r = 0; + + if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0) + trans_flags |= MAIL_TRANSACTION_EXTERNAL; + if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_SYNC) != 0) + trans_flags |= MAIL_TRANSACTION_SYNC; + + if (mail_transaction_log_append_begin(log->index, trans_flags, &ctx) < 0) + return -1; + ret = mail_transaction_log_file_refresh(t, ctx); + if (ret > 0) T_BEGIN { + mail_index_transaction_finish(t); + mail_index_transaction_export(t, ctx, changes_r); + } T_END; + + mail_transaction_log_get_head(log, &log_seq1, &log_offset1); + if (mail_transaction_log_append_commit(&ctx) < 0 || ret < 0) + return -1; + mail_transaction_log_get_head(log, &log_seq2, &log_offset2); + i_assert(log_seq1 == log_seq2); + + if (t->reset) { + /* get rid of the old index. it might just confuse readers, + especially if it's broken. */ + i_unlink_if_exists(log->index->filepath); + } + + *commit_size_r = log_offset2 - log_offset1; + + if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0 && + log_offset1 != log_offset2) { + /* mark the area covered by this transaction hidden */ + mail_index_view_add_hidden_transaction(t->view, log_seq1, + log_offset1, log_offset2 - log_offset1); + } + return 0; +} + +static int mail_index_transaction_commit_v(struct mail_index_transaction *t, + struct mail_index_transaction_commit_result *result_r) +{ + struct mail_index *index = t->view->index; + bool changed; + int ret; + + i_assert(t->first_new_seq > + mail_index_view_get_messages_count(t->view)); + + changed = MAIL_INDEX_TRANSACTION_HAS_CHANGES(t) || t->reset; + ret = !changed ? 0 : + mail_index_transaction_commit_real(t, &result_r->commit_size, + &result_r->changes_mask); + mail_transaction_log_get_head(index->log, &result_r->log_file_seq, + &result_r->log_file_offset); + + if (ret == 0 && !index->syncing && changed) { + /* if we're committing a normal transaction, we want to + have those changes in the index mapping immediately. this + is especially important when committing cache offset + updates. + + however if we're syncing the index now, the mapping must + be done later as MAIL_INDEX_SYNC_HANDLER_FILE so that + expunge handlers get run for the newly expunged messages + (and sync handlers that require HANDLER_FILE as well). */ + index->sync_commit_result = result_r; + mail_index_refresh(index); + index->sync_commit_result = NULL; + } + + mail_index_transaction_unref(&t); + return ret; +} + +static void mail_index_transaction_rollback_v(struct mail_index_transaction *t) +{ + mail_index_transaction_unref(&t); +} + +int mail_index_transaction_commit(struct mail_index_transaction **t) +{ + struct mail_index_transaction_commit_result result; + + return mail_index_transaction_commit_full(t, &result); +} + +int mail_index_transaction_commit_full(struct mail_index_transaction **_t, + struct mail_index_transaction_commit_result *result_r) +{ + struct mail_index_transaction *t = *_t; + struct mail_index *index = t->view->index; + bool index_undeleted = t->index_undeleted; + + if (mail_index_view_is_inconsistent(t->view)) { + mail_index_set_error_nolog(index, "View is inconsistent"); + mail_index_transaction_rollback(_t); + return -1; + } + if (!index_undeleted && !t->commit_deleted_index) { + if (t->view->index->index_deleted || + (t->view->index->index_delete_requested && + !t->view->index->syncing)) { + /* no further changes allowed */ + mail_index_set_error_nolog(index, "Index is marked deleted"); + mail_index_transaction_rollback(_t); + return -1; + } + } + + *_t = NULL; + i_zero(result_r); + if (t->v.commit(t, result_r) < 0) + return -1; + + if (index_undeleted) { + index->index_deleted = FALSE; + index->index_delete_requested = FALSE; + } + return 0; +} + +void mail_index_transaction_rollback(struct mail_index_transaction **_t) +{ + struct mail_index_transaction *t = *_t; + + *_t = NULL; + t->v.rollback(t); +} + +static struct mail_index_transaction_vfuncs trans_vfuncs = { + mail_index_transaction_reset_v, + mail_index_transaction_commit_v, + mail_index_transaction_rollback_v +}; + +struct mail_index_transaction * +mail_index_transaction_begin(struct mail_index_view *view, + enum mail_index_transaction_flags flags) +{ + struct mail_index_transaction *t; + + /* don't allow syncing view while there's ongoing transactions */ + mail_index_view_ref(view); + + t = i_new(struct mail_index_transaction, 1); + t->refcount = 1; + t->v = trans_vfuncs; + t->view = view; + t->flags = flags; + + if (view->syncing) { + /* transaction view cannot work if new records are being added + in two places. make sure it doesn't happen. */ + t->no_appends = TRUE; + t->first_new_seq = (uint32_t)-1; + } else { + t->first_new_seq = + mail_index_view_get_messages_count(t->view) + 1; + } + + i_array_init(&t->module_contexts, + I_MIN(5, mail_index_module_register.id)); + DLLIST_PREPEND(&view->transactions_list, t); + + if (array_is_created(&hook_mail_index_transaction_created)) { + struct hook_build_context *ctx = + hook_build_init((void *)&t->v, sizeof(t->v)); + hook_mail_index_transaction_created_t *callback; + array_foreach_elem(&hook_mail_index_transaction_created, callback) { + callback(t); + hook_build_update(ctx, t->vlast); + } + t->vlast = NULL; + hook_build_deinit(&ctx); + } + return t; +} diff --git a/src/lib-index/mail-index-util.c b/src/lib-index/mail-index-util.c new file mode 100644 index 0000000..2b27ae1 --- /dev/null +++ b/src/lib-index/mail-index-util.c @@ -0,0 +1,138 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "bsearch-insert-pos.h" +#include "mail-index-private.h" + +uint32_t mail_index_uint32_to_offset(uint32_t offset) +{ + i_assert(offset < 0x40000000); + i_assert((offset & 3) == 0); + + offset >>= 2; + offset = 0x00000080 | ((offset & 0x0000007f)) | + 0x00008000 | ((offset & 0x00003f80) >> 7 << 8) | + 0x00800000 | ((offset & 0x001fc000) >> 14 << 16) | + 0x80000000 | ((offset & 0x0fe00000) >> 21 << 24); + + return cpu32_to_be(offset); +} + +uint32_t mail_index_offset_to_uint32(uint32_t offset) +{ + offset = be32_to_cpu(offset); + + if ((offset & 0x80808080) != 0x80808080) + return 0; + + return (((offset & 0x0000007f)) | + ((offset & 0x00007f00) >> 8 << 7) | + ((offset & 0x007f0000) >> 16 << 14) | + ((offset & 0x7f000000) >> 24 << 21)) << 2; +} + +void mail_index_pack_num(uint8_t **p, uint32_t num) +{ + /* number continues as long as the highest bit is set */ + while (num >= 0x80) { + **p = (num & 0x7f) | 0x80; + *p += 1; + num >>= 7; + } + + **p = num; + *p += 1; +} + +int mail_index_unpack_num(const uint8_t **p, const uint8_t *end, + uint32_t *num_r) +{ + const uint8_t *c = *p; + uint32_t value = 0; + unsigned int bits = 0; + + for (;;) { + if (unlikely(c == end)) { + /* we should never see EOF */ + *num_r = 0; + return -1; + } + + value |= (*c & 0x7f) << bits; + if (*c < 0x80) + break; + + bits += 7; + c++; + } + + if (unlikely(bits >= 32)) { + /* broken input */ + *p = end; + *num_r = 0; + return -1; + } + + *p = c + 1; + *num_r = value; + return 0; +} + +static int mail_index_seq_record_cmp(const uint32_t *key_seq, + const uint32_t *data_seq) +{ + return (int)*key_seq - (int)*data_seq; +} + +bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array, + uint32_t seq, unsigned int *idx_r) +{ + return array_bsearch_insert_pos(array, &seq, + mail_index_seq_record_cmp, idx_r); +} + +void mail_index_seq_array_alloc(ARRAY_TYPE(seq_array) *array, + size_t record_size) +{ + size_t aligned_record_size = (record_size + 3) & ~3U; + + i_assert(!array_is_created(array)); + + array_create(array, default_pool, + sizeof(uint32_t) + aligned_record_size, + 1024 / (sizeof(uint32_t) + aligned_record_size)); +} + +bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq, + const void *record, size_t record_size, + void *old_record) +{ + void *p; + unsigned int idx, aligned_record_size; + + /* records need to be 32bit aligned */ + aligned_record_size = (record_size + 3) & ~3U; + + if (!array_is_created(array)) + mail_index_seq_array_alloc(array, record_size); + i_assert(array->arr.element_size == sizeof(seq) + aligned_record_size); + + if (mail_index_seq_array_lookup(array, seq, &idx)) { + /* already there, update */ + p = array_idx_modifiable(array, idx); + if (old_record != NULL) { + /* save the old record before overwriting it */ + memcpy(old_record, PTR_OFFSET(p, sizeof(seq)), + record_size); + } + memcpy(PTR_OFFSET(p, sizeof(seq)), record, record_size); + return TRUE; + } else { + /* insert */ + p = array_insert_space(array, idx); + memcpy(p, &seq, sizeof(seq)); + memcpy(PTR_OFFSET(p, sizeof(seq)), record, record_size); + return FALSE; + } +} diff --git a/src/lib-index/mail-index-util.h b/src/lib-index/mail-index-util.h new file mode 100644 index 0000000..b61e16a --- /dev/null +++ b/src/lib-index/mail-index-util.h @@ -0,0 +1,22 @@ +#ifndef MAIL_INDEX_UTIL_H +#define MAIL_INDEX_UTIL_H + +ARRAY_DEFINE_TYPE(seq_array, uint32_t); + +uint32_t mail_index_uint32_to_offset(uint32_t offset); +uint32_t mail_index_offset_to_uint32(uint32_t offset); + +#define MAIL_INDEX_PACK_MAX_SIZE ((sizeof(uint32_t) * 8 + 7) / 7) +void mail_index_pack_num(uint8_t **p, uint32_t num); +int mail_index_unpack_num(const uint8_t **p, const uint8_t *end, + uint32_t *num_r); + +bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array, + uint32_t seq, unsigned int *idx_r); +void mail_index_seq_array_alloc(ARRAY_TYPE(seq_array) *array, + size_t record_size); +bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq, + const void *record, size_t record_size, + void *old_record) ATTR_NULL(5); + +#endif diff --git a/src/lib-index/mail-index-view-private.h b/src/lib-index/mail-index-view-private.h new file mode 100644 index 0000000..6e8b091 --- /dev/null +++ b/src/lib-index/mail-index-view-private.h @@ -0,0 +1,120 @@ +#ifndef MAIL_INDEX_VIEW_PRIVATE_H +#define MAIL_INDEX_VIEW_PRIVATE_H + +#include "mail-index-private.h" + +struct mail_index_view_log_sync_area { + uint32_t log_file_seq; + unsigned int length; + uoff_t log_file_offset; +}; +ARRAY_DEFINE_TYPE(view_log_sync_area, struct mail_index_view_log_sync_area); + +struct mail_index_view_vfuncs { + void (*close)(struct mail_index_view *view); + uint32_t (*get_messages_count)(struct mail_index_view *view); + const struct mail_index_header * + (*get_header)(struct mail_index_view *view); + const struct mail_index_record * + (*lookup_full)(struct mail_index_view *view, uint32_t seq, + struct mail_index_map **map_r, bool *expunged_r); + void (*lookup_uid)(struct mail_index_view *view, uint32_t seq, + uint32_t *uid_r); + void (*lookup_seq_range)(struct mail_index_view *view, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r); + void (*lookup_first)(struct mail_index_view *view, + enum mail_flags flags, uint8_t flags_mask, + uint32_t *seq_r); + void (*lookup_keywords)(struct mail_index_view *view, uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keyword_idx); + void (*lookup_ext_full)(struct mail_index_view *view, uint32_t seq, + uint32_t ext_id, struct mail_index_map **map_r, + const void **data_r, bool *expunged_r); + void (*get_header_ext)(struct mail_index_view *view, + struct mail_index_map *map, uint32_t ext_id, + const void **data_r, size_t *data_size_r); + bool (*ext_get_reset_id)(struct mail_index_view *view, + struct mail_index_map *map, + uint32_t ext_id, uint32_t *reset_id_r); +}; + +union mail_index_view_module_context { + struct mail_index_module_register *reg; +}; + +struct mail_index_view { + struct mail_index_view *prev, *next; + int refcount; + + struct mail_index_view_vfuncs v; + struct mail_index *index; + struct mail_transaction_log_view *log_view; + + /* Source location where the mail_index_view_open() call was done. + This helps debugging especially if a view is leaked. */ + const char *source_filename; + unsigned int source_linenum; + + /* Set the view inconsistent if this doesn't match mail_index.indexid */ + uint32_t indexid; + /* Set the view inconsistent if this doesn't match + mail_index.inconsistency_id. */ + unsigned int inconsistency_id; + uint64_t highest_modseq; + + struct mail_index_map *map; + /* All mappings where we have returned records. They need to be kept + valid until view is synchronized. */ + ARRAY(struct mail_index_map *) map_refs; + + /* expunge <= head. The expunge seq/offset points to the log file + how far expunges have been synced. The head seq/offset points to + how far non-expunges have been synced. They're usually the same, + unless MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES has been used. */ + uint32_t log_file_expunge_seq, log_file_head_seq; + uoff_t log_file_expunge_offset, log_file_head_offset; + + /* Transaction log areas which are returned as + mail_index_view_sync_rec.hidden=TRUE. Used to implement + MAIL_INDEX_TRANSACTION_FLAG_HIDE. */ + ARRAY_TYPE(view_log_sync_area) syncs_hidden; + + /* Module-specific contexts. */ + ARRAY(union mail_index_view_module_context *) module_contexts; + + /* Linked list of all transactions opened for the view. */ + struct mail_index_transaction *transactions_list; + + /* View is currently inconsistent. It can't be synced. */ + bool inconsistent:1; + /* this view is being synced */ + bool syncing:1; +}; + +struct mail_index_view * +mail_index_view_open_with_map(struct mail_index *index, + struct mail_index_map *map); +void mail_index_view_clone(struct mail_index_view *dest, + const struct mail_index_view *src, + const char *source_filename, + unsigned int source_linenum); +#define mail_index_view_clone(dest, src) \ + mail_index_view_clone(dest, src, __FILE__, __LINE__) + +struct mail_index_view * +mail_index_view_dup_private(const struct mail_index_view *src, + const char *source_filename, + unsigned int source_linenum); +#define mail_index_view_dup_private(src) \ + mail_index_view_dup_private(src, __FILE__, __LINE__) +void mail_index_view_ref(struct mail_index_view *view); +void mail_index_view_unref_maps(struct mail_index_view *view); +void mail_index_view_add_hidden_transaction(struct mail_index_view *view, + uint32_t log_file_seq, + uoff_t log_file_offset, + unsigned int length); + +struct mail_index_view *mail_index_dummy_view_open(struct mail_index *index); + +#endif diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c new file mode 100644 index 0000000..d8a5793 --- /dev/null +++ b/src/lib-index/mail-index-view-sync.c @@ -0,0 +1,1045 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log.h" + + +struct mail_index_view_sync_ctx { + struct mail_index_view *view; + enum mail_index_view_sync_flags flags; + struct mail_index_sync_map_ctx sync_map_ctx; + + /* After syncing view, map is replaced with sync_new_map. */ + struct mail_index_map *sync_new_map; + + ARRAY_TYPE(seq_range) expunges; + unsigned int finish_min_msg_count; + + const struct mail_transaction_header *hdr; + const void *data; + + /* temporary variables while handling lost transaction logs: */ + ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw; + buffer_t *lost_kw_buf; + uint32_t lost_new_ext_idx; + /* result of lost transaction logs: */ + ARRAY_TYPE(seq_range) lost_flags; + unsigned int lost_flag_idx; + + size_t data_offset; + bool failed:1; + bool sync_map_update:1; + bool skipped_expunges:1; + bool last_read:1; + bool log_was_lost:1; + bool hidden:1; +}; + +static int +view_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges, + bool *reset_r, bool *partial_sync_r, + const char **error_r) +{ + const struct mail_index_header *hdr = &view->index->map->hdr; + uint32_t start_seq, end_seq; + uoff_t start_offset, end_offset; + const char *reason; + int ret; + + *partial_sync_r = FALSE; + + if (sync_expunges) { + /* Sync everything after the last expunge syncing position. + We'll just skip over the non-expunge transaction records + that have already been synced previously. */ + start_seq = view->log_file_expunge_seq; + start_offset = view->log_file_expunge_offset; + } else { + /* Sync only new changes since the last view sync. */ + start_seq = view->log_file_head_seq; + start_offset = view->log_file_head_offset; + } + /* Sync the view up to the (already refreshed) index map. */ + end_seq = hdr->log_file_seq; + end_offset = hdr->log_file_head_offset; + + if (end_seq < view->log_file_head_seq || + (end_seq == view->log_file_head_seq && + end_offset < view->log_file_head_offset)) { + *error_r = t_strdup_printf( + "%s log position went backwards " + "(%u,%"PRIuUOFF_T" < %u,%"PRIuUOFF_T")", + view->index->filepath, end_seq, end_offset, + view->log_file_head_seq, view->log_file_head_offset); + return -1; + } + + for (;;) { + /* the view begins from the first non-synced transaction */ + ret = mail_transaction_log_view_set(view->log_view, + start_seq, start_offset, + end_seq, end_offset, + reset_r, &reason); + if (ret <= 0) { + *error_r = t_strdup_printf( + "Failed to map view for %s: %s", + view->index->filepath, reason); + return ret; + } + + if (!*reset_r || sync_expunges) + break; + + /* log was reset, but we don't want to sync expunges. + we can't do this, so sync only up to the reset. */ + mail_transaction_log_view_get_prev_pos(view->log_view, + &end_seq, &end_offset); + end_seq--; end_offset = UOFF_T_MAX; + if (end_seq < start_seq) { + /* we have only this reset log */ + mail_transaction_log_view_clear(view->log_view, + view->log_file_expunge_seq); + break; + } + *partial_sync_r = TRUE; + } + return 1; +} + +static unsigned int +view_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx) +{ + struct mail_index_view *view = ctx->view; + struct seq_range *src, *src_end, *dest; + unsigned int count, expunge_count = 0; + uint32_t prev_seq = 0; + + /* convert UIDs to sequences */ + src = dest = array_get_modifiable(&ctx->expunges, &count); + src_end = src + count; + for (; src != src_end; src++) { + if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2, + &dest->seq1, &dest->seq2)) + count--; + else { + i_assert(dest->seq1 > prev_seq); + prev_seq = dest->seq2; + + expunge_count += dest->seq2 - dest->seq1 + 1; + dest++; + } + } + array_delete(&ctx->expunges, count, + array_count(&ctx->expunges) - count); + return expunge_count; +} + +static void +view_sync_add_expunge_range(ARRAY_TYPE(seq_range) *dest, + const struct seq_range *src, size_t src_size) +{ + unsigned int i, src_count; + + i_assert(src_size % sizeof(*src) == 0); + + src_count = src_size / sizeof(*src); + for (i = 0; i < src_count; i++) + seq_range_array_add_range(dest, src[i].seq1, src[i].seq2); +} + +static void +view_sync_add_expunge_guids(ARRAY_TYPE(seq_range) *dest, + const struct mail_transaction_expunge_guid *src, + size_t src_size) +{ + unsigned int i, src_count; + + i_assert(src_size % sizeof(*src) == 0); + + src_count = src_size / sizeof(*src); + for (i = 0; i < src_count; i++) + seq_range_array_add(dest, src[i].uid); +} + +static int +view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx, + unsigned int *expunge_count_r) +{ + struct mail_index_view *view = ctx->view; + const struct mail_transaction_header *hdr; + const void *data; + int ret; + + /* get a list of expunge transactions. there may be some that we have + already synced, but it doesn't matter because they'll get dropped + out when converting to sequences. the uid ranges' validity has + already been verified, so we can use them directly. */ + mail_transaction_log_view_mark(view->log_view); + while ((ret = mail_transaction_log_view_next(view->log_view, + &hdr, &data)) > 0) { + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* skip expunge requests */ + continue; + } + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) { + view_sync_add_expunge_guids(&ctx->expunges, + data, hdr->size); + } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) { + view_sync_add_expunge_range(&ctx->expunges, + data, hdr->size); + } + } + mail_transaction_log_view_rewind(view->log_view); + + *expunge_count_r = view_sync_expunges2seqs(ctx); + return ret; +} + +static bool have_existing_expunges(struct mail_index_view *view, + const struct seq_range *range, size_t size) +{ + const struct seq_range *range_end; + uint32_t seq1, seq2; + + range_end = CONST_PTR_OFFSET(range, size); + for (; range != range_end; range++) { + if (mail_index_lookup_seq_range(view, range->seq1, range->seq2, + &seq1, &seq2)) + return TRUE; + } + return FALSE; +} + +static bool +have_existing_guid_expunge(struct mail_index_view *view, + const struct mail_transaction_expunge_guid *expunges, + size_t size) +{ + const struct mail_transaction_expunge_guid *expunges_end; + uint32_t seq; + + expunges_end = CONST_PTR_OFFSET(expunges, size); + for (; expunges != expunges_end; expunges++) { + if (mail_index_lookup_seq(view, expunges->uid, &seq)) + return TRUE; + } + return FALSE; +} + +static bool view_sync_have_expunges(struct mail_index_view *view) +{ + const struct mail_transaction_header *hdr; + const void *data; + bool have_expunges = FALSE; + int ret; + + if (mail_transaction_log_view_is_last(view->log_view)) + return FALSE; + + mail_transaction_log_view_mark(view->log_view); + + while ((ret = mail_transaction_log_view_next(view->log_view, + &hdr, &data)) > 0) { + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* skip expunge requests */ + continue; + } + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) { + /* we have an expunge. see if it still exists. */ + if (have_existing_guid_expunge(view, data, hdr->size)) { + have_expunges = TRUE; + break; + } + } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) { + /* we have an expunge. see if it still exists. */ + if (have_existing_expunges(view, data, hdr->size)) { + have_expunges = TRUE; + break; + } + } + } + + mail_transaction_log_view_rewind(view->log_view); + + /* handle failures as having expunges (which is safer). + we'll probably fail later. */ + return ret < 0 || have_expunges; +} + +static int uint_cmp(const void *p1, const void *p2) +{ + const unsigned int *u1 = p1, *u2 = p2; + + if (*u1 < *u2) + return -1; + if (*u1 > *u2) + return 1; + return 0; +} + +static bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx) +{ + unsigned int *old_idx, *new_idx; + unsigned int old_count, new_count; + + old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count); + new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count); + if (old_count != new_count) + return FALSE; + + qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp); + qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp); + return memcmp(old_idx, new_idx, old_count * sizeof(*old_idx)) == 0; +} + +static int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx, + uint32_t uid) +{ + struct mail_transaction_header thdr; + struct mail_transaction_keyword_update kw_up; + const unsigned int *kw_idx; + const char *const *kw_names; + unsigned int i, count; + + kw_idx = array_get(&ctx->lost_new_kw, &count); + if (count == 0) + return 0; + kw_names = array_front(&ctx->view->index->keywords); + + i_zero(&thdr); + thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL; + i_zero(&kw_up); + kw_up.modify_type = MODIFY_ADD; + /* add new flags one by one */ + for (i = 0; i < count; i++) { + kw_up.name_size = strlen(kw_names[kw_idx[i]]); + buffer_set_used_size(ctx->lost_kw_buf, 0); + buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up)); + buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]], + kw_up.name_size); + if (ctx->lost_kw_buf->used % 4 != 0) { + buffer_append_zero(ctx->lost_kw_buf, + 4 - ctx->lost_kw_buf->used % 4); + } + buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid)); + buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid)); + + thdr.size = ctx->lost_kw_buf->used; + if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr, + ctx->lost_kw_buf->data) < 0) + return -1; + } + return 0; +} + +static int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx, + uint32_t old_seq, uint32_t new_seq) +{ + struct mail_index_map *old_map = ctx->view->map; + struct mail_index_map *new_map = ctx->view->index->map; + const struct mail_index_record *old_rec, *new_rec; + struct mail_transaction_header thdr; + const struct mail_index_ext *ext; + const uint64_t *modseqp; + uint64_t new_modseq; + bool changed = FALSE; + + old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, old_seq); + new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, new_seq); + + i_zero(&thdr); + if (old_rec->flags != new_rec->flags) { + struct mail_transaction_flag_update flag_update; + + /* check this before syncing the record, since it updates + old_rec. */ + if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) != + (new_rec->flags & MAIL_INDEX_FLAGS_MASK)) + changed = TRUE; + + thdr.type = MAIL_TRANSACTION_FLAG_UPDATE | + MAIL_TRANSACTION_EXTERNAL; + thdr.size = sizeof(flag_update); + + i_zero(&flag_update); + flag_update.uid1 = flag_update.uid2 = new_rec->uid; + flag_update.add_flags = new_rec->flags; + flag_update.remove_flags = ENUM_NEGATE(new_rec->flags) & 0xff; + if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr, + &flag_update) < 0) + return -1; + } + + mail_index_map_lookup_keywords(old_map, old_seq, &ctx->lost_old_kw); + mail_index_map_lookup_keywords(new_map, new_seq, &ctx->lost_new_kw); + if (!view_sync_lost_keywords_equal(ctx)) { + struct mail_transaction_keyword_reset kw_reset; + + thdr.type = MAIL_TRANSACTION_KEYWORD_RESET | + MAIL_TRANSACTION_EXTERNAL; + thdr.size = sizeof(kw_reset); + + /* remove all old flags by resetting them */ + i_zero(&kw_reset); + kw_reset.uid1 = kw_reset.uid2 = new_rec->uid; + if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr, + &kw_reset) < 0) + return -1; + + if (view_sync_update_keywords(ctx, new_rec->uid) < 0) + return -1; + changed = TRUE; + } + + if (changed) { + /* flags or keywords changed */ + } else if (ctx->view->highest_modseq != 0 && + ctx->lost_new_ext_idx != (uint32_t)-1) { + /* if modseq has changed include this message in changed flags + list, even if we didn't see any changes above. */ + ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx); + modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset); + new_modseq = *modseqp; + + if (new_modseq > ctx->view->highest_modseq) + changed = TRUE; + } + + /* without modseqs lost_flags isn't updated perfectly correctly, because + by the time we're comparing old flags it may have changed from what + we last sent to the client (because the map is shared). This could + be avoided by always keeping a private copy of the map in the view, + but that's a waste of memory for as rare of a problem as this. */ + if (changed) + seq_range_array_add(&ctx->lost_flags, new_rec->uid); + return 0; +} + +static int +view_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx, + unsigned int *expunge_count_r) +{ + struct mail_index_view *view = ctx->view; + struct mail_index_map *old_map = view->map; + struct mail_index_map *new_map = view->index->map; + const unsigned int old_count = old_map->hdr.messages_count; + const unsigned int new_count = new_map->hdr.messages_count; + const struct mail_index_record *old_rec, *new_rec; + struct mail_transaction_header thdr; + uint32_t seqi, seqj; + + /* we don't update the map in the same order as it's typically done. + map->rec_map may already have some messages appended that we don't + want. get an atomic map to make sure these get removed. */ + (void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx); + + if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id, + &ctx->lost_new_ext_idx)) + ctx->lost_new_ext_idx = (uint32_t)-1; + + i_array_init(&ctx->lost_flags, 64); + t_array_init(&ctx->lost_old_kw, 32); + t_array_init(&ctx->lost_new_kw, 32); + ctx->lost_kw_buf = t_buffer_create(128); + + /* handle expunges and sync flags */ + seqi = seqj = 1; + while (seqi <= old_count && seqj <= new_count) { + old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi); + new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj); + if (old_rec->uid == new_rec->uid) { + /* message found - check if flags have changed */ + if (view_sync_apply_lost_changes(ctx, seqi, seqj) < 0) + return -1; + seqi++; seqj++; + } else if (old_rec->uid < new_rec->uid) { + /* message expunged */ + seq_range_array_add(&ctx->expunges, old_rec->uid); + seqi++; + } else { + /* new message appeared out of nowhere */ + mail_index_set_error(view->index, + "%s view is inconsistent: " + "uid=%u inserted in the middle of mailbox", + view->index->filepath, new_rec->uid); + return -1; + } + } + /* if there are old messages left, they're all expunged */ + for (; seqi <= old_count; seqi++) { + old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi); + seq_range_array_add(&ctx->expunges, old_rec->uid); + } + /* if there are new messages left, they're all new messages */ + thdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL; + thdr.size = sizeof(*new_rec); + for (; seqj <= new_count; seqj++) { + new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj); + if (mail_index_sync_record(&ctx->sync_map_ctx, + &thdr, new_rec) < 0) + return -1; + mail_index_map_lookup_keywords(new_map, seqj, + &ctx->lost_new_kw); + if (view_sync_update_keywords(ctx, new_rec->uid) < 0) + return -1; + } + *expunge_count_r = view_sync_expunges2seqs(ctx); + + /* we have no idea how far we've synced - make sure these aren't used */ + old_map->hdr.log_file_seq = 0; + old_map->hdr.log_file_head_offset = 0; + old_map->hdr.log_file_tail_offset = 0; + + if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) { + /* Expunges aren't wanted to be synced. Remember if we skipped + over any expunges. If yes, we must not update + log_file_expunge_seq/offset at the end of the view sync + so that a later sync can finish the expunges. */ + array_clear(&ctx->expunges); + ctx->skipped_expunges = *expunge_count_r > 0; + } + /* After the view sync is finished, update + log_file_head_seq/offset, since we've synced everything + (except maybe the expunges) up to this point. */ + view->log_file_head_seq = new_map->hdr.log_file_seq; + view->log_file_head_offset = new_map->hdr.log_file_head_offset; + return 0; +} + +static int mail_index_view_sync_init_fix(struct mail_index_view_sync_ctx *ctx) +{ + struct mail_index_view *view = ctx->view; + uint32_t seq; + uoff_t offset; + const char *reason; + bool reset; + int ret; + + /* replace the view's map */ + view->index->map->refcount++; + mail_index_unmap(&view->map); + view->map = view->index->map; + + /* update log positions */ + view->log_file_head_seq = seq = view->map->hdr.log_file_seq; + view->log_file_head_offset = offset = + view->map->hdr.log_file_head_offset; + + ret = mail_transaction_log_view_set(view->log_view, seq, offset, + seq, offset, &reset, &reason); + if (ret <= 0) { + mail_index_set_error(view->index, "Failed to fix view for %s: %s", + view->index->filepath, reason); + return ret; + } + view->inconsistent = FALSE; + return 0; +} + +struct mail_index_view_sync_ctx * +mail_index_view_sync_begin(struct mail_index_view *view, + enum mail_index_view_sync_flags flags) +{ + struct mail_index_view_sync_ctx *ctx; + struct mail_index_map *tmp_map; + unsigned int expunge_count = 0; + bool reset, partial_sync, sync_expunges, have_expunges; + const char *error; + int ret; + + i_assert(!view->syncing); + i_assert(view->transactions_list == NULL); + + view->syncing = TRUE; + + /* Syncing the view invalidates all previous looked up records. + Unreference the mappings this view keeps because of them. */ + mail_index_view_unref_maps(view); + + ctx = i_new(struct mail_index_view_sync_ctx, 1); + ctx->view = view; + ctx->flags = flags; + + sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0; + if (sync_expunges) + i_array_init(&ctx->expunges, 64); + if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0) { + /* just get this view synced - don't return anything */ + i_assert(sync_expunges); + if (mail_index_view_sync_init_fix(ctx) < 0) + ctx->failed = TRUE; + return ctx; + } + if (mail_index_view_is_inconsistent(view)) { + mail_index_set_error(view->index, "%s view is inconsistent", + view->index->filepath); + ctx->failed = TRUE; + return ctx; + } + + ret = view_sync_set_log_view_range(view, sync_expunges, &reset, + &partial_sync, &error); + if (ret < 0) { + mail_index_set_error(view->index, "%s", error); + ctx->failed = TRUE; + return ctx; + } + + if (ret == 0) { + /* Log the warning only when all expunges have been synced + by previous syncs. This way when there's a _FLAG_NOEXPUNGES + sync, there's no second warning logged when the expunges + finally are synced. */ + if (view->log_file_expunge_seq == view->log_file_head_seq && + view->log_file_expunge_offset == view->log_file_head_offset) { + e_warning(view->index->event, + "%s - generating missing logs", error); + } + ctx->log_was_lost = TRUE; + if (!sync_expunges) + i_array_init(&ctx->expunges, 64); + mail_index_sync_map_init(&ctx->sync_map_ctx, view, + MAIL_INDEX_SYNC_HANDLER_VIEW); + ret = view_sync_get_log_lost_changes(ctx, &expunge_count); + mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx); + mail_index_sync_map_deinit(&ctx->sync_map_ctx); + if (ret < 0) { + mail_index_set_error(view->index, + "%s view syncing failed to apply changes", + view->index->filepath); + view->inconsistent = TRUE; + ctx->failed = TRUE; + return ctx; + } + have_expunges = expunge_count > 0; + } else if (sync_expunges) { + /* get list of all expunges first */ + if (view_sync_get_expunges(ctx, &expunge_count) < 0) { + ctx->failed = TRUE; + return ctx; + } + have_expunges = expunge_count > 0; + } else if (view->log_file_expunge_seq == view->log_file_head_seq && + view->log_file_expunge_offset == view->log_file_head_offset) { + /* Previous syncs haven't left any pending expunges. See if + this sync will. */ + have_expunges = view_sync_have_expunges(view); + } else { + /* Expunges weren't synced in the previous sync either. + We already know there are missing expunges. */ + ctx->skipped_expunges = TRUE; + have_expunges = TRUE; + } + + ctx->finish_min_msg_count = reset ? 0 : + view->map->hdr.messages_count - expunge_count; + if (!reset) + ; + else if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_2ND_INDEX) != 0 && + view->map->hdr.messages_count == 0) { + /* The secondary index is still empty, so it may have + just been created for the first time. This is + expected, so it shouldn't cause the view to become + inconsistent. */ + if (mail_index_view_sync_init_fix(ctx) < 0) + ctx->failed = TRUE; + return ctx; + } else { + view->inconsistent = TRUE; + mail_index_set_error(view->index, + "%s reset, view is now inconsistent", + view->index->filepath); + ctx->failed = TRUE; + return ctx; + } + + if (!have_expunges && !partial_sync) { + /* no expunges, we can just replace the map */ + if (view->index->map->hdr.messages_count < + ctx->finish_min_msg_count) { + mail_index_set_error(view->index, + "Index %s lost messages without expunging " + "(%u -> %u)", view->index->filepath, + view->map->hdr.messages_count, + view->index->map->hdr.messages_count); + ctx->finish_min_msg_count = 0; + view->inconsistent = TRUE; + } + + view->index->map->refcount++; + mail_index_unmap(&view->map); + view->map = view->index->map; + } else { + /* a) expunges seen. b) doing a partial sync because we saw + a reset. + + Create a private map which we update. If we're syncing + expunges the map will finally be replaced with the head map + to remove the expunged messages. */ + ctx->sync_map_update = TRUE; + + if (view->map->refcount > 1) { + tmp_map = mail_index_map_clone(view->map); + mail_index_unmap(&view->map); + view->map = tmp_map; + } + + if (sync_expunges) { + ctx->sync_new_map = view->index->map; + ctx->sync_new_map->refcount++; + } + } + mail_index_sync_map_init(&ctx->sync_map_ctx, view, + MAIL_INDEX_SYNC_HANDLER_VIEW); + +#ifdef DEBUG + mail_index_map_check(view->map); +#endif + return ctx; +} + +static bool +view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset) +{ + const struct mail_index_view_log_sync_area *sync; + + if (!array_is_created(&view->syncs_hidden)) + return FALSE; + + array_foreach(&view->syncs_hidden, sync) { + if (sync->log_file_offset <= offset && + offset - sync->log_file_offset < sync->length && + sync->log_file_seq == seq) + return TRUE; + } + + return FALSE; +} + +static bool +mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx, + const struct mail_transaction_header *hdr) +{ + struct mail_index_view *view = ctx->view; + uint32_t seq; + uoff_t offset, next_offset; + + mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset); + next_offset = offset + sizeof(*hdr) + hdr->size; + + if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE | + MAIL_TRANSACTION_EXPUNGE_GUID)) != 0 && + (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) { + if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) { + i_assert(!LOG_IS_BEFORE(seq, offset, + view->log_file_expunge_seq, + view->log_file_expunge_offset)); + if (!ctx->skipped_expunges) { + view->log_file_expunge_seq = seq; + view->log_file_expunge_offset = offset; + ctx->skipped_expunges = TRUE; + } + return FALSE; + } + if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq, + view->log_file_expunge_offset)) { + /* already synced */ + return FALSE; + } + } + + if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq, + view->log_file_head_offset)) { + /* already synced */ + return FALSE; + } + + view->log_file_head_seq = seq; + view->log_file_head_offset = next_offset; + return TRUE; +} + +static int +mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx) +{ + struct mail_transaction_log_view *log_view = ctx->view->log_view; + struct mail_index_view *view = ctx->view; + const struct mail_transaction_header *hdr; + uint32_t seq; + uoff_t offset; + int ret; + bool synced_to_map; + + do { + /* Get the next transaction from log. */ + ret = mail_transaction_log_view_next(log_view, &ctx->hdr, + &ctx->data); + if (ret <= 0) { + if (ret < 0) + return -1; + + ctx->hdr = NULL; + ctx->last_read = TRUE; + return 0; + } + + hdr = ctx->hdr; + /* skip records we've already synced */ + } while (!mail_index_view_sync_want(ctx, hdr)); + + mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset); + + /* If we started from a map that we didn't create ourself, + some of the transactions may already be synced. at the end + of this view sync we'll update file_seq=0 so that this check + always becomes FALSE for subsequent syncs. */ + synced_to_map = view->map->hdr.log_file_seq != 0 && + LOG_IS_BEFORE(seq, offset, view->map->hdr.log_file_seq, + view->map->hdr.log_file_head_offset); + + /* Apply transaction to view's mapping if needed (meaning we + didn't just re-map the view to head mapping). */ + if (ctx->sync_map_update && !synced_to_map) { + if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE | + MAIL_TRANSACTION_EXPUNGE_GUID)) == 0) { + ret = mail_index_sync_record(&ctx->sync_map_ctx, + hdr, ctx->data); + } + if (ret < 0) + return -1; + } + + ctx->hidden = view_sync_is_hidden(view, seq, offset); + return 1; +} + +static bool +mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx, + struct mail_index_view_sync_rec *rec) +{ + const struct mail_transaction_header *hdr = ctx->hdr; + const void *data = ctx->data; + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_FLAG_UPDATE: { + const struct mail_transaction_flag_update *update = + CONST_PTR_OFFSET(data, ctx->data_offset); + + /* data contains mail_transaction_flag_update[] */ + for (;;) { + ctx->data_offset += sizeof(*update); + if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(update)) + break; + + /* skip internal flag changes */ + if (ctx->data_offset == ctx->hdr->size) + return FALSE; + + update = CONST_PTR_OFFSET(data, ctx->data_offset); + } + + if (update->add_flags != 0 || update->remove_flags != 0) + rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; + else + rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ; + rec->uid1 = update->uid1; + rec->uid2 = update->uid2; + break; + } + case MAIL_TRANSACTION_KEYWORD_UPDATE: { + const struct mail_transaction_keyword_update *update = data; + const uint32_t *uids; + + /* data contains mail_transaction_keyword_update header, + the keyword name and an array of { uint32_t uid1, uid2; } */ + + if (ctx->data_offset == 0) { + /* skip over the header and name */ + ctx->data_offset = sizeof(*update) + update->name_size; + if ((ctx->data_offset % 4) != 0) + ctx->data_offset += 4 - (ctx->data_offset % 4); + } + + uids = CONST_PTR_OFFSET(data, ctx->data_offset); + rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; + rec->uid1 = uids[0]; + rec->uid2 = uids[1]; + + ctx->data_offset += sizeof(uint32_t) * 2; + break; + } + case MAIL_TRANSACTION_KEYWORD_RESET: { + const struct mail_transaction_keyword_reset *reset = + CONST_PTR_OFFSET(data, ctx->data_offset); + + /* data contains mail_transaction_keyword_reset[] */ + rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; + rec->uid1 = reset->uid1; + rec->uid2 = reset->uid2; + ctx->data_offset += sizeof(*reset); + break; + } + default: + ctx->hdr = NULL; + return FALSE; + } + + rec->hidden = ctx->hidden; + return TRUE; +} + +static bool +mail_index_view_sync_next_lost(struct mail_index_view_sync_ctx *ctx, + struct mail_index_view_sync_rec *sync_rec) +{ + const struct seq_range *range; + unsigned int count; + + range = array_get(&ctx->lost_flags, &count); + if (ctx->lost_flag_idx == count) { + ctx->last_read = TRUE; + return FALSE; + } + + sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; + sync_rec->uid1 = range[ctx->lost_flag_idx].seq1; + sync_rec->uid2 = range[ctx->lost_flag_idx].seq2; + sync_rec->hidden = FALSE; + ctx->lost_flag_idx++; + return TRUE; +} + +bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx, + struct mail_index_view_sync_rec *sync_rec) +{ + int ret; + + if (ctx->log_was_lost) + return mail_index_view_sync_next_lost(ctx, sync_rec); + + do { + if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) { + ret = mail_index_view_sync_get_next_transaction(ctx); + if (ret <= 0) { + if (ret < 0) + ctx->failed = TRUE; + return FALSE; + } + + ctx->data_offset = 0; + } + } while (!mail_index_view_sync_get_rec(ctx, sync_rec)); + + return TRUE; +} + +void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx, + const ARRAY_TYPE(seq_range) **expunges_r) +{ + *expunges_r = &ctx->expunges; +} + +static void +mail_index_view_sync_clean_log_syncs(struct mail_index_view *view) +{ + const struct mail_index_view_log_sync_area *syncs; + unsigned int i, count; + + if (!array_is_created(&view->syncs_hidden)) + return; + + /* Clean up to view's tail */ + syncs = array_get(&view->syncs_hidden, &count); + for (i = 0; i < count; i++) { + if ((syncs[i].log_file_offset + + syncs[i].length > view->log_file_expunge_offset && + syncs[i].log_file_seq == view->log_file_expunge_seq) || + syncs[i].log_file_seq > view->log_file_expunge_seq) + break; + } + if (i > 0) + array_delete(&view->syncs_hidden, 0, i); +} + +int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx, + bool *delayed_expunges_r) +{ + struct mail_index_view_sync_ctx *ctx = *_ctx; + struct mail_index_view *view = ctx->view; + int ret = ctx->failed ? -1 : 0; + + i_assert(view->syncing); + + *_ctx = NULL; + *delayed_expunges_r = ctx->skipped_expunges; + + if ((!ctx->last_read || view->inconsistent) && + (ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) { + /* we didn't sync everything */ + view->inconsistent = TRUE; + ret = -1; + } + if (ctx->sync_map_ctx.modseq_ctx != NULL) + mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx); + + if (ctx->sync_new_map != NULL) { + mail_index_unmap(&view->map); + view->map = ctx->sync_new_map; + } else if (ctx->sync_map_update) { + /* log offsets have no meaning in views. make sure they're not + tried to be used wrong by setting them to zero. */ + view->map->hdr.log_file_seq = 0; + view->map->hdr.log_file_head_offset = 0; + view->map->hdr.log_file_tail_offset = 0; + } + + i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count); + + if (!ctx->skipped_expunges) { + view->log_file_expunge_seq = view->log_file_head_seq; + view->log_file_expunge_offset = view->log_file_head_offset; + } + + if (ctx->sync_map_ctx.view != NULL) + mail_index_sync_map_deinit(&ctx->sync_map_ctx); + mail_index_view_sync_clean_log_syncs(ctx->view); + +#ifdef DEBUG + mail_index_map_check(view->map); +#endif + + /* set log view to empty range so unneeded memory gets freed */ + mail_transaction_log_view_clear(view->log_view, + view->log_file_expunge_seq); + + if (array_is_created(&ctx->expunges)) + array_free(&ctx->expunges); + if (array_is_created(&ctx->lost_flags)) + array_free(&ctx->lost_flags); + + view->highest_modseq = mail_index_map_modseq_get_highest(view->map); + view->syncing = FALSE; + i_free(ctx); + return ret; +} + +void mail_index_view_add_hidden_transaction(struct mail_index_view *view, + uint32_t log_file_seq, + uoff_t log_file_offset, + unsigned int length) +{ + struct mail_index_view_log_sync_area *area; + + if (!array_is_created(&view->syncs_hidden)) + i_array_init(&view->syncs_hidden, 32); + + area = array_append_space(&view->syncs_hidden); + area->log_file_seq = log_file_seq; + area->log_file_offset = log_file_offset; + area->length = length; +} diff --git a/src/lib-index/mail-index-view.c b/src/lib-index/mail-index-view.c new file mode 100644 index 0000000..0850de1 --- /dev/null +++ b/src/lib-index/mail-index-view.c @@ -0,0 +1,651 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "llist.h" +#include "mail-index-view-private.h" +#include "mail-transaction-log.h" + +#undef mail_index_view_clone +#undef mail_index_view_dup_private + +struct mail_index_view * +mail_index_view_dup_private(const struct mail_index_view *src, + const char *source_filename, + unsigned int source_linenum) +{ + struct mail_index_view *view; + struct mail_index_map *map; + + view = i_new(struct mail_index_view, 1); + mail_index_view_clone(view, src, source_filename, source_linenum); + + map = mail_index_map_clone(view->map); + mail_index_unmap(&view->map); + view->map = map; + return view; +} + +void mail_index_view_clone(struct mail_index_view *dest, + const struct mail_index_view *src, + const char *source_filename, + unsigned int source_linenum) +{ + i_zero(dest); + dest->refcount = 1; + dest->v = src->v; + dest->index = src->index; + if (src->log_view != NULL) { + dest->log_view = + mail_transaction_log_view_open(src->index->log); + } + + dest->indexid = src->indexid; + dest->inconsistency_id = src->inconsistency_id; + dest->map = src->map; + if (dest->map != NULL) + dest->map->refcount++; + + dest->log_file_expunge_seq = src->log_file_expunge_seq; + dest->log_file_expunge_offset = src->log_file_expunge_offset; + dest->log_file_head_seq = src->log_file_head_seq; + dest->log_file_head_offset = src->log_file_head_offset; + + i_array_init(&dest->module_contexts, + I_MIN(5, mail_index_module_register.id)); + + dest->source_filename = source_filename; + dest->source_linenum = source_linenum; + + DLLIST_PREPEND(&dest->index->views, dest); +} + +void mail_index_view_ref(struct mail_index_view *view) +{ + view->refcount++; +} + +static void view_close(struct mail_index_view *view) +{ + i_assert(view->refcount == 0); + i_assert(view->index->views != NULL); + + DLLIST_REMOVE(&view->index->views, view); + + mail_transaction_log_view_close(&view->log_view); + + if (array_is_created(&view->syncs_hidden)) + array_free(&view->syncs_hidden); + mail_index_unmap(&view->map); + if (array_is_created(&view->map_refs)) { + mail_index_view_unref_maps(view); + array_free(&view->map_refs); + } + array_free(&view->module_contexts); + i_free(view); +} + +bool mail_index_view_is_inconsistent(struct mail_index_view *view) +{ + if (view->index->indexid != view->indexid || + view->index->inconsistency_id != view->inconsistency_id) + view->inconsistent = TRUE; + return view->inconsistent; +} + +struct mail_index *mail_index_view_get_index(struct mail_index_view *view) +{ + return view->index; +} + +bool mail_index_view_have_transactions(struct mail_index_view *view) +{ + return view->transactions_list != NULL; +} + +static void mail_index_view_ref_map(struct mail_index_view *view, + struct mail_index_map *map) +{ + struct mail_index_map *const *maps; + unsigned int i, count; + + if (array_is_created(&view->map_refs)) { + maps = array_get(&view->map_refs, &count); + + /* if map is already referenced, do nothing */ + for (i = 0; i < count; i++) { + if (maps[i] == map) + return; + } + } else { + i_array_init(&view->map_refs, 4); + } + + /* reference the given mapping. the reference is dropped when the view + is synchronized or closed. */ + map->refcount++; + array_push_back(&view->map_refs, &map); +} + +void mail_index_view_unref_maps(struct mail_index_view *view) +{ + struct mail_index_map **maps; + unsigned int i, count; + + if (!array_is_created(&view->map_refs)) + return; + + maps = array_get_modifiable(&view->map_refs, &count); + for (i = 0; i < count; i++) + mail_index_unmap(&maps[i]); + + array_clear(&view->map_refs); +} + +static uint32_t view_get_messages_count(struct mail_index_view *view) +{ + return view->map->hdr.messages_count; +} + +static const struct mail_index_header * +view_get_header(struct mail_index_view *view) +{ + return &view->map->hdr; +} + +static const struct mail_index_record * +view_lookup_full(struct mail_index_view *view, uint32_t seq, + struct mail_index_map **map_r, bool *expunged_r) +{ + static struct mail_index_record broken_rec; + struct mail_index_map *map; + const struct mail_index_record *rec, *head_rec; + + i_assert(seq > 0 && seq <= mail_index_view_get_messages_count(view)); + + /* look up the record */ + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + if (unlikely(rec->uid == 0)) { + if (!view->inconsistent) { + mail_index_set_error(view->index, + "Corrupted Index file %s: Record [%u].uid=0", + view->index->filepath, seq); + (void)mail_index_fsck(view->index); + view->inconsistent = TRUE; + } + + /* we'll need to return something so the caller doesn't crash */ + *map_r = view->map; + if (expunged_r != NULL) + *expunged_r = TRUE; + return &broken_rec; + } + if (view->map == view->index->map) { + /* view's mapping is latest. we can use it directly. */ + *map_r = view->map; + if (expunged_r != NULL) + *expunged_r = FALSE; + return rec; + } + + /* look up the record from head mapping. it may contain some changes. + + start looking up from the same sequence as in the old view. + if there are no expunges, it's there. otherwise it's somewhere + before (since records can't be inserted). + + usually there are only a few expunges, so just going downwards from + our initial sequence position is probably faster than binary + search. */ + if (seq > view->index->map->hdr.messages_count) + seq = view->index->map->hdr.messages_count; + if (seq == 0) { + /* everything is expunged from head. use the old record. */ + *map_r = view->map; + if (expunged_r != NULL) + *expunged_r = TRUE; + return rec; + } + + map = view->index->map; + do { + head_rec = MAIL_INDEX_REC_AT_SEQ(map, seq); + if (head_rec->uid <= rec->uid) + break; + } while (--seq > 0); + + if (head_rec->uid == rec->uid) { + /* found it. use it. reference the index mapping so that the + returned record doesn't get invalidated after next sync. */ + mail_index_view_ref_map(view, view->index->map); + *map_r = view->index->map; + if (expunged_r != NULL) + *expunged_r = FALSE; + return head_rec; + } else { + /* expunged from head. use the old record. */ + *map_r = view->map; + if (expunged_r != NULL) + *expunged_r = TRUE; + return rec; + } +} + +static void view_lookup_uid(struct mail_index_view *view, uint32_t seq, + uint32_t *uid_r) +{ + i_assert(seq > 0 && seq <= mail_index_view_get_messages_count(view)); + + *uid_r = MAIL_INDEX_REC_AT_SEQ(view->map, seq)->uid; +} + +static void view_lookup_seq_range(struct mail_index_view *view, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r) +{ + mail_index_map_lookup_seq_range(view->map, first_uid, last_uid, + first_seq_r, last_seq_r); +} + +static void view_lookup_first(struct mail_index_view *view, + enum mail_flags flags, uint8_t flags_mask, + uint32_t *seq_r) +{ +#define LOW_UPDATE(x) \ + STMT_START { if ((x) > low_uid) low_uid = x; } STMT_END + const struct mail_index_header *hdr = &view->map->hdr; + const struct mail_index_record *rec; + uint32_t seq, seq2, low_uid = 1; + + *seq_r = 0; + + if ((flags_mask & MAIL_SEEN) != 0 && (flags & MAIL_SEEN) == 0) + LOW_UPDATE(hdr->first_unseen_uid_lowwater); + if ((flags_mask & MAIL_DELETED) != 0 && (flags & MAIL_DELETED) != 0) + LOW_UPDATE(hdr->first_deleted_uid_lowwater); + + if (low_uid == 1) + seq = 1; + else { + if (!mail_index_lookup_seq_range(view, low_uid, hdr->next_uid, + &seq, &seq2)) + return; + } + + i_assert(hdr->messages_count <= view->map->rec_map->records_count); + for (; seq <= hdr->messages_count; seq++) { + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + if ((rec->flags & flags_mask) == (uint8_t)flags) { + *seq_r = seq; + break; + } + } +} + +static void +mail_index_data_lookup_keywords(struct mail_index_map *map, + const unsigned char *data, + ARRAY_TYPE(keyword_indexes) *keyword_idx) +{ + const unsigned int *keyword_idx_map; + unsigned int i, j, keyword_count, index_idx; + uint32_t idx, hdr_size; + uint16_t record_size, record_align; + + array_clear(keyword_idx); + if (data == NULL) { + /* no keywords at all in index */ + return; + } + (void)mail_index_ext_get_size(map, map->index->keywords_ext_id, + &hdr_size, &record_size, + &record_align); + + /* keyword_idx_map[] contains file => index keyword mapping */ + if (!array_is_created(&map->keyword_idx_map)) + return; + + keyword_idx_map = array_get(&map->keyword_idx_map, &keyword_count); + for (i = 0; i < record_size; i++) { + /* first do the quick check to see if there's keywords at all */ + if (data[i] == 0) + continue; + + idx = i * CHAR_BIT; + for (j = 0; j < CHAR_BIT; j++, idx++) { + if ((data[i] & (1 << j)) == 0) + continue; + + if (idx >= keyword_count) { + /* extra bits set in keyword bytes. + shouldn't happen, but just ignore. */ + break; + } + + index_idx = keyword_idx_map[idx]; + array_push_back(keyword_idx, &index_idx); + } + } +} + +static void view_lookup_keywords(struct mail_index_view *view, uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keyword_idx) +{ + struct mail_index_map *map; + const void *data; + + mail_index_lookup_ext_full(view, seq, view->index->keywords_ext_id, + &map, &data, NULL); + mail_index_data_lookup_keywords(map, data, keyword_idx); +} + +static const void * +view_map_lookup_ext_full(struct mail_index_map *map, + const struct mail_index_record *rec, uint32_t ext_id) +{ + const struct mail_index_ext *ext; + uint32_t idx; + + if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) + return NULL; + + ext = array_idx(&map->extensions, idx); + return ext->record_offset == 0 ? NULL : + CONST_PTR_OFFSET(rec, ext->record_offset); +} + +static void +view_lookup_ext_full(struct mail_index_view *view, uint32_t seq, + uint32_t ext_id, struct mail_index_map **map_r, + const void **data_r, bool *expunged_r) +{ + const struct mail_index_record *rec; + + rec = view->v.lookup_full(view, seq, map_r, expunged_r); + *data_r = view_map_lookup_ext_full(*map_r, rec, ext_id); +} + +static void view_get_header_ext(struct mail_index_view *view, + struct mail_index_map *map, uint32_t ext_id, + const void **data_r, size_t *data_size_r) +{ + const struct mail_index_ext *ext; + uint32_t idx; + + if (map == NULL) { + /* no mapping given, use head mapping */ + map = view->index->map; + } + + if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) { + /* extension doesn't exist in this index file */ + *data_r = NULL; + *data_size_r = 0; + return; + } + + ext = array_idx(&map->extensions, idx); + *data_r = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset); + *data_size_r = ext->hdr_size; +} + +static bool view_ext_get_reset_id(struct mail_index_view *view ATTR_UNUSED, + struct mail_index_map *map, + uint32_t ext_id, uint32_t *reset_id_r) +{ + const struct mail_index_ext *ext; + uint32_t idx; + + if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) + return FALSE; + + ext = array_idx(&map->extensions, idx); + *reset_id_r = ext->reset_id; + return TRUE; +} + +void mail_index_view_close(struct mail_index_view **_view) +{ + struct mail_index_view *view = *_view; + + *_view = NULL; + if (--view->refcount > 0) + return; + + i_assert(view->transactions_list == NULL); + + view->v.close(view); +} + +uint32_t mail_index_view_get_messages_count(struct mail_index_view *view) +{ + return view->v.get_messages_count(view); +} + +const struct mail_index_header * +mail_index_get_header(struct mail_index_view *view) +{ + return view->v.get_header(view); +} + +const struct mail_index_record * +mail_index_lookup(struct mail_index_view *view, uint32_t seq) +{ + struct mail_index_map *map; + + return mail_index_lookup_full(view, seq, &map, NULL); +} + +const struct mail_index_record * +mail_index_lookup_full(struct mail_index_view *view, uint32_t seq, + struct mail_index_map **map_r, bool *expunged_r) +{ + return view->v.lookup_full(view, seq, map_r, expunged_r); +} + +bool mail_index_is_expunged(struct mail_index_view *view, uint32_t seq) +{ + struct mail_index_map *map; + bool expunged; + + (void)view->v.lookup_full(view, seq, &map, &expunged); + return expunged; +} + +void mail_index_map_lookup_keywords(struct mail_index_map *map, uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keyword_idx) +{ + const struct mail_index_ext *ext; + const struct mail_index_record *rec; + const void *data; + uint32_t idx; + + if (!mail_index_map_get_ext_idx(map, map->index->keywords_ext_id, &idx)) + data = NULL; + else { + rec = MAIL_INDEX_REC_AT_SEQ(map, seq); + ext = array_idx(&map->extensions, idx); + data = ext->record_offset == 0 ? NULL : + CONST_PTR_OFFSET(rec, ext->record_offset); + } + mail_index_data_lookup_keywords(map, data, keyword_idx); +} + +void mail_index_lookup_keywords(struct mail_index_view *view, uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keyword_idx) +{ + view->v.lookup_keywords(view, seq, keyword_idx); +} + +void mail_index_lookup_view_flags(struct mail_index_view *view, uint32_t seq, + enum mail_flags *flags_r, + ARRAY_TYPE(keyword_indexes) *keyword_idx) +{ + const struct mail_index_record *rec; + const unsigned char *keyword_data; + + i_assert(seq > 0 && seq <= mail_index_view_get_messages_count(view)); + + rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); + *flags_r = rec->flags; + + keyword_data = view_map_lookup_ext_full(view->map, rec, + view->index->keywords_ext_id); + mail_index_data_lookup_keywords(view->map, keyword_data, keyword_idx); +} + +void mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq, + uint32_t *uid_r) +{ + view->v.lookup_uid(view, seq, uid_r); +} + +bool mail_index_lookup_seq_range(struct mail_index_view *view, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r) +{ + view->v.lookup_seq_range(view, first_uid, last_uid, + first_seq_r, last_seq_r); + return *first_seq_r != 0; +} + +bool mail_index_lookup_seq(struct mail_index_view *view, + uint32_t uid, uint32_t *seq_r) +{ + view->v.lookup_seq_range(view, uid, uid, seq_r, seq_r); + return *seq_r != 0; +} + +void mail_index_lookup_first(struct mail_index_view *view, + enum mail_flags flags, uint8_t flags_mask, + uint32_t *seq_r) +{ + view->v.lookup_first(view, flags, flags_mask, seq_r); +} + +void mail_index_lookup_ext(struct mail_index_view *view, uint32_t seq, + uint32_t ext_id, const void **data_r, + bool *expunged_r) +{ + struct mail_index_map *map; + + mail_index_lookup_ext_full(view, seq, ext_id, &map, data_r, expunged_r); +} + +void mail_index_lookup_ext_full(struct mail_index_view *view, uint32_t seq, + uint32_t ext_id, struct mail_index_map **map_r, + const void **data_r, bool *expunged_r) +{ + view->v.lookup_ext_full(view, seq, ext_id, map_r, data_r, expunged_r); +} + +void mail_index_get_header_ext(struct mail_index_view *view, uint32_t ext_id, + const void **data_r, size_t *data_size_r) +{ + view->v.get_header_ext(view, NULL, ext_id, data_r, data_size_r); +} + +void mail_index_map_get_header_ext(struct mail_index_view *view, + struct mail_index_map *map, uint32_t ext_id, + const void **data_r, size_t *data_size_r) +{ + view->v.get_header_ext(view, map, ext_id, data_r, data_size_r); +} + +bool mail_index_ext_get_reset_id(struct mail_index_view *view, + struct mail_index_map *map, + uint32_t ext_id, uint32_t *reset_id_r) +{ + return view->v.ext_get_reset_id(view, map, ext_id, reset_id_r); +} + +void mail_index_ext_get_size(struct mail_index_map *map, uint32_t ext_id, + uint32_t *hdr_size_r, uint16_t *record_size_r, + uint16_t *record_align_r) +{ + const struct mail_index_ext *ext; + uint32_t idx; + + i_assert(map != NULL); + + if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) { + /* extension doesn't exist in this index file */ + *hdr_size_r = 0; + *record_size_r = 0; + *record_align_r = 0; + return; + } + + ext = array_idx(&map->extensions, idx); + *hdr_size_r = ext->hdr_size; + *record_size_r = ext->record_size; + *record_align_r = ext->record_align; +} + +static struct mail_index_view_vfuncs view_vfuncs = { + view_close, + view_get_messages_count, + view_get_header, + view_lookup_full, + view_lookup_uid, + view_lookup_seq_range, + view_lookup_first, + view_lookup_keywords, + view_lookup_ext_full, + view_get_header_ext, + view_ext_get_reset_id +}; + +struct mail_index_view * +mail_index_view_open_with_map(struct mail_index *index, + struct mail_index_map *map) +{ + struct mail_index_view *view; + + view = i_new(struct mail_index_view, 1); + view->refcount = 1; + view->v = view_vfuncs; + view->index = index; + view->log_view = mail_transaction_log_view_open(index->log); + + view->indexid = index->indexid; + view->inconsistency_id = index->inconsistency_id; + view->map = map; + view->map->refcount++; + + view->log_file_expunge_seq = view->log_file_head_seq = + view->map->hdr.log_file_seq; + view->log_file_expunge_offset = view->log_file_head_offset = + view->map->hdr.log_file_head_offset; + + i_array_init(&view->module_contexts, + I_MIN(5, mail_index_module_register.id)); + DLLIST_PREPEND(&index->views, view); + return view; +} + +#undef mail_index_view_open +struct mail_index_view * +mail_index_view_open(struct mail_index *index, + const char *source_filename, unsigned int source_linenum) +{ + struct mail_index_view *view; + + view = mail_index_view_open_with_map(index, index->map); + /* these can be used to debug mail_index_view_close() leaks */ + view->source_filename = source_filename; + view->source_linenum = source_linenum; + return view; +} + +const struct mail_index_ext * +mail_index_view_get_ext(struct mail_index_view *view, uint32_t ext_id) +{ + uint32_t idx; + + if (!mail_index_map_get_ext_idx(view->map, ext_id, &idx)) + return NULL; + + return array_idx(&view->map->extensions, idx); +} diff --git a/src/lib-index/mail-index-write.c b/src/lib-index/mail-index-write.c new file mode 100644 index 0000000..689cb9c --- /dev/null +++ b/src/lib-index/mail-index-write.c @@ -0,0 +1,215 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "nfs-workarounds.h" +#include "read-full.h" +#include "write-full.h" +#include "ostream.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" + +#include <stdio.h> + +#define MAIL_INDEX_MIN_UPDATE_SIZE 1024 +/* if we're updating >= count-n messages, recreate the index */ +#define MAIL_INDEX_MAX_OVERWRITE_NEG_SEQ_COUNT 10 + +static int mail_index_create_backup(struct mail_index *index) +{ + const char *backup_path, *tmp_backup_path; + int ret; + + if (index->fd != -1) { + /* we very much want to avoid creating a backup file that + hasn't been written to disk yet */ + if (fdatasync(index->fd) < 0) { + mail_index_set_error(index, "fdatasync(%s) failed: %m", + index->filepath); + return -1; + } + } + + backup_path = t_strconcat(index->filepath, ".backup", NULL); + tmp_backup_path = t_strconcat(backup_path, ".tmp", NULL); + ret = link(index->filepath, tmp_backup_path); + if (ret < 0 && errno == EEXIST) { + if (unlink(tmp_backup_path) < 0 && errno != ENOENT) { + mail_index_set_error(index, "unlink(%s) failed: %m", + tmp_backup_path); + return -1; + } + ret = link(index->filepath, tmp_backup_path); + } + if (ret < 0) { + if (errno == ENOENT) { + /* no dovecot.index file, ignore */ + return 0; + } + mail_index_set_error(index, "link(%s, %s) failed: %m", + index->filepath, tmp_backup_path); + return -1; + } + + if (rename(tmp_backup_path, backup_path) < 0) { + mail_index_set_error(index, "rename(%s, %s) failed: %m", + tmp_backup_path, backup_path); + return -1; + } + return 0; +} + +static int mail_index_recreate(struct mail_index *index) +{ + struct mail_index_map *map = index->map; + struct ostream *output; + unsigned int base_size; + const char *path; + int ret = 0, fd; + + i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); + i_assert(map->hdr.indexid == index->indexid); + i_assert((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) == 0); + i_assert(index->indexid != 0); + + fd = mail_index_create_tmp_file(index, index->filepath, &path); + if (fd == -1) + return -1; + + output = o_stream_create_fd_file(fd, 0, FALSE); + o_stream_cork(output); + + struct mail_index_header hdr = map->hdr; + /* Write tail_offset the same as head_offset. This function must not + be called unless it's safe to do this. See the explanations in + mail_index_sync_commit(). */ + hdr.log_file_tail_offset = hdr.log_file_head_offset; + + base_size = I_MIN(hdr.base_header_size, sizeof(hdr)); + o_stream_nsend(output, &hdr, base_size); + o_stream_nsend(output, MAIL_INDEX_MAP_HDR_OFFSET(map, base_size), + hdr.header_size - base_size); + o_stream_nsend(output, map->rec_map->records, + map->rec_map->records_count * hdr.record_size); + if (o_stream_finish(output) < 0) { + mail_index_file_set_syscall_error(index, path, "write()"); + ret = -1; + } + o_stream_destroy(&output); + + if (ret == 0 && index->set.fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync(fd) < 0) { + mail_index_file_set_syscall_error(index, path, + "fdatasync()"); + ret = -1; + } + } + + if (close(fd) < 0) { + mail_index_file_set_syscall_error(index, path, "close()"); + ret = -1; + } + + if ((index->flags & MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS) != 0) + (void)mail_index_create_backup(index); + + if (ret == 0 && rename(path, index->filepath) < 0) { + mail_index_set_error(index, "rename(%s, %s) failed: %m", + path, index->filepath); + ret = -1; + } + + if (ret < 0) + i_unlink(path); + return ret; +} + +static bool mail_index_should_recreate(struct mail_index *index) +{ + struct stat st1, st2; + + if (nfs_safe_stat(index->filepath, &st1) < 0) { + if (errno != ENOENT) { + mail_index_set_syscall_error(index, "stat()"); + return FALSE; + } else if (index->fd == -1) { + /* main index hasn't been created yet */ + return TRUE; + } else { + /* mailbox was just deleted? don't log an error */ + return FALSE; + } + } + if (index->fd == -1) { + /* main index was just created by another process */ + return FALSE; + } + if (fstat(index->fd, &st2) < 0) { + if (!ESTALE_FSTAT(errno)) + mail_index_set_syscall_error(index, "fstat()"); + return FALSE; + } + if (st1.st_ino != st2.st_ino || + !CMP_DEV_T(st1.st_dev, st2.st_dev)) { + /* Index has already been recreated since we last read it. + We can't trust our decisions about whether to recreate it. */ + return FALSE; + } + return TRUE; +} + +void mail_index_write(struct mail_index *index, bool want_rotate, + const char *reason) +{ + struct mail_index_header *hdr = &index->map->hdr; + bool rotated = FALSE; + + i_assert(index->log_sync_locked); + + if (index->readonly) + return; + + /* rotate the .log before writing index, so the index will point to + the latest log. Note that it's the caller's responsibility to make + sure that the .log can be safely rotated (i.e. everything has been + synced). */ + if (want_rotate) { + if (mail_transaction_log_rotate(index->log, FALSE) == 0) { + struct mail_transaction_log_file *file = + index->log->head; + /* Log rotation refreshes the index, which may cause the + map to change. Because we're locked, it's not + supposed to happen and will likely lead to an + assert-crash below, but we still need to make sure + we're using the latest map to do the checks. */ + hdr = &index->map->hdr; + i_assert(file->hdr.prev_file_seq == hdr->log_file_seq); + i_assert(file->hdr.prev_file_offset == hdr->log_file_head_offset); + hdr->log_file_seq = file->hdr.file_seq; + hdr->log_file_head_offset = + hdr->log_file_tail_offset = file->hdr.hdr_size; + /* Assume .log.2 was created successfully. If it + wasn't, it just causes an extra stat() and gets + fixed later on. */ + hdr->log2_rotate_time = ioloop_time; + rotated = TRUE; + } + } + + if (MAIL_INDEX_IS_IN_MEMORY(index)) + ; + else if (!rotated && !mail_index_should_recreate(index)) { + /* make sure we don't keep getting back in here */ + index->reopen_main_index = TRUE; + } else { + if (mail_index_recreate(index) < 0) { + (void)mail_index_move_to_memory(index); + return; + } + event_set_name(index->event, "mail_index_recreated"); + e_debug(index->event, "Recreated %s (file_seq=%u) because: %s", + index->filepath, hdr->log_file_seq, reason); + } + + index->main_index_hdr_log_file_seq = hdr->log_file_seq; + index->main_index_hdr_log_file_tail_offset = hdr->log_file_tail_offset; +} diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c new file mode 100644 index 0000000..8f89309 --- /dev/null +++ b/src/lib-index/mail-index.c @@ -0,0 +1,1110 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "eacces-error.h" +#include "hash.h" +#include "str-sanitize.h" +#include "mmap-util.h" +#include "nfs-workarounds.h" +#include "read-full.h" +#include "write-full.h" +#include "mail-index-alloc-cache.h" +#include "mail-index-private.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log-private.h" +#include "mail-transaction-log-view-private.h" +#include "mail-cache.h" + +#include <stdio.h> +#include <stddef.h> +#include <time.h> +#include <sys/stat.h> +#include <ctype.h> + +struct mail_index_module_register mail_index_module_register = { 0 }; + +struct event_category event_category_mail_index = { + .name = "mail-index", +}; + +static void mail_index_close_nonopened(struct mail_index *index); + +static const struct mail_index_optimization_settings default_optimization_set = { + .index = { + .rewrite_min_log_bytes = 8 * 1024, + .rewrite_max_log_bytes = 128 * 1024, + }, + .log = { + .min_size = 32 * 1024, + .max_size = 1024 * 1024, + .min_age_secs = 5 * 60, + .log2_max_age_secs = 3600 * 24 * 2, + }, + .cache = { + .unaccessed_field_drop_secs = 3600 * 24 * 30, + .record_max_size = 64 * 1024, + .max_size = 1024 * 1024 * 1024, + .purge_min_size = 32 * 1024, + .purge_delete_percentage = 20, + .purge_continued_percentage = 200, + .purge_header_continue_count = 4, + }, +}; + +struct mail_index *mail_index_alloc(struct event *parent_event, + const char *dir, const char *prefix) +{ + struct mail_index *index; + + index = i_new(struct mail_index, 1); + index->dir = i_strdup(dir); + index->prefix = i_strdup(prefix); + index->fd = -1; + index->event = event_create(parent_event); + event_add_category(index->event, &event_category_mail_index); + + index->extension_pool = + pool_alloconly_create(MEMPOOL_GROWING"index extension", 1024); + p_array_init(&index->extensions, index->extension_pool, 5); + i_array_init(&index->module_contexts, + I_MIN(5, mail_index_module_register.id)); + + index->set.mode = 0600; + index->set.gid = (gid_t)-1; + index->set.lock_method = FILE_LOCK_METHOD_FCNTL; + index->set.max_lock_timeout_secs = UINT_MAX; + index->optimization_set = default_optimization_set; + + index->keywords_ext_id = + mail_index_ext_register(index, MAIL_INDEX_EXT_KEYWORDS, + 128, 2, 1); + index->keywords_pool = pool_alloconly_create("keywords", 512); + i_array_init(&index->keywords, 16); + hash_table_create(&index->keywords_hash, index->keywords_pool, 0, + strcase_hash, strcasecmp); + index->log = mail_transaction_log_alloc(index); + mail_index_modseq_init(index); + return index; +} + +void mail_index_free(struct mail_index **_index) +{ + struct mail_index *index = *_index; + + *_index = NULL; + + i_assert(index->open_count == 0); + + mail_transaction_log_free(&index->log); + hash_table_destroy(&index->keywords_hash); + pool_unref(&index->extension_pool); + pool_unref(&index->keywords_pool); + + array_free(&index->keywords); + array_free(&index->module_contexts); + + event_unref(&index->event); + i_free(index->set.cache_dir); + i_free(index->set.ext_hdr_init_data); + i_free(index->set.gid_origin); + i_free(index->last_error.text); + i_free(index->dir); + i_free(index->prefix); + i_free(index->need_recreate); + i_free(index); +} + +void mail_index_set_cache_dir(struct mail_index *index, const char *dir) +{ + i_free(index->set.cache_dir); + index->set.cache_dir = i_strdup(dir); +} + +void mail_index_set_fsync_mode(struct mail_index *index, + enum fsync_mode mode, + enum mail_index_fsync_mask mask) +{ + index->set.fsync_mode = mode; + index->set.fsync_mask = mask; +} + +bool mail_index_use_existing_permissions(struct mail_index *index) +{ + struct stat st; + + if (MAIL_INDEX_IS_IN_MEMORY(index)) + return FALSE; + + if (stat(index->dir, &st) < 0) { + if (errno != ENOENT) + e_error(index->event, "stat(%s) failed: %m", index->dir); + return FALSE; + } + + index->set.mode = st.st_mode & 0666; + if (S_ISDIR(st.st_mode) && (st.st_mode & S_ISGID) != 0) { + /* directory's GID is used automatically for new files */ + index->set.gid = (gid_t)-1; + } else if ((st.st_mode & 0070) >> 3 == (st.st_mode & 0007)) { + /* group has same permissions as world, so don't bother + changing it */ + index->set.gid = (gid_t)-1; + } else if (getegid() == st.st_gid) { + /* using our own gid, no need to change it */ + index->set.gid = (gid_t)-1; + } else { + index->set.gid = st.st_gid; + } + + i_free(index->set.gid_origin); + if (index->set.gid != (gid_t)-1) + index->set.gid_origin = i_strdup("preserved existing GID"); + return TRUE; +} + +void mail_index_set_permissions(struct mail_index *index, + mode_t mode, gid_t gid, const char *gid_origin) +{ + index->set.mode = mode & 0666; + index->set.gid = gid; + + i_free(index->set.gid_origin); + index->set.gid_origin = i_strdup(gid_origin); +} + +void mail_index_set_lock_method(struct mail_index *index, + enum file_lock_method lock_method, + unsigned int max_timeout_secs) +{ + index->set.lock_method = lock_method; + index->set.max_lock_timeout_secs = max_timeout_secs; +} + +void mail_index_set_optimization_settings(struct mail_index *index, + const struct mail_index_optimization_settings *set) +{ + struct mail_index_optimization_settings *dest = + &index->optimization_set; + + /* index */ + if (set->index.rewrite_min_log_bytes != 0) + dest->index.rewrite_min_log_bytes = set->index.rewrite_min_log_bytes; + if (set->index.rewrite_max_log_bytes != 0) + dest->index.rewrite_max_log_bytes = set->index.rewrite_max_log_bytes; + + /* log */ + if (set->log.min_size != 0) + dest->log.min_size = set->log.min_size; + if (set->log.max_size != 0) + dest->log.max_size = set->log.max_size; + if (set->log.min_age_secs != 0) + dest->log.min_age_secs = set->log.min_age_secs; + if (set->log.log2_max_age_secs != 0) + dest->log.log2_max_age_secs = set->log.log2_max_age_secs; + + /* cache */ + if (set->cache.unaccessed_field_drop_secs != 0) + dest->cache.unaccessed_field_drop_secs = + set->cache.unaccessed_field_drop_secs; + if (set->cache.max_size != 0) + dest->cache.max_size = set->cache.max_size; + if (set->cache.purge_min_size != 0) + dest->cache.purge_min_size = set->cache.purge_min_size; + if (set->cache.purge_delete_percentage != 0) + dest->cache.purge_delete_percentage = + set->cache.purge_delete_percentage; + if (set->cache.purge_continued_percentage != 0) + dest->cache.purge_continued_percentage = + set->cache.purge_continued_percentage; + if (set->cache.purge_header_continue_count != 0) + dest->cache.purge_header_continue_count = + set->cache.purge_header_continue_count; + if (set->cache.record_max_size != 0) + dest->cache.record_max_size = set->cache.record_max_size; +} + +void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id, + const void *data, size_t size) +{ + const struct mail_index_registered_ext *rext; + + i_assert(index->set.ext_hdr_init_data == NULL || + index->set.ext_hdr_init_id == ext_id); + + rext = array_idx(&index->extensions, ext_id); + i_assert(rext->hdr_size == size); + + index->set.ext_hdr_init_id = ext_id; + i_free(index->set.ext_hdr_init_data); + index->set.ext_hdr_init_data = i_malloc(size); + memcpy(index->set.ext_hdr_init_data, data, size); +} + +bool mail_index_ext_name_is_valid(const char *name) +{ + size_t i; + + for (i = 0; name[i] != '\0'; i++) { + if (!i_isalnum(name[i]) && name[i] != '-' && name[i] != '_' && + name[i] != ' ') + return FALSE; + + } + return i == 0 || i < MAIL_INDEX_EXT_NAME_MAX_LENGTH; +} + +uint32_t mail_index_ext_register(struct mail_index *index, const char *name, + uint32_t default_hdr_size, + uint16_t default_record_size, + uint16_t default_record_align) +{ + struct mail_index_registered_ext rext; + uint32_t ext_id; + + if (!mail_index_ext_name_is_valid(name)) + i_panic("mail_index_ext_register(%s): Invalid name", name); + + if (default_record_size != 0 && default_record_align == 0) { + i_panic("mail_index_ext_register(%s): " + "Invalid record alignment", name); + } + + if (mail_index_ext_lookup(index, name, &ext_id)) + return ext_id; + + i_zero(&rext); + rext.name = p_strdup(index->extension_pool, name); + rext.index_idx = array_count(&index->extensions); + rext.hdr_size = default_hdr_size; + rext.record_size = default_record_size; + rext.record_align = default_record_align; + + array_push_back(&index->extensions, &rext); + return rext.index_idx; +} + +void mail_index_ext_register_resize_defaults(struct mail_index *index, + uint32_t ext_id, + uint32_t default_hdr_size, + uint16_t default_record_size, + uint16_t default_record_align) +{ + struct mail_index_registered_ext *rext; + + rext = array_idx_modifiable(&index->extensions, ext_id); + rext->hdr_size = default_hdr_size; + rext->record_size = default_record_size; + rext->record_align = default_record_align; +} + +bool mail_index_ext_lookup(struct mail_index *index, const char *name, + uint32_t *ext_id_r) +{ + const struct mail_index_registered_ext *extensions; + unsigned int i, count; + + extensions = array_get(&index->extensions, &count); + for (i = 0; i < count; i++) { + if (strcmp(extensions[i].name, name) == 0) { + *ext_id_r = i; + return TRUE; + } + } + + *ext_id_r = (uint32_t)-1; + return FALSE; +} + +void mail_index_register_expunge_handler(struct mail_index *index, + uint32_t ext_id, + mail_index_expunge_handler_t *cb) +{ + struct mail_index_registered_ext *rext; + + rext = array_idx_modifiable(&index->extensions, ext_id); + i_assert(rext->expunge_handler == NULL || rext->expunge_handler == cb); + + rext->expunge_handler = cb; +} + +void mail_index_unregister_expunge_handler(struct mail_index *index, + uint32_t ext_id) +{ + struct mail_index_registered_ext *rext; + + rext = array_idx_modifiable(&index->extensions, ext_id); + i_assert(rext->expunge_handler != NULL); + + rext->expunge_handler = NULL; +} + +bool mail_index_keyword_lookup(struct mail_index *index, + const char *keyword, unsigned int *idx_r) +{ + char *key; + void *value; + + /* keywords_hash keeps a name => index mapping of keywords. + Keywords are never removed from it, so the index values are valid + for the lifetime of the mail_index. */ + if (hash_table_lookup_full(index->keywords_hash, keyword, + &key, &value)) { + *idx_r = POINTER_CAST_TO(value, unsigned int); + return TRUE; + } + + *idx_r = UINT_MAX; + return FALSE; +} + +void mail_index_keyword_lookup_or_create(struct mail_index *index, + const char *keyword, + unsigned int *idx_r) +{ + char *keyword_dup; + + i_assert(*keyword != '\0'); + + if (mail_index_keyword_lookup(index, keyword, idx_r)) + return; + + keyword = keyword_dup = p_strdup(index->keywords_pool, keyword); + *idx_r = array_count(&index->keywords); + + hash_table_insert(index->keywords_hash, keyword_dup, + POINTER_CAST(*idx_r)); + array_push_back(&index->keywords, &keyword); + + /* keep the array NULL-terminated, but the NULL itself invisible */ + array_append_zero(&index->keywords); + array_pop_back(&index->keywords); +} + +const ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index) +{ + return &index->keywords; +} + +struct mail_keywords * +mail_index_keywords_create(struct mail_index *index, + const char *const keywords[]) +{ + struct mail_keywords *k; + unsigned int src, dest, i, count; + + count = str_array_length(keywords); + if (count == 0) { + k = i_new(struct mail_keywords, 1); + k->index = index; + k->refcount = 1; + return k; + } + + /* @UNSAFE */ + k = i_malloc(MALLOC_ADD(sizeof(struct mail_keywords), + MALLOC_MULTIPLY(sizeof(k->idx[0]), count))); + k->index = index; + k->refcount = 1; + + /* look up the keywords from index. they're never removed from there + so we can permanently store indexes to them. */ + for (src = dest = 0; src < count; src++) { + mail_index_keyword_lookup_or_create(index, keywords[src], + &k->idx[dest]); + /* ignore if this is a duplicate */ + for (i = 0; i < src; i++) { + if (k->idx[i] == k->idx[dest]) + break; + } + if (i == src) + dest++; + } + k->count = dest; + return k; +} + +struct mail_keywords * +mail_index_keywords_create_from_indexes(struct mail_index *index, + const ARRAY_TYPE(keyword_indexes) + *keyword_indexes) +{ + struct mail_keywords *k; + const unsigned int *indexes; + unsigned int src, dest, i, count; + + indexes = array_get(keyword_indexes, &count); + if (count == 0) { + k = i_new(struct mail_keywords, 1); + k->index = index; + k->refcount = 1; + return k; + } + + /* @UNSAFE */ + k = i_malloc(MALLOC_ADD(sizeof(struct mail_keywords), + MALLOC_MULTIPLY(sizeof(k->idx[0]), count))); + k->index = index; + k->refcount = 1; + + /* copy but skip duplicates */ + for (src = dest = 0; src < count; src++) { + for (i = 0; i < src; i++) { + if (k->idx[i] == indexes[src]) + break; + } + if (i == src) + k->idx[dest++] = indexes[src]; + } + k->count = dest; + return k; +} + +void mail_index_keywords_ref(struct mail_keywords *keywords) +{ + keywords->refcount++; +} + +void mail_index_keywords_unref(struct mail_keywords **_keywords) +{ + struct mail_keywords *keywords = *_keywords; + + i_assert(keywords->refcount > 0); + + *_keywords = NULL; + if (--keywords->refcount == 0) + i_free(keywords); +} + +int mail_index_try_open_only(struct mail_index *index) +{ + i_assert(index->fd == -1); + i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); + + /* Note that our caller must close index->fd by itself. */ + if (index->readonly) + errno = EACCES; + else { + index->fd = nfs_safe_open(index->filepath, O_RDWR); + index->readonly = FALSE; + } + + if (index->fd == -1 && errno == EACCES) { + index->fd = open(index->filepath, O_RDONLY); + index->readonly = TRUE; + } + + if (index->fd == -1) { + if (errno != ENOENT) { + mail_index_set_syscall_error(index, "open()"); + return -1; + } + + /* have to create it */ + return 0; + } + return 1; +} + +static int +mail_index_try_open(struct mail_index *index) +{ + int ret; + + i_assert(index->fd == -1); + + if (MAIL_INDEX_IS_IN_MEMORY(index)) + return 0; + + ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD); + if (ret == 0 && !index->readonly) { + /* it's corrupted - recreate it */ + if (index->fd != -1) { + if (close(index->fd) < 0) + mail_index_set_syscall_error(index, "close()"); + index->fd = -1; + } + } + return ret; +} + +int mail_index_create_tmp_file(struct mail_index *index, + const char *path_prefix, const char **path_r) +{ + mode_t old_mask; + const char *path; + int fd; + + i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); + + path = *path_r = t_strconcat(path_prefix, ".tmp", NULL); + old_mask = umask(0); + fd = open(path, O_RDWR|O_CREAT|O_EXCL, index->set.mode); + umask(old_mask); + if (fd == -1 && errno == EEXIST) { + /* stale temp file. unlink and recreate rather than overwriting, + just to make sure locking problems won't cause corruption */ + if (i_unlink(path) < 0) + return -1; + old_mask = umask(0); + fd = open(path, O_RDWR|O_CREAT|O_EXCL, index->set.mode); + umask(old_mask); + } + if (fd == -1) { + mail_index_file_set_syscall_error(index, path, "creat()"); + return -1; + } + + mail_index_fchown(index, fd, path); + return fd; +} + +static const char *mail_index_get_cache_path(struct mail_index *index) +{ + const char *dir; + + if (index->set.cache_dir != NULL) + dir = index->set.cache_dir; + else if (index->dir != NULL) + dir = index->dir; + else + return NULL; + return t_strconcat(dir, "/", index->prefix, + MAIL_CACHE_FILE_SUFFIX, NULL); +} + +static int mail_index_open_files(struct mail_index *index, + enum mail_index_open_flags flags) +{ + int ret; + + ret = mail_transaction_log_open(index->log); + if (ret == 0) { + if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0) + return 0; + + /* if dovecot.index exists, read it first so that we can get + the correct indexid and log sequence */ + (void)mail_index_try_open(index); + + if (index->indexid == 0) { + /* Create a new indexid for us. If we're opening index + into memory, index->map doesn't exist yet. */ + index->indexid = ioloop_time; + index->initial_create = TRUE; + if (index->map != NULL) + index->map->hdr.indexid = index->indexid; + } + + ret = mail_transaction_log_create(index->log, FALSE); + if (index->map != NULL) { + /* log creation could have changed it if someone else + just created it. */ + index->map->hdr.indexid = index->indexid; + } + index->initial_create = FALSE; + } + if (ret >= 0) { + ret = index->map != NULL ? 1 : mail_index_try_open(index); + if (ret == 0 && !index->readonly) { + /* corrupted */ + mail_transaction_log_close(index->log); + ret = mail_transaction_log_create(index->log, TRUE); + if (ret == 0) { + if (index->map != NULL) + mail_index_unmap(&index->map); + index->map = mail_index_map_alloc(index); + } + } + } + if (ret < 0) { + /* open/create failed, fallback to in-memory indexes */ + if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0) + return -1; + + if (mail_index_move_to_memory(index) < 0) + return -1; + } + + if (index->cache == NULL) { + const char *path = mail_index_get_cache_path(index); + index->cache = mail_cache_open_or_create_path(index, path); + } + return 1; +} + +static int +mail_index_open_opened(struct mail_index *index, + enum mail_index_open_flags flags) +{ + int ret; + + i_assert(index->map != NULL); + + if ((index->map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { + /* index was marked corrupted. we'll probably need to + recreate the files. */ + mail_index_unmap(&index->map); + mail_index_close_file(index); + mail_transaction_log_close(index->log); + if ((ret = mail_index_open_files(index, flags)) <= 0) + return ret; + } + + index->open_count++; + return 1; +} + +int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags) +{ + int ret; + + if (index->open_count > 0) { + if ((ret = mail_index_open_opened(index, flags)) <= 0) { + /* doesn't exist and create flag not used */ + } + return ret; + } + + index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ? + i_strdup("(in-memory index)") : + i_strconcat(index->dir, "/", index->prefix, NULL); + + mail_index_reset_error(index); + index->readonly = FALSE; + index->log_sync_locked = FALSE; + index->flags = flags; + index->readonly = (flags & MAIL_INDEX_OPEN_FLAG_READONLY) != 0; + if ((flags & MAIL_INDEX_OPEN_FLAG_DEBUG) != 0) + event_set_forced_debug(index->event, TRUE); + else + event_unset_forced_debug(index->event); + + if ((flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0 && + index->set.fsync_mode != FSYNC_MODE_ALWAYS) + i_fatal("nfs flush requires mail_fsync=always"); + if ((flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0 && + (flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) + i_fatal("nfs flush requires mmap_disable=yes"); + + /* NOTE: increase open_count only after mail_index_open_files(). + it's used elsewhere to check if we're doing an initial opening + of the index files */ + if ((ret = mail_index_open_files(index, flags)) <= 0) { + /* doesn't exist and create flag not used */ + mail_index_close_nonopened(index); + return ret; + } + + index->open_count++; + + if (index->log->head == NULL) { + mail_index_close(index); + mail_index_set_error(index, "Index is corrupted " + "(log->view->head == NULL)"); + return -1; + } + + i_assert(index->map != NULL); + mail_index_alloc_cache_index_opened(index); + return 1; +} + +int mail_index_open_or_create(struct mail_index *index, + enum mail_index_open_flags flags) +{ + int ret; + + flags |= MAIL_INDEX_OPEN_FLAG_CREATE; + ret = mail_index_open(index, flags); + i_assert(ret != 0); + return ret < 0 ? -1 : 0; +} + +void mail_index_close_file(struct mail_index *index) +{ + if (index->fd != -1) { + if (close(index->fd) < 0) + mail_index_set_syscall_error(index, "close()"); + index->fd = -1; + } +} + +static void mail_index_close_nonopened(struct mail_index *index) +{ + i_assert(!index->syncing); + + if (index->views != NULL) { + i_panic("Leaked view for index %s: Opened in %s:%u", + index->filepath, index->views->source_filename, + index->views->source_linenum); + } + i_assert(index->views == NULL); + + if (index->map != NULL) + mail_index_unmap(&index->map); + + mail_index_close_file(index); + mail_transaction_log_close(index->log); + if (index->cache != NULL) + mail_cache_free(&index->cache); + + i_free_and_null(index->filepath); + + index->indexid = 0; +} + +void mail_index_close(struct mail_index *index) +{ + i_assert(index->open_count > 0); + + mail_index_alloc_cache_index_closing(index); + if (--index->open_count == 0) + mail_index_close_nonopened(index); +} + +int mail_index_unlink(struct mail_index *index) +{ + const char *path; + int last_errno = 0; + + if (MAIL_INDEX_IS_IN_MEMORY(index) || index->readonly) + return 0; + + /* main index */ + if (unlink(index->filepath) < 0 && errno != ENOENT) + last_errno = errno; + + /* logs */ + path = t_strconcat(index->filepath, MAIL_TRANSACTION_LOG_SUFFIX, NULL); + if (unlink(path) < 0 && errno != ENOENT) + last_errno = errno; + + path = t_strconcat(index->filepath, + MAIL_TRANSACTION_LOG_SUFFIX".2", NULL); + if (unlink(path) < 0 && errno != ENOENT) + last_errno = errno; + + /* cache */ + path = t_strconcat(index->filepath, MAIL_CACHE_FILE_SUFFIX, NULL); + if (unlink(path) < 0 && errno != ENOENT) + last_errno = errno; + + if (last_errno == 0) + return 0; + else { + errno = last_errno; + return -1; + } +} + +int mail_index_reopen_if_changed(struct mail_index *index, bool *reopened_r, + const char **reason_r) +{ + struct stat st1, st2; + int ret; + + *reopened_r = FALSE; + + if (MAIL_INDEX_IS_IN_MEMORY(index)) { + *reason_r = "in-memory index"; + return 0; + } + + if (index->fd == -1) + goto final; + + if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) + nfs_flush_file_handle_cache(index->filepath); + if (nfs_safe_stat(index->filepath, &st2) < 0) { + if (errno == ENOENT) { + *reason_r = "index not found via stat()"; + return 0; + } + mail_index_set_syscall_error(index, "stat()"); + return -1; + } + + if (fstat(index->fd, &st1) < 0) { + if (!ESTALE_FSTAT(errno)) { + mail_index_set_syscall_error(index, "fstat()"); + return -1; + } + /* deleted/recreated, reopen */ + *reason_r = "index is stale"; + } else if (st1.st_ino == st2.st_ino && + CMP_DEV_T(st1.st_dev, st2.st_dev)) { + /* the same file */ + *reason_r = "index unchanged"; + return 1; + } else { + *reason_r = "index inode changed"; + } + + /* new file, new locks. the old fd can keep its locks, they don't + matter anymore as no-one's going to modify the file. */ + mail_index_close_file(index); + +final: + if ((ret = mail_index_try_open_only(index)) == 0) + *reason_r = "index not found via open()"; + else if (ret > 0) { + *reason_r = "index opened"; + *reopened_r = TRUE; + } + return ret; +} + +int mail_index_refresh(struct mail_index *index) +{ + int ret; + + ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD); + return ret <= 0 ? -1 : 0; +} + +struct mail_cache *mail_index_get_cache(struct mail_index *index) +{ + return index->cache; +} + +void mail_index_set_error(struct mail_index *index, const char *fmt, ...) +{ + va_list va; + + i_free(index->last_error.text); + + if (fmt == NULL) + index->last_error.text = NULL; + else { + va_start(va, fmt); + index->last_error.text = i_strdup_vprintf(fmt, va); + va_end(va); + + e_error(index->event, "%s", index->last_error.text); + } +} + +void mail_index_set_error_nolog(struct mail_index *index, const char *str) +{ + i_assert(str != NULL); + + char *old_error = index->last_error.text; + index->last_error.text = i_strdup(str); + i_free(old_error); +} + +bool mail_index_is_in_memory(struct mail_index *index) +{ + return MAIL_INDEX_IS_IN_MEMORY(index); +} + +static void mail_index_set_as_in_memory(struct mail_index *index) +{ + i_free_and_null(index->dir); + + i_free(index->filepath); + index->filepath = i_strdup("(in-memory index)"); +} + +int mail_index_move_to_memory(struct mail_index *index) +{ + struct mail_index_map *map; + + if (MAIL_INDEX_IS_IN_MEMORY(index)) + return index->map == NULL ? -1 : 0; + + if ((index->flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) + return -1; + + if (index->map == NULL) { + /* index was never even opened. just mark it as being in + memory and let the caller re-open the index. */ + i_assert(index->fd == -1); + mail_index_set_as_in_memory(index); + return -1; + } + + /* move index map to memory */ + if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) { + map = mail_index_map_clone(index->map); + mail_index_unmap(&index->map); + index->map = map; + } + + if (index->log != NULL) { + /* move transaction log to memory */ + if (mail_transaction_log_move_to_memory(index->log) < 0) + return -1; + } + + if (index->fd != -1) { + if (close(index->fd) < 0) + mail_index_set_syscall_error(index, "close()"); + index->fd = -1; + } + mail_index_set_as_in_memory(index); + return 0; +} + +void mail_index_mark_corrupted(struct mail_index *index) +{ + index->indexid = 0; + + index->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED; + if (!index->readonly) { + if (unlink(index->filepath) < 0 && + errno != ENOENT && errno != ESTALE) + mail_index_set_syscall_error(index, "unlink()"); + (void)mail_transaction_log_unlink(index->log); + } +} + +bool mail_index_is_deleted(struct mail_index *index) +{ + return index->index_delete_requested || index->index_deleted; +} + +int mail_index_get_modification_time(struct mail_index *index, time_t *mtime_r) +{ + struct stat st; + const char *path; + + *mtime_r = 0; + if (MAIL_INDEX_IS_IN_MEMORY(index)) { + /* this function doesn't make sense for in-memory indexes */ + return 0; + } + + /* index may not be open, so index->filepath may be NULL */ + path = t_strconcat(index->dir, "/", index->prefix, + MAIL_TRANSACTION_LOG_SUFFIX, NULL); + if (stat(path, &st) < 0) { + if (errno == ENOENT) { + /* .log is always supposed to exist - don't bother + trying to stat(dovecot.index) */ + return 0; + } + mail_index_file_set_syscall_error(index, path, "stat()"); + return -1; + } + *mtime_r = st.st_mtime; + return 0; +} + +void mail_index_fchown(struct mail_index *index, int fd, const char *path) +{ + mode_t mode; + + if (index->set.gid == (gid_t)-1) { + /* no gid changing */ + return; + } else if (fchown(fd, (uid_t)-1, index->set.gid) == 0) { + /* success */ + return; + } if ((index->set.mode & 0060) >> 3 == (index->set.mode & 0006)) { + /* group and world permissions are the same, so group doesn't + really matter. ignore silently. */ + return; + } + if (errno != EPERM) + mail_index_file_set_syscall_error(index, path, "fchown()"); + else { + mail_index_set_error(index, "%s", + eperm_error_get_chgrp("fchown", path, index->set.gid, + index->set.gid_origin)); + } + + /* continue, but change permissions so that only the common + subset of group and world is used. this makes sure no one + gets any extra permissions. */ + mode = ((index->set.mode & 0060) >> 3) & (index->set.mode & 0006); + mode |= (mode << 3) | (index->set.mode & 0600); + if (fchmod(fd, mode) < 0) + mail_index_file_set_syscall_error(index, path, "fchmod()"); +} + +int mail_index_lock_sync(struct mail_index *index, const char *lock_reason) +{ + uint32_t file_seq; + uoff_t file_offset; + + return mail_transaction_log_sync_lock(index->log, lock_reason, + &file_seq, &file_offset); +} + +void mail_index_unlock(struct mail_index *index, const char *long_lock_reason) +{ + mail_transaction_log_sync_unlock(index->log, long_lock_reason); +} + +bool mail_index_is_locked(struct mail_index *index) +{ + return index->log_sync_locked; +} + +void mail_index_set_syscall_error(struct mail_index *index, + const char *function) +{ + mail_index_file_set_syscall_error(index, index->filepath, function); +} + +void mail_index_file_set_syscall_error(struct mail_index *index, + const char *filepath, + const char *function) +{ + const char *errstr; + + i_assert(filepath != NULL); + i_assert(function != NULL); + + if (errno == ENOENT) { + struct stat st; + int old_errno = errno; + i_assert(index->log->filepath != NULL); + if (nfs_safe_stat(index->log->filepath, &st) < 0 && + errno == ENOENT) { + /* the index log has gone away */ + index->index_deleted = TRUE; + errno = old_errno; + return; + } + errno = old_errno; + } + + if (ENOSPACE(errno)) { + index->last_error.nodiskspace = TRUE; + if ((index->flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) == 0) + return; + } + + if (errno == EACCES) { + function = t_strcut(function, '('); + if (strcmp(function, "creat") == 0 || + str_begins(function, "file_dotlock_")) + errstr = eacces_error_get_creating(function, filepath); + else + errstr = eacces_error_get(function, filepath); + mail_index_set_error(index, "%s", errstr); + } else { + const char *suffix = errno != EFBIG ? "" : + " (process was started with ulimit -f limit)"; + mail_index_set_error(index, "%s failed with file %s: " + "%m%s", function, filepath, suffix); + } +} + +const char *mail_index_get_error_message(struct mail_index *index) +{ + return index->last_error.text; +} + +void mail_index_reset_error(struct mail_index *index) +{ + i_free(index->last_error.text); + i_zero(&index->last_error); +} diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h new file mode 100644 index 0000000..c1947cf --- /dev/null +++ b/src/lib-index/mail-index.h @@ -0,0 +1,817 @@ +#ifndef MAIL_INDEX_H +#define MAIL_INDEX_H + +#include "file-lock.h" +#include "fsync-mode.h" +#include "guid.h" +#include "mail-types.h" +#include "seq-range-array.h" + +#define MAIL_INDEX_MAJOR_VERSION 7 +#define MAIL_INDEX_MINOR_VERSION 3 + +#define MAIL_INDEX_HEADER_MIN_SIZE 120 + +/* Log a warning when transaction log has been locked for this many seconds. + This lock is held also between mail_index_sync_begin()..commit(). */ +#define MAIL_TRANSACTION_LOG_LOCK_WARN_SECS 30 + +enum mail_index_open_flags { + /* Create index if it doesn't exist */ + MAIL_INDEX_OPEN_FLAG_CREATE = 0x01, + /* Don't try to mmap() index files */ + MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE = 0x04, + /* Rely on O_EXCL when creating dotlocks */ + MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL = 0x10, + /* Flush NFS attr/data/write cache when necessary */ + MAIL_INDEX_OPEN_FLAG_NFS_FLUSH = 0x40, + /* Open the index read-only */ + MAIL_INDEX_OPEN_FLAG_READONLY = 0x80, + /* Create backups of dovecot.index files once in a while */ + MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS = 0x100, + /* If we run out of disk space, fail modifications instead of moving + indexes to memory. */ + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200, + /* We're only going to save new messages to the index. + Avoid unnecessary reads. */ + MAIL_INDEX_OPEN_FLAG_SAVEONLY = 0x400, + /* Enable debug logging */ + MAIL_INDEX_OPEN_FLAG_DEBUG = 0x800, + /* MAIL_INDEX_MAIL_FLAG_DIRTY can be used as a backend-specific flag. + All special handling of the flag is disabled by this. */ + MAIL_INDEX_OPEN_FLAG_NO_DIRTY = 0x1000, +}; + +enum mail_index_header_compat_flags { + /* All fields in these index files are in little-endian format. + If the current CPU endianess doesn't match this, the indexes can't + be used. There is currently no support to translate endianess. */ + MAIL_INDEX_COMPAT_LITTLE_ENDIAN = 0x01 +}; + +enum mail_index_header_flag { + /* mail_index_mark_corrupted() was just called by this process. + Reopen or recreate it. This flag is never actually written to + disk. */ + MAIL_INDEX_HDR_FLAG_CORRUPTED = 0x0001, + /* There are messages with MAIL_INDEX_MAIL_FLAG_DIRTY flag. */ + MAIL_INDEX_HDR_FLAG_HAVE_DIRTY = 0x0002, + /* Index has been fsck'd. The caller may want to resync the index + to make sure it's valid and drop this flag. */ + MAIL_INDEX_HDR_FLAG_FSCKD = 0x0004, +}; + +enum mail_index_mail_flags { + /* This flag used to contain MAIL_RECENT flag, but is always zero + with the current index file format. */ + MAIL_INDEX_MAIL_FLAG_UNUSED = 0x20, + /* For private use by backend. Replacing flags doesn't change this. */ + MAIL_INDEX_MAIL_FLAG_BACKEND = 0x40, + /* Message flags haven't been written to backend. If + MAIL_INDEX_OPEN_FLAG_NO_DIRTY is set, this is treated as a + backend-specific flag with no special internal handling. */ + MAIL_INDEX_MAIL_FLAG_DIRTY = 0x80, + + /* Force updating this message's modseq via a flag update record. + Note that this flag isn't saved to disk. */ + MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ = 0x100 +}; + +#define MAIL_INDEX_FLAGS_MASK \ + (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_DELETED | MAIL_SEEN | MAIL_DRAFT) + +struct mail_index_header { + /* Major version is increased only when you can't have backwards + compatibility. If the field doesn't match MAIL_INDEX_MAJOR_VERSION, + don't even try to read it. */ + uint8_t major_version; + /* Minor version is increased when the file format changes in a + backwards compatible way. If the field is smaller than + MAIL_INDEX_MINOR_VERSION, upgrade the file format and update the + minor_version field as well. If minor_version is higher than + MAIL_INDEX_MINOR_VERSION, leave it as it is. It likely means that a + new Dovecot version is currently being upgraded to, but the file was + still accessed by an old version. */ + uint8_t minor_version; + + /* sizeof(struct mail_index_header) when creating a new index. If the + header is smaller, fill the missing fields with 0. If the header is + larger, preserve the size and unknown fields. */ + uint16_t base_header_size; + uint32_t header_size; /* base + extended header size */ + /* sizeof(struct mail_index_record) + extensions */ + uint32_t record_size; + + uint8_t compat_flags; /* enum mail_index_header_compat_flags */ + uint8_t unused[3]; + + /* Unique index file ID. Initialized with the current UNIX timestamp. + This is used to make sure that the main index, transaction log and + cache file are all part of the same index. */ + uint32_t indexid; + uint32_t flags; /* enum mail_index_header_flag */ + + /* IMAP UIDVALIDITY. Initially can be 0, but must be non-0 after the + first mailbox sync. The UIDVALIDITY shouldn't normally change after + the mailbox is created. */ + uint32_t uid_validity; + /* UID for the next saved message (must not be lower than this). This + value can only increase. */ + uint32_t next_uid; + + /* Number of messages in the index */ + uint32_t messages_count; + uint32_t unused_old_recent_messages_count; + /* Number of messages with MAIL_SEEN flag */ + uint32_t seen_messages_count; + /* Number of messages with MAIL_DELETED flag */ + uint32_t deleted_messages_count; + + /* The specified UID and all mails after it have MAIL_RECENT flag */ + uint32_t first_recent_uid; + /* There are no UIDs lower than this without MAIL_SEEN flag. There are + no guarantees whether this UID has MAIL_SEEN flag, or whether the it + even exists. Used to optimize finding the first unseen message. */ + uint32_t first_unseen_uid_lowwater; + /* Similarly to above, used to optimize finding the first deleted + message. */ + uint32_t first_deleted_uid_lowwater; + + /* The index is synced up to this log_file_seq and + log_file_head_offset. However, non-external transaction records + between tail_offset..head_offset haven't been synced to the + mailbox yet. For example there may be pending expunges or flag + changes, which will be synced on the next mail_index_sync_*() + calls. */ + uint32_t log_file_seq; + uint32_t log_file_tail_offset; + uint32_t log_file_head_offset; + + uint32_t unused_old_sync_size_part1; + /* Timestamp of when .log was rotated into .log.2. This can be used to + optimize checking when it's time to unlink it without stat()ing it. + 0 = unknown, -1 = .log.2 doesn't exists. */ + uint32_t log2_rotate_time; + /* Timestamp when the mailbox backend-specific code last checked + whether there are old temporary files (left by crashes) that should + be deleted. 0 = unknown. */ + uint32_t last_temp_file_scan; + + /* UNIX timestamp to the beginning of the day (in server's local + timezone) when new messages were last added to the index file. */ + uint32_t day_stamp; + /* These fields are updated when day_stamp < today. The [0..6] are + first moved to [1..7], then [0] is set to the first appended UID. So + they contain the first UID of the day for last 8 days when messages + were appended. + + These are used by cache purging to decide when to drop + MAIL_CACHE_DECISION_TEMP fields. */ + uint32_t day_first_uid[8]; +}; + +#define MAIL_INDEX_RECORD_MIN_SIZE (sizeof(uint32_t) + sizeof(uint8_t)) +struct mail_index_record { + uint32_t uid; + uint8_t flags; /* enum mail_flags | enum mail_index_mail_flags */ +}; + +struct mail_keywords { + struct mail_index *index; + unsigned int count; + int refcount; + + /* variable sized list of keyword indexes */ + unsigned int idx[FLEXIBLE_ARRAY_MEMBER]; +}; + +enum mail_index_transaction_flags { + /* If transaction is marked as hidden, the changes are marked with + hidden=TRUE when the view is synchronized. */ + MAIL_INDEX_TRANSACTION_FLAG_HIDE = 0x01, + /* External transactions describe changes to mailbox that have already + happened. */ + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL = 0x02, + /* Don't add flag updates unless they actually change something. + This is reliable only when syncing, otherwise someone else might + have already committed a transaction that had changed the flags. */ + MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES = 0x04, + /* fsync() this transaction (unless fsyncs are disabled) */ + MAIL_INDEX_TRANSACTION_FLAG_FSYNC = 0x08, + /* Sync transaction describes changes to mailbox that already happened + to another mailbox with whom we're syncing with (dsync) */ + MAIL_INDEX_TRANSACTION_FLAG_SYNC = 0x10 +}; + +enum mail_index_sync_type { + MAIL_INDEX_SYNC_TYPE_EXPUNGE = 0x02, + MAIL_INDEX_SYNC_TYPE_FLAGS = 0x04, + MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD = 0x08, + MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE = 0x10 +}; + +enum mail_index_fsync_mask { + MAIL_INDEX_FSYNC_MASK_APPENDS = 0x01, + MAIL_INDEX_FSYNC_MASK_EXPUNGES = 0x02, + MAIL_INDEX_FSYNC_MASK_FLAGS = 0x04, + MAIL_INDEX_FSYNC_MASK_KEYWORDS = 0x08 +}; + +enum mail_index_sync_flags { + /* Resync all dirty messages' flags. */ + MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY = 0x01, + /* Drop recent flags from all messages */ + MAIL_INDEX_SYNC_FLAG_DROP_RECENT = 0x02, + /* Create the transaction with AVOID_FLAG_UPDATES flag */ + MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES = 0x04, + /* If there are no new transactions and nothing else to do, + return 0 in mail_index_sync_begin() */ + MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES = 0x08, + /* Create the transaction with FSYNC flag */ + MAIL_INDEX_SYNC_FLAG_FSYNC = 0x10, + /* If we see "delete index" request transaction, finish it. + This flag also allows committing more changes to a deleted index. */ + MAIL_INDEX_SYNC_FLAG_DELETING_INDEX = 0x20, + /* Same as MAIL_INDEX_SYNC_FLAG_DELETING_INDEX, but finish index + deletion only once and fail the rest (= avoid race conditions when + multiple processes try to mark the index deleted) */ + MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX = 0x40, + /* Update header's tail_offset to head_offset, even if it's the only + thing we do and there's no strict need for it. */ + MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET = 0x80 +}; + +enum mail_index_view_sync_flags { + /* Don't sync expunges */ + MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES = 0x01, + /* Make sure view isn't inconsistent after syncing. This also means + that you don't care about view_sync_next()'s output, so it won't + return anything. */ + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT = 0x02, + /* Indicate that this is a secondary view, this can be used to indicate + that inconsistencies can be expected and if found should be fixed + by fully syncing. */ + MAIL_INDEX_VIEW_SYNC_FLAG_2ND_INDEX = 0x04, +}; + +struct mail_index_sync_rec { + uint32_t uid1, uid2; + enum mail_index_sync_type type; + + /* MAIL_INDEX_SYNC_TYPE_FLAGS: */ + uint8_t add_flags; + uint8_t remove_flags; + + /* MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD, .._REMOVE: */ + unsigned int keyword_idx; + + /* MAIL_INDEX_SYNC_TYPE_EXPUNGE: */ + guid_128_t guid_128; +}; + +enum mail_index_view_sync_type { + /* Flags or keywords changed */ + MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS = 0x01, + MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ = 0x02 +}; + +struct mail_index_view_sync_rec { + uint32_t uid1, uid2; + enum mail_index_view_sync_type type; + + /* TRUE if this was a hidden transaction. */ + bool hidden:1; +}; + +enum mail_index_transaction_change { + MAIL_INDEX_TRANSACTION_CHANGE_APPEND = BIT(0), + MAIL_INDEX_TRANSACTION_CHANGE_EXPUNGE = BIT(1), + MAIL_INDEX_TRANSACTION_CHANGE_FLAGS = BIT(2), + MAIL_INDEX_TRANSACTION_CHANGE_KEYWORDS = BIT(3), + MAIL_INDEX_TRANSACTION_CHANGE_MODSEQ = BIT(4), + MAIL_INDEX_TRANSACTION_CHANGE_ATTRIBUTE = BIT(5), + + MAIL_INDEX_TRANSACTION_CHANGE_OTHERS = BIT(30), +}; + +struct mail_index_transaction_commit_result { + /* seq/offset points to end of transaction */ + uint32_t log_file_seq; + uoff_t log_file_offset; + /* number of bytes in the written transaction. + all of it was written to the same file. */ + uoff_t commit_size; + + enum mail_index_transaction_change changes_mask; + unsigned int ignored_modseq_changes; +}; + +struct mail_index_base_optimization_settings { + /* Rewrite the index when the number of bytes that needs to be read + from the .log on refresh is between these min/max values. */ + uoff_t rewrite_min_log_bytes; + uoff_t rewrite_max_log_bytes; +}; + +struct mail_index_log_optimization_settings { + /* Rotate transaction log after it's a) min_size or larger and it was + created at least min_age_secs or b) larger than max_size. */ + uoff_t min_size; + uoff_t max_size; + unsigned int min_age_secs; + + /* Delete .log.2 when it's older than log2_stale_secs. Don't be too + eager, because older files are useful for QRESYNC and dsync. */ + unsigned int log2_max_age_secs; +}; + +struct mail_index_cache_optimization_settings { + /* Drop fields that haven't been accessed for n seconds */ + unsigned int unaccessed_field_drop_secs; + /* If cache record becomes larger than this, don't add it. */ + unsigned int record_max_size; + + /* Maximum size for the cache file. Internally the limit is 1 GB. */ + uoff_t max_size; + /* Never purge the file if it's smaller than this */ + uoff_t purge_min_size; + /* Purge the file when n% of records are deleted */ + unsigned int purge_delete_percentage; + /* Purge the file when n% of rows contain continued rows. + For example 200% means that the record has 2 continued rows, i.e. + it exists in 3 separate segments in the cache file. */ + unsigned int purge_continued_percentage; + /* Purge the file when we need to follow more than n next_offsets to + find the latest cache header. */ + unsigned int purge_header_continue_count; +}; + +struct mail_index_optimization_settings { + struct mail_index_base_optimization_settings index; + struct mail_index_log_optimization_settings log; + struct mail_index_cache_optimization_settings cache; +}; + +struct mail_index; +struct mail_index_map; +struct mail_index_view; +struct mail_index_transaction; +struct mail_index_sync_ctx; +struct mail_index_view_sync_ctx; + +struct mail_index *mail_index_alloc(struct event *parent_event, + const char *dir, const char *prefix); +void mail_index_free(struct mail_index **index); + +/* Change .cache file's directory. */ +void mail_index_set_cache_dir(struct mail_index *index, const char *dir); +/* Specify how often to do fsyncs. If mode is FSYNC_MODE_OPTIMIZED, the mask + can be used to specify which transaction types to fsync. */ +void mail_index_set_fsync_mode(struct mail_index *index, enum fsync_mode mode, + enum mail_index_fsync_mask mask); +/* Try to set the index's permissions based on its index directory. Returns + TRUE if successful (directory existed), FALSE if mail_index_set_permissions() + should be called. */ +bool mail_index_use_existing_permissions(struct mail_index *index); +void mail_index_set_permissions(struct mail_index *index, + mode_t mode, gid_t gid, const char *gid_origin); +/* Set locking method and maximum time to wait for a lock + (UINT_MAX = default). */ +void mail_index_set_lock_method(struct mail_index *index, + enum file_lock_method lock_method, + unsigned int max_timeout_secs); +/* Override the default optimization-related settings. Anything set to 0 will + use the default. */ +void mail_index_set_optimization_settings(struct mail_index *index, + const struct mail_index_optimization_settings *set); +/* When creating a new index file or reseting an existing one, add the given + extension header data immediately to it. */ +void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id, + const void *data, size_t size); + +/* Open index. Returns 1 if ok, 0 if index doesn't exist and CREATE flags + wasn't given, -1 if error. */ +int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags); +/* Open or create index. Returns 0 if ok, -1 if error. */ +int mail_index_open_or_create(struct mail_index *index, + enum mail_index_open_flags flags); +void mail_index_close(struct mail_index *index); +/* unlink() all the index files. */ +int mail_index_unlink(struct mail_index *index); + +/* Returns TRUE if index is currently in memory. */ +bool mail_index_is_in_memory(struct mail_index *index); +/* Move the index into memory. Returns 0 if ok, -1 if error occurred. */ +int mail_index_move_to_memory(struct mail_index *index); + +struct mail_cache *mail_index_get_cache(struct mail_index *index); + +/* Refresh index so mail_index_lookup*() will return latest values. Note that + immediately after this call there may already be changes, so if you need to + rely on validity of the returned values, use some external locking for it. */ +int ATTR_NOWARN_UNUSED_RESULT +mail_index_refresh(struct mail_index *index); + +/* View can be used to look into index. Sequence numbers inside view change + only when you synchronize it. The view acquires required locks + automatically, but you'll have to drop them manually. */ +struct mail_index_view * +mail_index_view_open(struct mail_index *index, + const char *source_filename, unsigned int source_linenum); +#define mail_index_view_open(index) \ + mail_index_view_open(index, __FILE__, __LINE__) +void mail_index_view_close(struct mail_index_view **view); + +/* Returns the index for given view. */ +struct mail_index *mail_index_view_get_index(struct mail_index_view *view); +/* Returns number of mails in view. */ +uint32_t mail_index_view_get_messages_count(struct mail_index_view *view); +/* Returns TRUE if we lost track of changes for some reason. */ +bool mail_index_view_is_inconsistent(struct mail_index_view *view); +/* Returns TRUE if there are open transactions open for the view. */ +bool mail_index_view_have_transactions(struct mail_index_view *view); + +/* Transaction has to be opened to be able to modify index. You can have + multiple transactions open simultaneously. Committed transactions won't + show up until you've synchronized the view. Expunges won't show up until + you've synchronized the mailbox (mail_index_sync_begin). */ +struct mail_index_transaction * +mail_index_transaction_begin(struct mail_index_view *view, + enum mail_index_transaction_flags flags); +int mail_index_transaction_commit(struct mail_index_transaction **t); +int mail_index_transaction_commit_full(struct mail_index_transaction **t, + struct mail_index_transaction_commit_result *result_r); +void mail_index_transaction_rollback(struct mail_index_transaction **t); +/* Discard all changes in the transaction. */ +void mail_index_transaction_reset(struct mail_index_transaction *t); +/* When committing transaction, drop flag/keyword updates for messages whose + mdoseq is larger than max_modseq. Save those messages' sequences to the + given array. */ +void mail_index_transaction_set_max_modseq(struct mail_index_transaction *t, + uint64_t max_modseq, + ARRAY_TYPE(seq_range) *seqs); + +/* Returns the view transaction was created for. */ +struct mail_index_view * +mail_index_transaction_get_view(struct mail_index_transaction *t); +/* Returns TRUE if the given sequence is being expunged in this transaction. */ +bool mail_index_transaction_is_expunged(struct mail_index_transaction *t, + uint32_t seq); + +/* Returns a view containing the mailbox state after changes in transaction + are applied. The view can still be used after transaction has been + committed. */ +struct mail_index_view * +mail_index_transaction_open_updated_view(struct mail_index_transaction *t); + +/* Begin synchronizing mailbox with index file. Returns 1 if ok, + 0 if MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES is set and there's nothing to + sync, -1 if error. + + mail_index_sync_next() returns all changes from previously committed + transactions which haven't yet been committed to the actual mailbox. + They're returned in ascending order and they never overlap (if we add more + sync types, then they might). You must go through all of them and update + the mailbox accordingly. + + Changes done to the returned transaction are expected to describe the + mailbox's current state. + + The returned view already contains all the changes (except expunge + requests). After applying sync records on top of backend flags they should + match flags in the view. If they don't, there have been external changes. + + Returned expunges are treated as expunge requests. They're not really + removed from the index until you mark them expunged to the returned + transaction. If it's not possible to expunge the message (e.g. permission + denied), simply don't mark them expunged. + + Returned sequence numbers describe the mailbox state at the beginning of + synchronization, ie. expunges don't affect them. */ +int mail_index_sync_begin(struct mail_index *index, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + enum mail_index_sync_flags flags); +/* Like mail_index_sync_begin(), but returns 1 if OK and if index is already + synchronized up to the given log_file_seq+offset, the synchronization isn't + started and this function returns 0. This should be done when you wish to + sync your committed transaction instead of doing a full mailbox + synchronization. */ +int mail_index_sync_begin_to(struct mail_index *index, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + struct mail_index_transaction **trans_r, + uint32_t log_file_seq, uoff_t log_file_offset, + enum mail_index_sync_flags flags); +/* Returns TRUE if it currently looks like syncing would return changes. */ +bool mail_index_sync_have_any(struct mail_index *index, + enum mail_index_sync_flags flags); +/* Returns TRUE if it currently looks like syncing would return expunges. */ +bool mail_index_sync_have_any_expunges(struct mail_index *index); +/* Returns the log file seq+offsets for the area which this sync is handling. */ +void mail_index_sync_get_offsets(struct mail_index_sync_ctx *ctx, + uint32_t *seq1_r, uoff_t *offset1_r, + uint32_t *seq2_r, uoff_t *offset2_r); +/* Returns -1 if error, 0 if sync is finished, 1 if record was filled. */ +bool mail_index_sync_next(struct mail_index_sync_ctx *ctx, + struct mail_index_sync_rec *sync_rec); +/* Returns TRUE if there's more to sync. */ +bool mail_index_sync_have_more(struct mail_index_sync_ctx *ctx); +/* Returns TRUE if sync has any expunges to handle. */ +bool mail_index_sync_has_expunges(struct mail_index_sync_ctx *ctx); +/* Reset syncing to initial state after mail_index_sync_begin(), so you can + go through all the sync records again with mail_index_sync_next(). */ +void mail_index_sync_reset(struct mail_index_sync_ctx *ctx); +/* Update result when refreshing index at the end of sync. */ +void mail_index_sync_set_commit_result(struct mail_index_sync_ctx *ctx, + struct mail_index_transaction_commit_result *result); +/* Don't log a warning even if syncing took over + MAIL_TRANSACTION_LOG_LOCK_WARN_SECS seconds. Usually this is called because + the caller itself already logged a warning about it. */ +void mail_index_sync_no_warning(struct mail_index_sync_ctx *ctx); +/* If a warning is logged because syncing took over + MAIL_TRANSACTION_LOG_LOCK_WARN_SECS seconds, log this as the reason for the + syncing. */ +void mail_index_sync_set_reason(struct mail_index_sync_ctx *ctx, + const char *reason); +/* Commit synchronization by writing all changes to mail index file. */ +int mail_index_sync_commit(struct mail_index_sync_ctx **ctx); +/* Rollback synchronization - none of the changes listed by sync_next() are + actually written to index file. */ +void mail_index_sync_rollback(struct mail_index_sync_ctx **ctx); + +/* Lock the index exclusively. This is the same locking as what happens when + syncing the index. It's not necessary to normally call this function, unless + doing something special such as rebuilding the index outside syncing. + Returns 0 on success, -1 if locking failed for any reason. */ +int mail_index_lock_sync(struct mail_index *index, const char *lock_reason); +/* Unlock the locked index. The index must have been locked previously with + mail_index_lock_sync(). If the lock had been kept for excessively long, + a warning is logged with the long_lock_reason. */ +void mail_index_unlock(struct mail_index *index, const char *long_lock_reason); +/* Returns TRUE if index is currently exclusively locked. */ +bool mail_index_is_locked(struct mail_index *index); + +/* Mark index file corrupted in memory and delete it from disk. + Invalidates all views. This should be called only for index files that can + safely be recreated without any data loss. */ +void mail_index_mark_corrupted(struct mail_index *index) ATTR_COLD; +/* Check and fix any found problems. Returns -1 if we couldn't lock for sync, + 0 if everything went ok. */ +int mail_index_fsck(struct mail_index *index) ATTR_COLD; +/* Returns TRUE if mail_index_fsck() has been called since the last + mail_index_reset_fscked() call. */ +bool mail_index_reset_fscked(struct mail_index *index); + +/* Synchronize changes in view. You have to go through all records, or view + will be marked inconsistent. Only sync_mask type records are + synchronized. */ +struct mail_index_view_sync_ctx * +mail_index_view_sync_begin(struct mail_index_view *view, + enum mail_index_view_sync_flags flags); +bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx, + struct mail_index_view_sync_rec *sync_rec); +void +mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx, + const ARRAY_TYPE(seq_range) **expunges_r); +int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **ctx, + bool *delayed_expunges_r); + +/* Returns the index header. */ +const struct mail_index_header * +mail_index_get_header(struct mail_index_view *view); +/* Returns the wanted message record. */ +const struct mail_index_record * +mail_index_lookup(struct mail_index_view *view, uint32_t seq); +const struct mail_index_record * +mail_index_lookup_full(struct mail_index_view *view, uint32_t seq, + struct mail_index_map **map_r, bool *expunged_r); +/* Returns TRUE if the given message has already been expunged from index. */ +bool mail_index_is_expunged(struct mail_index_view *view, uint32_t seq); +/* Note that returned keyword indexes aren't sorted. */ +void mail_index_lookup_keywords(struct mail_index_view *view, uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keyword_idx); +/* Return keywords from given map. */ +void mail_index_map_lookup_keywords(struct mail_index_map *map, uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keyword_idx); +/* mail_index_lookup[_keywords]() returns the latest flag changes. + This function instead attempts to return the flags and keywords done by the + last view sync. */ +void mail_index_lookup_view_flags(struct mail_index_view *view, uint32_t seq, + enum mail_flags *flags_r, + ARRAY_TYPE(keyword_indexes) *keyword_idx); +/* Returns the UID for given message. May be slightly faster than + mail_index_lookup()->uid. */ +void mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq, + uint32_t *uid_r); +/* Convert UID range to sequence range. If no UIDs are found, returns FALSE and + sequences are set to 0. Note that any of the returned sequences may have + been expunged already. */ +bool mail_index_lookup_seq_range(struct mail_index_view *view, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r); +bool mail_index_lookup_seq(struct mail_index_view *view, + uint32_t uid, uint32_t *seq_r); +/* Find first mail with (mail->flags & flags_mask) == flags. Useful mostly for + taking advantage of lowwater-fields in headers. */ +void mail_index_lookup_first(struct mail_index_view *view, + enum mail_flags flags, uint8_t flags_mask, + uint32_t *seq_r); + +/* Append a new record to index. */ +void mail_index_append(struct mail_index_transaction *t, uint32_t uid, + uint32_t *seq_r); +/* Assign new UIDs for mails with uid=0 or uid<min_allowed_uid. All the new + UIDs are >= first_new_uid, an also higher than the highest seen uid (i.e. it + doesn't try to fill UID gaps). Assumes that mailbox is locked in a way that + UIDs can be safely assigned. Returns UIDs for all assigned messages, in + their sequence order (so UIDs are not necessary ascending). */ +void mail_index_append_finish_uids_full(struct mail_index_transaction *t, + uint32_t min_allowed_uid, + uint32_t first_new_uid, + ARRAY_TYPE(seq_range) *uids_r); +/* Call mail_index_append_finish_uids_full() with first_uid used for both + min_allowed_uid and first_new_uid. */ +void mail_index_append_finish_uids(struct mail_index_transaction *t, + uint32_t first_uid, + ARRAY_TYPE(seq_range) *uids_r); +/* Expunge record from index. Note that this doesn't affect sequence numbers + until transaction is committed and mailbox is synced. */ +void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq); +/* Like mail_index_expunge(), but also write message GUID to transaction log. */ +void mail_index_expunge_guid(struct mail_index_transaction *t, uint32_t seq, + const guid_128_t guid_128); +/* Revert all changes done in this transaction to the given existing mail. */ +void mail_index_revert_changes(struct mail_index_transaction *t, uint32_t seq); +/* Update flags in index. */ +void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq, + enum modify_type modify_type, + enum mail_flags flags); +void mail_index_update_flags_range(struct mail_index_transaction *t, + uint32_t seq1, uint32_t seq2, + enum modify_type modify_type, + enum mail_flags flags); +/* Specified attribute's value was changed. This is just a notification so the + change gets assigned its own modseq and any log readers can find out about + this change. */ +void mail_index_attribute_set(struct mail_index_transaction *t, + bool pvt, const char *key, + time_t timestamp, uint32_t value_len); +/* Attribute was deleted. */ +void mail_index_attribute_unset(struct mail_index_transaction *t, + bool pvt, const char *key, time_t timestamp); +/* Update message's modseq to be at least min_modseq. */ +void mail_index_update_modseq(struct mail_index_transaction *t, uint32_t seq, + uint64_t min_modseq); +/* Update highest modseq to be at least min_modseq. */ +void mail_index_update_highest_modseq(struct mail_index_transaction *t, + uint64_t min_modseq); +/* Reset the index before committing this transaction. This is usually done + only when UIDVALIDITY changes. */ +void mail_index_reset(struct mail_index_transaction *t); +/* Remove MAIL_INDEX_HDR_FLAG_FSCKD from header if it exists. This must be + called only during syncing so that the mailbox is locked. */ +void mail_index_unset_fscked(struct mail_index_transaction *t); +/* Mark index deleted. No further changes will be possible after the + transaction has been committed. */ +void mail_index_set_deleted(struct mail_index_transaction *t); +/* Mark a deleted index as undeleted. Afterwards index can be changed again. */ +void mail_index_set_undeleted(struct mail_index_transaction *t); +/* Returns TRUE if index has been set deleted. This gets set only after + index has been opened/refreshed and the transaction has been seen. */ +bool mail_index_is_deleted(struct mail_index *index); +/* Returns the last time the index was modified. This can be called even if the + index isn't open. If the index doesn't exist, sets mtime to 0. */ +int mail_index_get_modification_time(struct mail_index *index, time_t *mtime_r); + +/* Lookup a keyword, returns TRUE if found, FALSE if not. */ +bool mail_index_keyword_lookup(struct mail_index *index, + const char *keyword, unsigned int *idx_r); +void mail_index_keyword_lookup_or_create(struct mail_index *index, + const char *keyword, + unsigned int *idx_r); +/* Return a pointer to array of NULL-terminated list of keywords. Note that + the array contents (and thus pointers inside it) may change after calling + mail_index_keywords_create() or mail_index_sync_begin(). */ +const ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index); + +/* Create a keyword list structure. */ +struct mail_keywords * +mail_index_keywords_create(struct mail_index *index, + const char *const keywords[]) ATTR_NULL(2); +struct mail_keywords * +mail_index_keywords_create_from_indexes(struct mail_index *index, + const ARRAY_TYPE(keyword_indexes) + *keyword_indexes); +void mail_index_keywords_ref(struct mail_keywords *keywords); +void mail_index_keywords_unref(struct mail_keywords **keywords); + +/* Update keywords for given message. */ +void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq, + enum modify_type modify_type, + struct mail_keywords *keywords); + +/* Update field in header. If prepend is TRUE, the header change is visible + before message syncing begins. */ +void mail_index_update_header(struct mail_index_transaction *t, + size_t offset, const void *data, size_t size, + bool prepend); + +/* Returns the full error message for last error. This message may + contain paths etc. so it shouldn't be shown to users. */ +const char *mail_index_get_error_message(struct mail_index *index); +/* Reset the error message. */ +void mail_index_reset_error(struct mail_index *index); + +/* Apply changes in MAIL_INDEX_SYNC_TYPE_FLAGS typed sync records to given + flags variable. */ +void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec, + uint8_t *flags); +/* Apply changes in MAIL_INDEX_SYNC_TYPE_KEYWORD_* typed sync records to given + keywords array. Returns TRUE If something was changed. */ +bool mail_index_sync_keywords_apply(const struct mail_index_sync_rec *sync_rec, + ARRAY_TYPE(keyword_indexes) *keywords); + +/* register index extension. name is a unique identifier for the extension. + returns unique identifier for the name. */ +uint32_t mail_index_ext_register(struct mail_index *index, const char *name, + uint32_t default_hdr_size, + uint16_t default_record_size, + uint16_t default_record_align); +/* Change an already registered extension's default sizes. */ +void mail_index_ext_register_resize_defaults(struct mail_index *index, + uint32_t ext_id, + uint32_t default_hdr_size, + uint16_t default_record_size, + uint16_t default_record_align); +/* Returns TRUE and sets ext_id_r if extension with given name is registered. */ +bool mail_index_ext_lookup(struct mail_index *index, const char *name, + uint32_t *ext_id_r); +/* Resize existing extension data. If size is grown, the new data will be + zero-filled. If size is shrinked, the data is simply dropped. */ +void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t hdr_size, uint16_t record_size, + uint16_t record_align); +/* Resize header, keeping the old record size. */ +void mail_index_ext_resize_hdr(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t hdr_size); + +/* Reset extension. Any updates for this extension which were issued before the + writer had seen this reset are discarded. reset_id is used to figure this + out, so it must be different every time. If clear_data=TRUE, records and + header is zeroed. */ +void mail_index_ext_reset(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t reset_id, bool clear_data); +/* Like mail_index_ext_reset(), but increase extension's reset_id atomically + when the transaction is being committed. If prev_reset_id doesn't match the + latest reset_id, the reset_id isn't increased and all extension changes are + ignored. */ +void mail_index_ext_reset_inc(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t prev_reset_id, bool clear_data); +/* Discard existing extension updates in this transaction and write new updates + using the given reset_id. The difference to mail_index_ext_reset() is that + this doesn't clear any existing record or header data. */ +void mail_index_ext_set_reset_id(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t reset_id); +/* Get the current reset_id for given extension. Returns TRUE if it exists. */ +bool mail_index_ext_get_reset_id(struct mail_index_view *view, + struct mail_index_map *map, + uint32_t ext_id, uint32_t *reset_id_r); + +/* Returns extension header. */ +void mail_index_get_header_ext(struct mail_index_view *view, uint32_t ext_id, + const void **data_r, size_t *data_size_r); +void mail_index_map_get_header_ext(struct mail_index_view *view, + struct mail_index_map *map, uint32_t ext_id, + const void **data_r, size_t *data_size_r); +/* Returns the wanted extension record for given message. If it doesn't exist, + *data_r is set to NULL. expunged_r is TRUE if the message has already been + expunged from the index. */ +void mail_index_lookup_ext(struct mail_index_view *view, uint32_t seq, + uint32_t ext_id, const void **data_r, + bool *expunged_r); +void mail_index_lookup_ext_full(struct mail_index_view *view, uint32_t seq, + uint32_t ext_id, struct mail_index_map **map_r, + const void **data_r, bool *expunged_r); +/* Get current extension sizes. Returns 1 if ok, 0 if extension doesn't exist + in view. Any of the _r parameters may be NULL. */ +void mail_index_ext_get_size(struct mail_index_map *map, uint32_t ext_id, + uint32_t *hdr_size_r, uint16_t *record_size_r, + uint16_t *record_align_r); +/* Update extension header field. */ +void mail_index_update_header_ext(struct mail_index_transaction *t, + uint32_t ext_id, size_t offset, + const void *data, size_t size); +/* Update extension record. If old_data_r is non-NULL and the record extension + was already updated in this transaction, it's set to contain the data it's + now overwriting. */ +void mail_index_update_ext(struct mail_index_transaction *t, uint32_t seq, + uint32_t ext_id, const void *data, void *old_data) + ATTR_NULL(5); +/* Increase/decrease number in extension atomically. Returns the sum of the + diffs for this seq. */ +int mail_index_atomic_inc_ext(struct mail_index_transaction *t, + uint32_t seq, uint32_t ext_id, int diff); + +#endif diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c new file mode 100644 index 0000000..662f72a --- /dev/null +++ b/src/lib-index/mail-transaction-log-append.c @@ -0,0 +1,256 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "write-full.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" + +void mail_transaction_log_append_add(struct mail_transaction_log_append_ctx *ctx, + enum mail_transaction_type type, + const void *data, size_t size) +{ + struct mail_transaction_header hdr; + + i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0); + i_assert((size % 4) == 0); + + if (size == 0) + return; + + i_zero(&hdr); + hdr.type = type | ctx->trans_flags; + if (type == MAIL_TRANSACTION_EXPUNGE || + type == MAIL_TRANSACTION_EXPUNGE_GUID) + hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT; + if (type == MAIL_TRANSACTION_BOUNDARY) + hdr.type |= MAIL_TRANSACTION_EXTERNAL; + hdr.size = sizeof(hdr) + size; + hdr.size = mail_index_uint32_to_offset(hdr.size); + + buffer_append(ctx->output, &hdr, sizeof(hdr)); + buffer_append(ctx->output, data, size); + + mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq, + MAIL_TRANSACTION_LOG_HDR_VERSION(&ctx->log->head->hdr)); + ctx->transaction_count++; +} + +static int +log_buffer_move_to_memory(struct mail_transaction_log_append_ctx *ctx) +{ + struct mail_transaction_log_file *file = ctx->log->head; + + /* first we need to truncate this latest write so that log syncing + doesn't break */ + if (ftruncate(file->fd, file->sync_offset) < 0) { + mail_index_file_set_syscall_error(ctx->log->index, + file->filepath, + "ftruncate()"); + } + + if (mail_index_move_to_memory(ctx->log->index) < 0) + return -1; + i_assert(MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)); + + i_assert(file->buffer_offset + file->buffer->used == file->sync_offset); + buffer_append_buf(file->buffer, ctx->output, 0, SIZE_MAX); + file->sync_offset = file->buffer_offset + file->buffer->used; + return 0; +} + +static int log_buffer_write(struct mail_transaction_log_append_ctx *ctx) +{ + struct mail_transaction_log_file *file = ctx->log->head; + + if (ctx->output->used == 0) + return 0; + + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) { + if (file->buffer == NULL) { + file->buffer = buffer_create_dynamic(default_pool, 4096); + file->buffer_offset = sizeof(file->hdr); + } + buffer_append_buf(file->buffer, ctx->output, 0, SIZE_MAX); + file->sync_offset = file->buffer_offset + file->buffer->used; + return 0; + } + + if (write_full(file->fd, ctx->output->data, ctx->output->used) < 0) { + /* write failure, fallback to in-memory indexes. */ + mail_index_file_set_syscall_error(ctx->log->index, + file->filepath, + "write_full()"); + return log_buffer_move_to_memory(ctx); + } + + i_assert(!ctx->sync_includes_this || + file->sync_offset + ctx->output->used == + file->max_tail_offset); + + if ((ctx->want_fsync && + file->log->index->set.fsync_mode != FSYNC_MODE_NEVER) || + file->log->index->set.fsync_mode == FSYNC_MODE_ALWAYS) { + if (fdatasync(file->fd) < 0) { + mail_index_file_set_syscall_error(ctx->log->index, + file->filepath, + "fdatasync()"); + return log_buffer_move_to_memory(ctx); + } + } + + if (file->mmap_base == NULL && file->buffer != NULL) { + /* we're reading from a file. avoid re-reading the data that + we just wrote. this is also important for some NFS clients, + which for some reason sometimes can't read() this data we + just wrote in the same process */ + i_assert(file->buffer_offset + + file->buffer->used == file->sync_offset); + buffer_append(file->buffer, ctx->output->data, + ctx->output->used); + } + file->sync_offset += ctx->output->used; + return 0; +} + +static void +log_append_sync_offset_if_needed(struct mail_transaction_log_append_ctx *ctx) +{ + struct mail_transaction_log_file *file = ctx->log->head; + struct mail_transaction_header_update *u; + struct mail_transaction_header *hdr; + uint32_t offset; + buffer_t buf; + unsigned char update_data[sizeof(*u) + sizeof(offset)]; + + if (!ctx->index_sync_transaction) { + /* this is a non-syncing transaction. update the tail offset + only if we're already writing something else to transaction + log anyway. */ + i_assert(!ctx->tail_offset_changed); + /* FIXME: For now we never do this update, because it would + cause errors about shrinking tail offsets with old Dovecot + versions. This is anyway just an optimization, so it doesn't + matter all that much if we don't do it here. Finish this + in v2.3. */ + /*if (ctx->output->used == 0)*/ + return; + } else if (file->max_tail_offset == file->sync_offset) { + /* we're synced all the way to tail offset, so this sync + transaction can also be included in the same tail offset. */ + if (ctx->output->used == 0 && !ctx->tail_offset_changed) { + /* nothing to write here after all (e.g. all unchanged + flag updates were dropped by export) */ + return; + } + + /* FIXME: when we remove exclusive log locking, we + can't rely on this. then write non-changed offset + check + real offset + rewrite the new offset if other transactions + weren't written in the middle */ + file->max_tail_offset += ctx->output->used + + sizeof(*hdr) + sizeof(*u) + sizeof(offset); + ctx->sync_includes_this = TRUE; + } else { + /* This is a syncing transaction. Since we're finishing a sync, + we may need to update the tail offset even if we don't have + anything else to do. */ + } + offset = file->max_tail_offset; + + if (file->last_read_hdr_tail_offset == offset) + return; + i_assert(offset > file->last_read_hdr_tail_offset); + + buffer_create_from_data(&buf, update_data, sizeof(update_data)); + u = buffer_append_space_unsafe(&buf, sizeof(*u)); + u->offset = offsetof(struct mail_index_header, log_file_tail_offset); + u->size = sizeof(offset); + buffer_append(&buf, &offset, sizeof(offset)); + + mail_transaction_log_append_add(ctx, MAIL_TRANSACTION_HEADER_UPDATE, + buf.data, buf.used); +} + +static int +mail_transaction_log_append_locked(struct mail_transaction_log_append_ctx *ctx) +{ + struct mail_transaction_log_file *file = ctx->log->head; + struct mail_transaction_boundary *boundary; + + if (file->sync_offset < file->last_size) { + /* there is some garbage at the end of the transaction log + (eg. previous write failed). remove it so reader doesn't + break because of it. */ + buffer_set_used_size(file->buffer, + file->sync_offset - file->buffer_offset); + if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) { + if (ftruncate(file->fd, file->sync_offset) < 0) { + mail_index_file_set_syscall_error(ctx->log->index, + file->filepath, "ftruncate()"); + } + } + } + + /* don't include log_file_tail_offset update in the transaction */ + boundary = buffer_get_space_unsafe(ctx->output, + sizeof(struct mail_transaction_header), + sizeof(*boundary)); + boundary->size = ctx->output->used; + + if (ctx->transaction_count <= 2) { + /* 0-1 changes. don't bother with the boundary */ + unsigned int boundary_size = + sizeof(struct mail_transaction_header) + + sizeof(*boundary); + + buffer_delete(ctx->output, 0, boundary_size); + } + + log_append_sync_offset_if_needed(ctx); + if (log_buffer_write(ctx) < 0) + return -1; + file->sync_highest_modseq = ctx->new_highest_modseq; + return 0; +} + +int mail_transaction_log_append_begin(struct mail_index *index, + enum mail_transaction_type flags, + struct mail_transaction_log_append_ctx **ctx_r) +{ + struct mail_transaction_log_append_ctx *ctx; + struct mail_transaction_boundary boundary; + + if (!index->log_sync_locked) { + if (mail_transaction_log_lock_head(index->log, "appending") < 0) + return -1; + } + ctx = i_new(struct mail_transaction_log_append_ctx, 1); + ctx->log = index->log; + ctx->output = buffer_create_dynamic(default_pool, 1024); + ctx->trans_flags = flags; + + i_zero(&boundary); + mail_transaction_log_append_add(ctx, MAIL_TRANSACTION_BOUNDARY, + &boundary, sizeof(boundary)); + + *ctx_r = ctx; + return 0; +} + +int mail_transaction_log_append_commit(struct mail_transaction_log_append_ctx **_ctx) +{ + struct mail_transaction_log_append_ctx *ctx = *_ctx; + struct mail_index *index = ctx->log->index; + int ret = 0; + + *_ctx = NULL; + + ret = mail_transaction_log_append_locked(ctx); + if (!index->log_sync_locked) + mail_transaction_log_file_unlock(index->log->head, "appending"); + + buffer_free(&ctx->output); + i_free(ctx); + return ret; +} diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c new file mode 100644 index 0000000..1820169 --- /dev/null +++ b/src/lib-index/mail-transaction-log-file.c @@ -0,0 +1,1685 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "file-dotlock.h" +#include "nfs-workarounds.h" +#include "read-full.h" +#include "write-full.h" +#include "mmap-util.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log-private.h" + +#define LOG_PREFETCH IO_BLOCK_SIZE +#define MEMORY_LOG_NAME "(in-memory transaction log file)" +#define LOG_NEW_DOTLOCK_SUFFIX ".newlock" + +static int +mail_transaction_log_file_sync(struct mail_transaction_log_file *file, + bool *retry_r, const char **reason_r); + +static void +log_file_set_syscall_error(struct mail_transaction_log_file *file, + const char *function) +{ + mail_index_file_set_syscall_error(file->log->index, + file->filepath, function); +} + +static void +mail_transaction_log_mark_corrupted(struct mail_transaction_log_file *file) +{ + unsigned int offset = + offsetof(struct mail_transaction_log_header, indexid); + int flags; + + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file) || + file->log->index->readonly) + return; + + /* indexid=0 marks the log file as corrupted. we opened the file with + O_APPEND, and now we need to drop it for pwrite() to work (at least + in Linux) */ + flags = fcntl(file->fd, F_GETFL, 0); + if (flags < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "fcntl(F_GETFL)"); + return; + } + if (fcntl(file->fd, F_SETFL, flags & ~O_APPEND) < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "fcntl(F_SETFL)"); + return; + } + if (pwrite_full(file->fd, &file->hdr.indexid, + sizeof(file->hdr.indexid), offset) < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "pwrite()"); + } +} + +void +mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file, + const char *fmt, ...) +{ + va_list va; + + file->corrupted = TRUE; + file->hdr.indexid = 0; + mail_transaction_log_mark_corrupted(file); + + va_start(va, fmt); + T_BEGIN { + mail_index_set_error(file->log->index, + "Corrupted transaction log file %s seq %u: %s " + "(sync_offset=%"PRIuUOFF_T")", + file->filepath, file->hdr.file_seq, + t_strdup_vprintf(fmt, va), file->sync_offset); + } T_END; + va_end(va); +} + +struct mail_transaction_log_file * +mail_transaction_log_file_alloc(struct mail_transaction_log *log, + const char *path) +{ + struct mail_transaction_log_file *file; + + file = i_new(struct mail_transaction_log_file, 1); + file->log = log; + file->filepath = i_strdup(path); + file->fd = -1; + return file; +} + +void mail_transaction_log_file_free(struct mail_transaction_log_file **_file) +{ + struct mail_transaction_log_file *file = *_file; + struct mail_transaction_log_file **p; + int old_errno = errno; + + *_file = NULL; + + i_assert(!file->locked); + i_assert(file->refcount == 0); + + for (p = &file->log->files; *p != NULL; p = &(*p)->next) { + if (*p == file) { + *p = file->next; + break; + } + } + + if (file == file->log->head) + file->log->head = NULL; + + buffer_free(&file->buffer); + + if (file->mmap_base != NULL) { + if (munmap(file->mmap_base, file->mmap_size) < 0) + log_file_set_syscall_error(file, "munmap()"); + } + + if (file->fd != -1) { + if (close(file->fd) < 0) + log_file_set_syscall_error(file, "close()"); + } + + i_free(file->filepath); + i_free(file->need_rotate); + i_free(file); + + errno = old_errno; +} + +static void +mail_transaction_log_file_skip_to_head(struct mail_transaction_log_file *file) +{ + struct mail_transaction_log *log = file->log; + struct mail_index_map *map = log->index->map; + const struct mail_index_modseq_header *modseq_hdr; + uoff_t head_offset; + + if (map == NULL || file->hdr.file_seq != map->hdr.log_file_seq || + map->hdr.log_file_head_offset == 0) + return; + + /* we can get a valid log offset from index file. initialize + sync_offset from it so we don't have to read the whole log + file from beginning. */ + head_offset = map->hdr.log_file_head_offset; + + modseq_hdr = mail_index_map_get_modseq_header(map); + if (head_offset < file->hdr.hdr_size) { + mail_index_set_error(log->index, + "%s: log_file_head_offset too small", + log->index->filepath); + file->sync_offset = file->hdr.hdr_size; + file->sync_highest_modseq = file->hdr.initial_modseq; + } else if (modseq_hdr == NULL && file->hdr.initial_modseq == 0) { + /* modseqs not used yet */ + file->sync_offset = head_offset; + file->sync_highest_modseq = 0; + } else if (modseq_hdr == NULL || + modseq_hdr->log_seq != file->hdr.file_seq) { + /* highest_modseq not synced, start from beginning */ + file->sync_offset = file->hdr.hdr_size; + file->sync_highest_modseq = file->hdr.initial_modseq; + } else if (modseq_hdr->log_offset > head_offset) { + mail_index_set_error(log->index, + "%s: modseq_hdr.log_offset too large", + log->index->filepath); + file->sync_offset = file->hdr.hdr_size; + file->sync_highest_modseq = file->hdr.initial_modseq; + } else { + /* start from where we last stopped tracking modseqs */ + file->sync_offset = modseq_hdr->log_offset; + file->sync_highest_modseq = modseq_hdr->highest_modseq; + } + if (file->hdr.file_seq == log->index->map->hdr.log_file_seq) { + file->last_read_hdr_tail_offset = + log->index->map->hdr.log_file_tail_offset; + } + if (file->last_read_hdr_tail_offset > file->max_tail_offset) + file->max_tail_offset = file->last_read_hdr_tail_offset; +} + +static void +mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file) +{ + struct mail_transaction_log_file **p; + const char *reason; + bool retry; + + file->sync_offset = file->hdr.hdr_size; + file->sync_highest_modseq = file->hdr.initial_modseq; + mail_transaction_log_file_skip_to_head(file); + + /* insert it to correct position */ + for (p = &file->log->files; *p != NULL; p = &(*p)->next) { + if ((*p)->hdr.file_seq > file->hdr.file_seq) + break; + i_assert((*p)->hdr.file_seq < file->hdr.file_seq); + } + + file->next = *p; + *p = file; + + if (file->buffer != NULL) { + /* if we read any unfinished data, make sure the buffer gets + truncated. */ + (void)mail_transaction_log_file_sync(file, &retry, &reason); + buffer_set_used_size(file->buffer, + file->sync_offset - file->buffer_offset); + } +} + +static int +mail_transaction_log_init_hdr(struct mail_transaction_log *log, + struct mail_transaction_log_header *hdr) +{ + struct mail_index *index = log->index; + struct mail_transaction_log_file *file; + + i_assert(index->indexid != 0); + + i_zero(hdr); + hdr->major_version = MAIL_TRANSACTION_LOG_MAJOR_VERSION; + hdr->minor_version = MAIL_TRANSACTION_LOG_MINOR_VERSION; + hdr->hdr_size = sizeof(struct mail_transaction_log_header); + hdr->indexid = log->index->indexid; + hdr->create_stamp = ioloop_time; +#ifndef WORDS_BIGENDIAN + hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#endif + + if (index->fd != -1) { + /* not creating index - make sure we have latest header */ + if (!index->mapping) { + if (mail_index_map(index, + MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0) + return -1; + } else { + /* if we got here from mapping, the .log file is + corrupted. use whatever values we got from index + file */ + } + } + if (index->map != NULL) { + hdr->prev_file_seq = index->map->hdr.log_file_seq; + hdr->prev_file_offset = index->map->hdr.log_file_head_offset; + hdr->file_seq = index->map->hdr.log_file_seq + 1; + hdr->initial_modseq = + mail_index_map_modseq_get_highest(index->map); + } else { + hdr->file_seq = 1; + } + if (hdr->initial_modseq == 0) { + /* modseq tracking in log files is required for many reasons + nowadays, even if per-message modseqs aren't enabled in + dovecot.index. */ + hdr->initial_modseq = 1; + } + + if (log->head != NULL) { + /* make sure the sequence always increases to avoid crashes + later. this catches the buggy case where two processes + happen to replace the same log file. */ + for (file = log->head->next; file != NULL; file = file->next) { + if (hdr->file_seq <= file->hdr.file_seq) + hdr->file_seq = file->hdr.file_seq + 1; + } + + if (hdr->file_seq <= log->head->hdr.file_seq) { + /* make sure the sequence grows */ + hdr->file_seq = log->head->hdr.file_seq+1; + } + if (hdr->initial_modseq < log->head->sync_highest_modseq) { + /* this should be always up-to-date */ + hdr->initial_modseq = log->head->sync_highest_modseq; + } + } + return 0; +} + +struct mail_transaction_log_file * +mail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + + file = mail_transaction_log_file_alloc(log, MEMORY_LOG_NAME); + if (mail_transaction_log_init_hdr(log, &file->hdr) < 0) { + i_free(file); + return NULL; + } + + file->buffer = buffer_create_dynamic(default_pool, 4096); + file->buffer_offset = sizeof(file->hdr); + + mail_transaction_log_file_add_to_list(file); + return file; +} + +static int +mail_transaction_log_file_dotlock(struct mail_transaction_log_file *file) +{ + struct dotlock_settings dotlock_set; + int ret; + + if (file->log->dotlock_refcount > 0) + ret = 1; + else { + i_assert(file->log->dotlock_refcount == 0); + mail_transaction_log_get_dotlock_set(file->log, &dotlock_set); + ret = file_dotlock_create(&dotlock_set, file->filepath, 0, + &file->log->dotlock); + } + if (ret > 0) { + file->log->dotlock_refcount++; + file->locked = TRUE; + file->lock_create_time = time(NULL); + return 0; + } + if (ret < 0) { + log_file_set_syscall_error(file, "file_dotlock_create()"); + return -1; + } + + mail_index_set_error(file->log->index, + "Timeout (%us) while waiting for " + "dotlock for transaction log file %s", + dotlock_set.timeout, file->filepath); + return -1; +} + +static int +mail_transaction_log_file_undotlock(struct mail_transaction_log_file *file) +{ + int ret; + + i_assert(file->log->dotlock_refcount >= 0); + if (--file->log->dotlock_refcount > 0) + return 0; + + ret = file_dotlock_delete(&file->log->dotlock); + if (ret < 0) { + log_file_set_syscall_error(file, "file_dotlock_delete()"); + return -1; + } + + if (ret == 0) { + mail_index_set_error(file->log->index, + "Dotlock was lost for transaction log file %s", + file->filepath); + return -1; + } + return 0; +} + +int mail_transaction_log_file_lock(struct mail_transaction_log_file *file) +{ + unsigned int lock_timeout_secs; + int ret; + + if (file->locked) + return 0; + + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) { + file->locked = TRUE; + return 0; + } + + if (file->log->index->set.lock_method == FILE_LOCK_METHOD_DOTLOCK) + return mail_transaction_log_file_dotlock(file); + + if (file->log->index->readonly) { + mail_index_set_error(file->log->index, + "Index is read-only, can't write-lock %s", + file->filepath); + return -1; + } + + i_assert(file->file_lock == NULL); + lock_timeout_secs = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT, + file->log->index->set.max_lock_timeout_secs); + ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd, + F_WRLCK, lock_timeout_secs, + &file->file_lock); + if (ret > 0) { + file->locked = TRUE; + file->lock_create_time = time(NULL); + return 0; + } + if (ret < 0) { + log_file_set_syscall_error(file, "mail_index_wait_lock_fd()"); + return -1; + } + + mail_index_set_error(file->log->index, + "Timeout (%us) while waiting for lock for " + "transaction log file %s%s", + lock_timeout_secs, file->filepath, + file_lock_find(file->fd, file->log->index->set.lock_method, F_WRLCK)); + return -1; +} + +void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file, + const char *lock_reason) +{ + unsigned int lock_time; + + if (!file->locked) + return; + + file->locked = FALSE; + file->locked_sync_offset_updated = FALSE; + + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) + return; + + lock_time = time(NULL) - file->lock_create_time; + if (lock_time >= MAIL_TRANSACTION_LOG_LOCK_WARN_SECS && lock_reason != NULL) { + i_warning("Transaction log file %s was locked for %u seconds (%s)", + file->filepath, lock_time, lock_reason); + } + + if (file->log->index->set.lock_method == FILE_LOCK_METHOD_DOTLOCK) { + (void)mail_transaction_log_file_undotlock(file); + return; + } + + file_unlock(&file->file_lock); +} + +static ssize_t +mail_transaction_log_file_read_header(struct mail_transaction_log_file *file) +{ + void *dest; + size_t pos, dest_size; + ssize_t ret; + + i_assert(file->buffer == NULL && file->mmap_base == NULL); + + i_zero(&file->hdr); + if (file->last_size < mmap_get_page_size() && file->last_size > 0) { + /* just read the entire transaction log to memory. + note that if some of the data hasn't been fully committed + yet (hdr.size=0), the buffer must be truncated later */ + file->buffer = buffer_create_dynamic(default_pool, 4096); + file->buffer_offset = 0; + dest_size = file->last_size; + dest = buffer_append_space_unsafe(file->buffer, dest_size); + } else { + /* read only the header */ + dest = &file->hdr; + dest_size = sizeof(file->hdr); + } + + /* it's not necessarily an error to read less than wanted header size, + since older versions of the log format used smaller headers. */ + pos = 0; + do { + ret = pread(file->fd, PTR_OFFSET(dest, pos), + dest_size - pos, pos); + if (ret > 0) + pos += ret; + } while (ret > 0 && pos < dest_size); + + if (file->buffer != NULL) { + buffer_set_used_size(file->buffer, pos); + memcpy(&file->hdr, file->buffer->data, + I_MIN(pos, sizeof(file->hdr))); + } + + return ret < 0 ? -1 : (ssize_t)pos; +} + +static int +mail_transaction_log_file_fail_dupe(struct mail_transaction_log_file *file) +{ + int ret; + + /* mark the old file corrupted. we can't safely remove + it from the list however, so return failure. */ + file->hdr.indexid = 0; + if (strcmp(file->filepath, file->log->head->filepath) != 0) { + /* only mark .2 corrupted, just to make sure we don't lose any + changes from .log in case we're somehow wrong */ + mail_transaction_log_mark_corrupted(file); + ret = 0; + } else { + ret = -1; + } + if (!file->corrupted) { + file->corrupted = TRUE; + mail_index_set_error(file->log->index, + "Transaction log %s: " + "duplicate transaction log sequence (%u)", + file->filepath, file->hdr.file_seq); + } + return ret; +} + +static int +mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file, + bool ignore_estale) +{ + struct mail_transaction_log_file *f; + int ret; + + i_assert(!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)); + + if (file->corrupted) + return 0; + + ret = mail_transaction_log_file_read_header(file); + if (ret < 0) { + if (errno != ESTALE || !ignore_estale) + log_file_set_syscall_error(file, "pread()"); + return -1; + } + if (file->hdr.major_version != MAIL_TRANSACTION_LOG_MAJOR_VERSION) { + /* incompatible version - fix silently */ + return 0; + } + if (ret < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) { + mail_transaction_log_file_set_corrupted(file, + "unexpected end of file while reading header"); + return 0; + } + + const unsigned int hdr_version = + MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr); + if (MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, COMPAT_FLAGS)) { + /* we have compatibility flags */ + enum mail_index_header_compat_flags compat_flags = 0; + +#ifndef WORDS_BIGENDIAN + compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#endif + if (file->hdr.compat_flags != compat_flags) { + /* architecture change */ + mail_index_set_error(file->log->index, + "Rebuilding index file %s: " + "CPU architecture changed", + file->log->index->filepath); + return 0; + } + } + if (file->hdr.hdr_size < MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) { + mail_transaction_log_file_set_corrupted(file, + "Header size too small"); + return 0; + } + if (file->hdr.hdr_size < sizeof(file->hdr)) { + /* @UNSAFE: smaller than we expected - zero out the fields we + shouldn't have filled */ + memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0, + sizeof(file->hdr) - file->hdr.hdr_size); + } + + if (file->hdr.indexid == 0) { + /* corrupted */ + file->corrupted = TRUE; + mail_index_set_error(file->log->index, + "Transaction log file %s: marked corrupted", + file->filepath); + return 0; + } + if (file->hdr.indexid != file->log->index->indexid) { + if (file->log->index->indexid != 0 && + !file->log->index->initial_create) { + /* index file was probably just rebuilt and we don't + know about it yet */ + mail_transaction_log_file_set_corrupted(file, + "indexid changed: %u -> %u", + file->log->index->indexid, file->hdr.indexid); + return 0; + } + + /* creating index file. since transaction log is created + first, use the indexid in it to create the main index + to avoid races. */ + file->log->index->indexid = file->hdr.indexid; + } + + /* make sure we already don't have a file with the same sequence + opened. it shouldn't happen unless the old log file was + corrupted. */ + for (f = file->log->files; f != NULL; f = f->next) { + if (f->hdr.file_seq == file->hdr.file_seq) { + if (strcmp(f->filepath, f->log->head->filepath) != 0) { + /* old "f" is the .log.2 */ + return mail_transaction_log_file_fail_dupe(f); + } else { + /* new "file" is probably the .log.2 */ + return mail_transaction_log_file_fail_dupe(file); + } + } + } + + file->sync_highest_modseq = file->hdr.initial_modseq; + return 1; +} + +static int +mail_transaction_log_file_stat(struct mail_transaction_log_file *file, + bool ignore_estale) +{ + struct stat st; + + if (fstat(file->fd, &st) < 0) { + if (!ESTALE_FSTAT(errno) || !ignore_estale) + log_file_set_syscall_error(file, "fstat()"); + return -1; + } + + file->st_dev = st.st_dev; + file->st_ino = st.st_ino; + file->last_mtime = st.st_mtime; + file->last_size = st.st_size; + return 0; +} + +static bool +mail_transaction_log_file_is_dupe(struct mail_transaction_log_file *file) +{ + struct mail_transaction_log_file *tmp; + + for (tmp = file->log->files; tmp != NULL; tmp = tmp->next) { + if (tmp->st_ino == file->st_ino && + CMP_DEV_T(tmp->st_dev, file->st_dev)) + return TRUE; + } + return FALSE; +} + +static void log_write_ext_hdr_init_data(struct mail_index *index, buffer_t *buf) +{ + const struct mail_index_registered_ext *rext; + struct mail_transaction_header *hdr; + struct mail_transaction_ext_intro *intro; + struct mail_transaction_ext_hdr_update *ext_hdr; + unsigned int hdr_offset; + + rext = array_idx(&index->extensions, index->set.ext_hdr_init_id); + + /* introduce the extension */ + hdr_offset = buf->used; + hdr = buffer_append_space_unsafe(buf, sizeof(*hdr)); + hdr->type = MAIL_TRANSACTION_EXT_INTRO; + + intro = buffer_append_space_unsafe(buf, sizeof(*intro)); + intro->ext_id = (uint32_t)-1; + intro->hdr_size = rext->hdr_size; + intro->record_size = rext->record_size; + intro->record_align = rext->record_align; + intro->name_size = strlen(rext->name); + buffer_append(buf, rext->name, intro->name_size); + if (buf->used % 4 != 0) + buffer_append_zero(buf, 4 - buf->used % 4); + + hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr)); + hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset); + + /* add the extension header data */ + hdr_offset = buf->used; + hdr = buffer_append_space_unsafe(buf, sizeof(*hdr)); + hdr->type = MAIL_TRANSACTION_EXT_HDR_UPDATE; + + ext_hdr = buffer_append_space_unsafe(buf, sizeof(*ext_hdr)); + ext_hdr->size = rext->hdr_size; + buffer_append(buf, index->set.ext_hdr_init_data, rext->hdr_size); + + hdr = buffer_get_space_unsafe(buf, hdr_offset, sizeof(*hdr)); + hdr->size = mail_index_uint32_to_offset(buf->used - hdr_offset); +} + +static int +mail_transaction_log_file_create2(struct mail_transaction_log_file *file, + int new_fd, bool reset, + struct dotlock **dotlock) +{ + struct mail_index *index = file->log->index; + struct stat st; + const char *path2; + buffer_t *writebuf; + int fd, ret; + bool rename_existing, need_lock; + + need_lock = file->log->head != NULL && file->log->head->locked; + + if (fcntl(new_fd, F_SETFL, O_APPEND) < 0) { + log_file_set_syscall_error(file, "fcntl(O_APPEND)"); + return -1; + } + + if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) { + /* although we check also mtime and file size below, it's done + only to fix broken log files. we don't bother flushing + attribute cache just for that. */ + nfs_flush_file_handle_cache(file->filepath); + } + + /* log creation is locked now - see if someone already created it. + note that if we're rotating, we need to keep the log locked until + the file has been rewritten. and because fcntl() locks are stupid, + if we go and open()+close() the file and we had it already opened, + its locks are lost. so we use stat() to check if the file has been + recreated, although it almost never is. */ + if (reset) + rename_existing = FALSE; + else if (nfs_safe_stat(file->filepath, &st) < 0) { + if (errno != ENOENT) { + log_file_set_syscall_error(file, "stat()"); + return -1; + } + rename_existing = FALSE; + } else if (st.st_ino == file->st_ino && + CMP_DEV_T(st.st_dev, file->st_dev) && + /* inode/dev checks are enough when we're rotating the file, + but not when we're replacing a broken log file */ + st.st_mtime == file->last_mtime && + (uoff_t)st.st_size == file->last_size) { + /* no-one else recreated the file */ + rename_existing = TRUE; + } else { + /* recreated. use the file if its header is ok */ + fd = nfs_safe_open(file->filepath, O_RDWR | O_APPEND); + if (fd == -1) { + if (errno != ENOENT) { + log_file_set_syscall_error(file, "open()"); + return -1; + } + } else { + file->fd = fd; + file->last_size = 0; + if (mail_transaction_log_file_read_hdr(file, + FALSE) > 0 && + mail_transaction_log_file_stat(file, FALSE) == 0) { + /* yes, it was ok */ + file_dotlock_delete(dotlock); + mail_transaction_log_file_add_to_list(file); + return 0; + } + file->fd = -1; + if (close(fd) < 0) + log_file_set_syscall_error(file, "close()"); + } + rename_existing = FALSE; + } + + if (index->fd == -1 && !rename_existing) { + /* creating the initial index */ + reset = TRUE; + } + + if (mail_transaction_log_init_hdr(file->log, &file->hdr) < 0) + return -1; + + if (reset) { + /* don't reset modseqs. if we're reseting due to rebuilding + indexes we'll probably want to keep uidvalidity and in such + cases we really don't want to shrink modseqs. */ + file->hdr.prev_file_seq = 0; + file->hdr.prev_file_offset = 0; + } + + writebuf = t_buffer_create(128); + buffer_append(writebuf, &file->hdr, sizeof(file->hdr)); + + if (index->set.ext_hdr_init_data != NULL && reset) + log_write_ext_hdr_init_data(index, writebuf); + if (write_full(new_fd, writebuf->data, writebuf->used) < 0) { + log_file_set_syscall_error(file, "write_full()"); + return -1; + } + + if (file->log->index->set.fsync_mode == FSYNC_MODE_ALWAYS) { + /* the header isn't important, so don't bother calling + fdatasync() unless it's required */ + if (fdatasync(new_fd) < 0) { + log_file_set_syscall_error(file, "fdatasync()"); + return -1; + } + } + + file->fd = new_fd; + ret = mail_transaction_log_file_stat(file, FALSE); + + if (need_lock && ret == 0) { + /* we'll need to preserve the lock */ + if (mail_transaction_log_file_lock(file) < 0) + ret = -1; + } + + /* if we return -1 the dotlock deletion code closes the fd */ + file->fd = -1; + if (ret < 0) + return -1; + + /* keep two log files */ + if (rename_existing) { + /* rename() would be nice and easy way to do this, except then + there's a race condition between the rename and + file_dotlock_replace(). during that time the log file + doesn't exist, which could cause problems. */ + path2 = t_strconcat(file->filepath, ".2", NULL); + if (i_unlink_if_exists(path2) < 0) { + /* try to link() anyway */ + } + if (nfs_safe_link(file->filepath, path2, FALSE) < 0 && + errno != ENOENT && errno != EEXIST) { + mail_index_set_error(index, "link(%s, %s) failed: %m", + file->filepath, path2); + /* ignore the error. we don't care that much about the + second log file and we're going to overwrite this + first one. */ + } + /* NOTE: here's a race condition where both .log and .log.2 + point to the same file. our reading code should ignore that + though by comparing the inodes. */ + } + + if (file_dotlock_replace(dotlock, + DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) <= 0) { + /* need to unlock to avoid assert-crash in + mail_transaction_log_file_free() */ + mail_transaction_log_file_unlock(file, "creation failed"); + return -1; + } + + /* success */ + file->fd = new_fd; + mail_transaction_log_file_add_to_list(file); + + i_assert(!need_lock || file->locked); + return 1; +} + +int mail_transaction_log_file_create(struct mail_transaction_log_file *file, + bool reset) +{ + struct mail_index *index = file->log->index; + struct dotlock_settings new_dotlock_set; + struct dotlock *dotlock; + mode_t old_mask; + int fd, ret; + + i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); + + if (file->log->index->readonly) { + mail_index_set_error(index, + "Can't create log file %s: Index is read-only", + file->filepath); + return -1; + } + + if (index->indexid == 0) { + mail_index_set_error(index, + "Can't create log file %s: Index is marked corrupted", + file->filepath); + return -1; + } + + mail_transaction_log_get_dotlock_set(file->log, &new_dotlock_set); + new_dotlock_set.lock_suffix = LOG_NEW_DOTLOCK_SUFFIX; + + /* With dotlocking we might already have path.lock created, so this + filename has to be different. */ + old_mask = umask(index->set.mode ^ 0666); + fd = file_dotlock_open(&new_dotlock_set, file->filepath, 0, &dotlock); + umask(old_mask); + + if (fd == -1) { + log_file_set_syscall_error(file, "file_dotlock_open()"); + return -1; + } + mail_index_fchown(index, fd, file_dotlock_get_lock_path(dotlock)); + + /* either fd gets used or the dotlock gets deleted and returned fd + is for the existing file */ + ret = mail_transaction_log_file_create2(file, fd, reset, &dotlock); + if (ret < 0) { + if (dotlock != NULL) + file_dotlock_delete(&dotlock); + return -1; + } + return ret; +} + +int mail_transaction_log_file_open(struct mail_transaction_log_file *file, + const char **reason_r) +{ + struct mail_index *index = file->log->index; + unsigned int i; + bool ignore_estale; + int ret; + + for (i = 0;; i++) { + if (!index->readonly) { + file->fd = nfs_safe_open(file->filepath, + O_RDWR | O_APPEND); + } else { + file->fd = nfs_safe_open(file->filepath, O_RDONLY); + } + if (file->fd == -1 && errno == EACCES) { + file->fd = nfs_safe_open(file->filepath, O_RDONLY); + index->readonly = TRUE; + } + if (file->fd == -1) { + if (errno == ENOENT) { + *reason_r = "File doesn't exist"; + return 0; + } + + log_file_set_syscall_error(file, "open()"); + *reason_r = t_strdup_printf("open() failed: %m"); + return -1; + } + + ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT; + if (mail_transaction_log_file_stat(file, ignore_estale) < 0) + ret = -1; + else if (mail_transaction_log_file_is_dupe(file)) { + /* probably our already opened .log file has been + renamed to .log.2 and we're trying to reopen it. + also possible that hit a race condition where .log + and .log.2 are linked. */ + *reason_r = "File is already open"; + return 0; + } else { + ret = mail_transaction_log_file_read_hdr(file, + ignore_estale); + } + if (ret > 0) { + /* success */ + break; + } + + if (ret == 0) { + /* corrupted */ + if (index->readonly) { + /* don't delete */ + } else { + i_unlink_if_exists(file->filepath); + } + *reason_r = "File is corrupted"; + return 0; + } + if (errno != ESTALE || + i == MAIL_INDEX_ESTALE_RETRY_COUNT) { + /* syscall error */ + *reason_r = t_strdup_printf("fstat() failed: %m"); + return -1; + } + + /* ESTALE - try again */ + buffer_free(&file->buffer); + } + + mail_transaction_log_file_add_to_list(file); + return 1; +} + +static int +log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file, + const void *data, unsigned int trans_size, + const char **error_r) +{ + const struct mail_transaction_header_update *u = data; + const struct mail_index_header *ihdr; + const unsigned int size = trans_size - sizeof(struct mail_transaction_header); + const unsigned int offset_pos = + offsetof(struct mail_index_header, log_file_tail_offset); + const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset); + uint32_t tail_offset; + + i_assert(offset_size == sizeof(tail_offset)); + + if (size < sizeof(*u) || size < sizeof(*u) + u->size) { + *error_r = "header update extends beyond record size"; + mail_transaction_log_file_set_corrupted(file, "%s", *error_r); + return -1; + } + + if (u->offset <= offset_pos && + u->offset + u->size >= offset_pos + offset_size) { + memcpy(&tail_offset, + CONST_PTR_OFFSET(u + 1, offset_pos - u->offset), + sizeof(tail_offset)); + + if (tail_offset < file->last_read_hdr_tail_offset) { + /* ignore shrinking tail offsets */ + return 1; + } else if (tail_offset > file->sync_offset + trans_size) { + mail_transaction_log_file_set_corrupted(file, + "log_file_tail_offset %u goes past sync offset %"PRIuUOFF_T, + tail_offset, file->sync_offset + trans_size); + } else { + file->last_read_hdr_tail_offset = tail_offset; + if (tail_offset > file->max_tail_offset) + file->max_tail_offset = tail_offset; + return 1; + } + } + return 0; +} + +static bool +flag_updates_have_non_internal(const struct mail_transaction_flag_update *u, + unsigned int count, unsigned int version) +{ + /* Hide internal flags from modseqs if the log file's version + is new enough. This allows upgrading without the modseqs suddenly + shrinking. */ + if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS)) + return TRUE; + + for (unsigned int i = 0; i < count; i++) { + if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(&u[i])) + return TRUE; + } + return FALSE; +} + +void mail_transaction_update_modseq(const struct mail_transaction_header *hdr, + const void *data, uint64_t *cur_modseq, + unsigned int version) +{ + uint32_t trans_size; + + trans_size = mail_index_offset_to_uint32(hdr->size); + i_assert(trans_size != 0); + + if (*cur_modseq != 0) { + /* tracking modseqs */ + } else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) == + MAIL_TRANSACTION_EXT_INTRO) { + /* modseqs not tracked yet. see if this is a modseq + extension introduction. */ + const struct mail_transaction_ext_intro *intro = data; + const unsigned int modseq_ext_len = + strlen(MAIL_INDEX_MODSEQ_EXT_NAME); + + if (intro->name_size == modseq_ext_len && + memcmp(intro + 1, MAIL_INDEX_MODSEQ_EXT_NAME, + modseq_ext_len) == 0) { + /* modseq tracking started */ + *cur_modseq += 1; + } + return; + } else { + /* not tracking modseqs */ + return; + } + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT: + case MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT: + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* ignore expunge requests */ + break; + } + /* fall through */ + case MAIL_TRANSACTION_APPEND: + case MAIL_TRANSACTION_KEYWORD_UPDATE: + case MAIL_TRANSACTION_KEYWORD_RESET: + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: + /* these changes increase modseq */ + *cur_modseq += 1; + break; + case MAIL_TRANSACTION_FLAG_UPDATE: { + const struct mail_transaction_flag_update *rec = data; + unsigned int count; + + count = (trans_size - sizeof(*hdr)) / sizeof(*rec); + if (flag_updates_have_non_internal(rec, count, version)) + *cur_modseq += 1; + break; + } + case MAIL_TRANSACTION_MODSEQ_UPDATE: { + const struct mail_transaction_modseq_update *rec, *end; + + end = CONST_PTR_OFFSET(data, trans_size - sizeof(*hdr)); + for (rec = data; rec < end; rec++) { + uint64_t modseq = ((uint64_t)rec->modseq_high32 << 32) | + rec->modseq_low32; + if (*cur_modseq < modseq) + *cur_modseq = modseq; + } + } + } +} + +static int +log_file_track_sync(struct mail_transaction_log_file *file, + const struct mail_transaction_header *hdr, + unsigned int trans_size, const char **error_r) +{ + const void *data = hdr + 1; + int ret; + + mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq, + MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr)); + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) + return 1; + + /* external transactions: */ + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_HEADER_UPDATE: + /* see if this updates mailbox_sync_offset */ + ret = log_file_track_mailbox_sync_offset_hdr(file, data, + trans_size, error_r); + if (ret != 0) + return ret < 0 ? -1 : 1; + break; + case MAIL_TRANSACTION_INDEX_DELETED: + if (file->sync_offset < file->index_undeleted_offset || + file->hdr.file_seq < file->log->index->index_delete_changed_file_seq) + break; + file->log->index->index_deleted = TRUE; + file->log->index->index_delete_requested = FALSE; + file->log->index->index_delete_changed_file_seq = file->hdr.file_seq; + file->index_deleted_offset = file->sync_offset + trans_size; + break; + case MAIL_TRANSACTION_INDEX_UNDELETED: + if (file->sync_offset < file->index_deleted_offset || + file->hdr.file_seq < file->log->index->index_delete_changed_file_seq) + break; + file->log->index->index_deleted = FALSE; + file->log->index->index_delete_requested = FALSE; + file->log->index->index_delete_changed_file_seq = file->hdr.file_seq; + file->index_undeleted_offset = file->sync_offset + trans_size; + break; + case MAIL_TRANSACTION_BOUNDARY: { + const struct mail_transaction_boundary *boundary = + (const void *)(hdr + 1); + size_t wanted_buffer_size; + + wanted_buffer_size = file->sync_offset - file->buffer_offset + + boundary->size; + if (wanted_buffer_size > file->buffer->used) { + /* the full transaction hasn't been written yet */ + return 0; + } + break; + } + } + + if (file->max_tail_offset == file->sync_offset) { + /* external transactions aren't synced to mailbox. we can + update mailbox sync offset to skip this transaction to + avoid re-reading it at the next sync. */ + file->max_tail_offset += trans_size; + } + return 1; +} + +static int +mail_transaction_log_file_sync(struct mail_transaction_log_file *file, + bool *retry_r, const char **reason_r) +{ + const struct mail_transaction_header *hdr; + const void *data; + struct stat st; + size_t size, avail; + uint32_t trans_size = 0; + int ret; + + i_assert(file->sync_offset >= file->buffer_offset); + + *retry_r = FALSE; + + data = buffer_get_data(file->buffer, &size); + if (file->buffer_offset + size < file->sync_offset) { + *reason_r = t_strdup_printf( + "log file shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")", + file->buffer_offset + (uoff_t)size, file->sync_offset); + mail_transaction_log_file_set_corrupted(file, "%s", *reason_r); + /* fix the sync_offset to avoid crashes later on */ + file->sync_offset = file->buffer_offset + size; + return 0; + } + while (file->sync_offset - file->buffer_offset + sizeof(*hdr) <= size) { + hdr = CONST_PTR_OFFSET(data, file->sync_offset - + file->buffer_offset); + trans_size = mail_index_offset_to_uint32(hdr->size); + if (trans_size == 0) { + /* unfinished or corrupted */ + break; + } + if (trans_size < sizeof(*hdr)) { + *reason_r = t_strdup_printf( + "hdr.size too small (%u)", trans_size); + mail_transaction_log_file_set_corrupted(file, "%s", *reason_r); + return 0; + } + + if (file->sync_offset - file->buffer_offset + trans_size > size) + break; + + /* transaction has been fully written */ + if ((ret = log_file_track_sync(file, hdr, trans_size, reason_r)) <= 0) { + if (ret < 0) + return 0; + break; + } + + file->sync_offset += trans_size; + } + + if (file->mmap_base != NULL && !file->locked) { + /* Now that all the mmaped pages have page faulted, check if + the file had changed while doing that. Only after the last + page has faulted, the size returned by fstat() can be + trusted. Otherwise it might point to a page boundary while + the next page is still being written. + + Without this check we might see partial transactions, + sometimes causing "Extension record updated without intro + prefix" errors. */ + if (fstat(file->fd, &st) < 0) { + log_file_set_syscall_error(file, "fstat()"); + *reason_r = t_strdup_printf("fstat() failed: %m"); + return -1; + } + if ((uoff_t)st.st_size != file->last_size) { + file->last_size = st.st_size; + *retry_r = TRUE; + *reason_r = "File size changed - retrying"; + return 0; + } + } + + avail = file->sync_offset - file->buffer_offset; + if (avail != size) { + /* There's more data than we could sync at the moment. If the + last record's size wasn't valid, we can't know if it will + be updated unless we've locked the log. */ + if (file->locked) { + *reason_r = "Unexpected garbage at EOF"; + mail_transaction_log_file_set_corrupted(file, "%s", *reason_r); + return 0; + } + /* The size field will be updated soon */ + mail_index_flush_read_cache(file->log->index, file->filepath, + file->fd, file->locked); + } + + if (file->next != NULL && + file->hdr.file_seq == file->next->hdr.prev_file_seq && + file->next->hdr.prev_file_offset != file->sync_offset) { + *reason_r = t_strdup_printf( + "Invalid transaction log size " + "(%"PRIuUOFF_T" vs %u): %s", file->sync_offset, + file->log->head->hdr.prev_file_offset, file->filepath); + mail_transaction_log_file_set_corrupted(file, "%s", *reason_r); + return 0; + } + + return 1; +} + +static int +mail_transaction_log_file_insert_read(struct mail_transaction_log_file *file, + uoff_t offset, const char **reason_r) +{ + void *data; + size_t size; + ssize_t ret; + + size = file->buffer_offset - offset; + buffer_copy(file->buffer, size, file->buffer, 0, SIZE_MAX); + + data = buffer_get_space_unsafe(file->buffer, 0, size); + ret = pread_full(file->fd, data, size, offset); + if (ret > 0) { + /* success */ + file->buffer_offset -= size; + return 1; + } + + /* failure. don't leave ourself to inconsistent state */ + buffer_copy(file->buffer, 0, file->buffer, size, SIZE_MAX); + buffer_set_used_size(file->buffer, file->buffer->used - size); + + if (ret == 0) { + *reason_r = "file shrank unexpectedly"; + mail_transaction_log_file_set_corrupted(file, "%s", *reason_r); + return 0; + } else if (errno == ESTALE) { + /* log file was deleted in NFS server, fail silently */ + *reason_r = t_strdup_printf("read() failed: %m"); + return 0; + } else { + log_file_set_syscall_error(file, "pread()"); + *reason_r = t_strdup_printf("read() failed: %m"); + return -1; + } +} + +static int +mail_transaction_log_file_read_more(struct mail_transaction_log_file *file, + const char **reason_r) +{ + void *data; + size_t size; + uint32_t read_offset; + ssize_t ret; + + read_offset = file->buffer_offset + file->buffer->used; + + do { + data = buffer_append_space_unsafe(file->buffer, LOG_PREFETCH); + ret = pread(file->fd, data, LOG_PREFETCH, read_offset); + if (ret > 0) + read_offset += ret; + + size = read_offset - file->buffer_offset; + buffer_set_used_size(file->buffer, size); + } while (ret > 0 || (ret < 0 && errno == EINTR)); + + file->last_size = read_offset; + + if (ret < 0) { + *reason_r = t_strdup_printf("pread() failed: %m"); + if (errno == ESTALE) { + /* log file was deleted in NFS server, fail silently */ + return 0; + } + log_file_set_syscall_error(file, "pread()"); + return -1; + } + return 1; +} + +static bool +mail_transaction_log_file_need_nfs_flush(struct mail_transaction_log_file *file) +{ + const struct mail_index_header *hdr = &file->log->index->map->hdr; + uoff_t max_offset = file->last_size; + + if (file->next != NULL && + file->hdr.file_seq == file->next->hdr.prev_file_seq && + file->next->hdr.prev_file_offset != max_offset) { + /* we already have a newer log file which says that we haven't + synced the entire file. */ + return TRUE; + } + + if (file->hdr.file_seq == hdr->log_file_seq && + max_offset < hdr->log_file_head_offset) + return TRUE; + + return FALSE; +} + +static int +mail_transaction_log_file_read(struct mail_transaction_log_file *file, + uoff_t start_offset, bool nfs_flush, + const char **reason_r) +{ + bool retry; + int ret; + + i_assert(file->mmap_base == NULL); + + /* NFS: if file isn't locked, we're optimistic that we can read enough + data without flushing attribute cache. if after reading we notice + that we really should have read more, flush the cache and try again. + if file is locked, the attribute cache was already flushed when + refreshing the log. */ + if (nfs_flush && + (file->log->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) { + if (!file->locked) + nfs_flush_attr_cache_unlocked(file->filepath); + else + nfs_flush_attr_cache_fd_locked(file->filepath, file->fd); + } + + if (file->buffer != NULL && file->buffer_offset > start_offset) { + /* we have to insert missing data to beginning of buffer */ + ret = mail_transaction_log_file_insert_read(file, start_offset, reason_r); + if (ret <= 0) + return ret; + } + + if (file->buffer == NULL) { + file->buffer = + buffer_create_dynamic(default_pool, LOG_PREFETCH); + file->buffer_offset = start_offset; + } + + if ((ret = mail_transaction_log_file_read_more(file, reason_r)) <= 0) + ; + else if (!nfs_flush && + (file->log->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0 && + mail_transaction_log_file_need_nfs_flush(file)) { + /* we didn't read enough data. flush and try again. */ + return mail_transaction_log_file_read(file, start_offset, TRUE, reason_r); + } else if ((ret = mail_transaction_log_file_sync(file, &retry, reason_r)) == 0) { + i_assert(!retry); /* retry happens only with mmap */ + } + i_assert(file->sync_offset >= file->buffer_offset); + buffer_set_used_size(file->buffer, + file->sync_offset - file->buffer_offset); + return ret; +} + +static bool +log_file_map_check_offsets(struct mail_transaction_log_file *file, + uoff_t start_offset, uoff_t end_offset, + const char **reason_r) +{ + struct stat st, st2; + + if (start_offset > file->sync_offset) { + /* broken start offset */ + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) { + *reason_r = t_strdup_printf( + "%s: start_offset (%"PRIuUOFF_T") > " + "current sync_offset (%"PRIuUOFF_T")", + file->filepath, start_offset, file->sync_offset); + return FALSE; + } + + if (fstat(file->fd, &st) < 0) { + log_file_set_syscall_error(file, "fstat()"); + st.st_size = -1; + } + *reason_r = t_strdup_printf( + "%s: start_offset (%"PRIuUOFF_T") > " + "current sync_offset (%"PRIuUOFF_T"), file size=%"PRIuUOFF_T, + file->filepath, start_offset, file->sync_offset, + st.st_size); + if (stat(file->filepath, &st2) == 0) { + if (st.st_ino != st2.st_ino) { + *reason_r = t_strdup_printf( + "%s, file unexpectedly replaced", *reason_r); + } + } else if (errno == ENOENT) { + *reason_r = t_strdup_printf( + "%s, file unexpectedly deleted", *reason_r); + } else { + log_file_set_syscall_error(file, "stat()"); + } + return FALSE; + } + if (end_offset != UOFF_T_MAX && end_offset > file->sync_offset) { + *reason_r = t_strdup_printf( + "%s: end_offset (%"PRIuUOFF_T") > " + "current sync_offset (%"PRIuUOFF_T")", + file->filepath, start_offset, file->sync_offset); + return FALSE; + } + + return TRUE; +} + +static int +mail_transaction_log_file_mmap(struct mail_transaction_log_file *file, + const char **reason_r) +{ + /* we may have switched to mmaping */ + buffer_free(&file->buffer); + + file->mmap_size = file->last_size; + file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, MAP_SHARED, + file->fd, 0); + if (file->mmap_base == MAP_FAILED) { + file->mmap_base = NULL; + if (ioloop_time != file->last_mmap_error_time) { + file->last_mmap_error_time = ioloop_time; + log_file_set_syscall_error(file, t_strdup_printf( + "mmap(size=%zu)", file->mmap_size)); + } + *reason_r = t_strdup_printf("mmap(size=%zu) failed: %m", + file->mmap_size); + file->mmap_size = 0; + return -1; + } + + if (file->mmap_size > mmap_get_page_size()) { + if (madvise(file->mmap_base, file->mmap_size, + MADV_SEQUENTIAL) < 0) + log_file_set_syscall_error(file, "madvise()"); + } + + buffer_create_from_const_data(&file->mmap_buffer, + file->mmap_base, file->mmap_size); + file->buffer = &file->mmap_buffer; + file->buffer_offset = 0; + return 0; +} + +static void +mail_transaction_log_file_munmap(struct mail_transaction_log_file *file) +{ + if (file->mmap_base == NULL) + return; + + i_assert(file->buffer != NULL); + if (munmap(file->mmap_base, file->mmap_size) < 0) + log_file_set_syscall_error(file, "munmap()"); + file->mmap_base = NULL; + file->mmap_size = 0; + buffer_free(&file->buffer); +} + +static int +mail_transaction_log_file_map_mmap(struct mail_transaction_log_file *file, + uoff_t start_offset, const char **reason_r) +{ + struct stat st; + bool retry; + int ret; + + /* we are going to mmap() this file, but it's not necessarily + mmaped currently. */ + i_assert(file->buffer_offset == 0 || file->mmap_base == NULL); + i_assert(file->mmap_size == 0 || file->mmap_base != NULL); + + if (fstat(file->fd, &st) < 0) { + log_file_set_syscall_error(file, "fstat()"); + *reason_r = t_strdup_printf("fstat() failed: %m"); + return -1; + } + file->last_size = st.st_size; + + if ((uoff_t)st.st_size < file->sync_offset) { + *reason_r = t_strdup_printf( + "file size shrank (%"PRIuUOFF_T" < %"PRIuUOFF_T")", + (uoff_t)st.st_size, file->sync_offset); + mail_transaction_log_file_set_corrupted(file, "%s", *reason_r); + return 0; + } + + if (file->buffer != NULL && file->buffer_offset <= start_offset && + (uoff_t)st.st_size == file->buffer_offset + file->buffer->used) { + /* we already have the whole file mapped */ + if ((ret = mail_transaction_log_file_sync(file, &retry, reason_r)) != 0 || + !retry) + return ret; + /* size changed, re-mmap */ + } + + do { + mail_transaction_log_file_munmap(file); + + if (file->last_size - start_offset < mmap_get_page_size()) { + /* just reading the file is probably faster */ + return mail_transaction_log_file_read(file, + start_offset, + FALSE, reason_r); + } + + if (mail_transaction_log_file_mmap(file, reason_r) < 0) + return -1; + ret = mail_transaction_log_file_sync(file, &retry, reason_r); + } while (retry); + + return ret; +} + +int mail_transaction_log_file_map(struct mail_transaction_log_file *file, + uoff_t start_offset, uoff_t end_offset, + const char **reason_r) +{ + uoff_t map_start_offset = start_offset; + size_t size; + int ret; + + if (file->hdr.indexid == 0) { + /* corrupted */ + *reason_r = "corrupted, indexid=0"; + return 0; + } + + i_assert(start_offset >= file->hdr.hdr_size); + i_assert(start_offset <= end_offset); + i_assert(file->buffer == NULL || file->mmap_base != NULL || + file->sync_offset >= file->buffer_offset + file->buffer->used); + + if (file->locked_sync_offset_updated && file == file->log->head && + end_offset == UOFF_T_MAX) { + /* we're not interested of going further than sync_offset */ + if (!log_file_map_check_offsets(file, start_offset, + end_offset, reason_r)) + return 0; + i_assert(start_offset <= file->sync_offset); + end_offset = file->sync_offset; + } + + if (file->buffer != NULL && file->buffer_offset <= start_offset) { + /* see if we already have it */ + size = file->buffer->used; + if (file->buffer_offset + size >= end_offset) + return 1; + } + + if (file->locked) { + /* set this only when we've synced to end of file while locked + (either end_offset=UOFF_T_MAX or we had to read anyway) */ + file->locked_sync_offset_updated = TRUE; + } + + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) { + if (start_offset < file->buffer_offset || file->buffer == NULL) { + /* we had moved the log to memory but failed to read + the beginning of the log file */ + *reason_r = "Beginning of the log isn't available"; + return 0; + } + return log_file_map_check_offsets(file, start_offset, + end_offset, reason_r) ? 1 : 0; + } + + if (start_offset > file->sync_offset) + mail_transaction_log_file_skip_to_head(file); + if (start_offset > file->sync_offset) { + /* although we could just skip over the unwanted data, we have + to sync everything so that modseqs are calculated + correctly */ + map_start_offset = file->sync_offset; + } + + if ((file->log->index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0) + ret = mail_transaction_log_file_map_mmap(file, map_start_offset, reason_r); + else { + mail_transaction_log_file_munmap(file); + ret = mail_transaction_log_file_read(file, map_start_offset, FALSE, reason_r); + } + + i_assert(file->buffer == NULL || file->mmap_base != NULL || + file->sync_offset >= file->buffer_offset + file->buffer->used); + if (ret <= 0) + return ret; + + i_assert(file->buffer != NULL); + return log_file_map_check_offsets(file, start_offset, end_offset, + reason_r) ? 1 : 0; +} + +int mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file *file) +{ + const char *error; + buffer_t *buf; + int ret = 0; + + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) + return 0; + + if (file->mmap_base != NULL) { + /* just copy to memory */ + i_assert(file->buffer_offset == 0); + + buf = buffer_create_dynamic(default_pool, file->mmap_size); + buffer_append(buf, file->mmap_base, file->mmap_size); + buffer_free(&file->buffer); + file->buffer = buf; + + /* and lose the mmap */ + if (munmap(file->mmap_base, file->mmap_size) < 0) + log_file_set_syscall_error(file, "munmap()"); + file->mmap_base = NULL; + } else if (file->buffer_offset != 0) { + /* we don't have the full log in the memory. read it. */ + ret = mail_transaction_log_file_read(file, 0, FALSE, &error); + if (ret <= 0) { + mail_index_set_error(file->log->index, + "%s: Failed to read into memory: %s", file->filepath, error); + } + } + file->last_size = 0; + + if (close(file->fd) < 0) + log_file_set_syscall_error(file, "close()"); + file->fd = -1; + + i_free(file->filepath); + file->filepath = i_strdup(file->log->filepath); + return ret < 0 ? -1 : 0; +} diff --git a/src/lib-index/mail-transaction-log-modseq.c b/src/lib-index/mail-transaction-log-modseq.c new file mode 100644 index 0000000..eb2b533 --- /dev/null +++ b/src/lib-index/mail-transaction-log-modseq.c @@ -0,0 +1,298 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log-private.h" + +static struct modseq_cache * +modseq_cache_hit(struct mail_transaction_log_file *file, unsigned int idx) +{ + struct modseq_cache cache; + + if (idx > 0) { + /* @UNSAFE: move it to top */ + cache = file->modseq_cache[idx]; + memmove(file->modseq_cache + 1, file->modseq_cache, + sizeof(*file->modseq_cache) * idx); + file->modseq_cache[0] = cache; + } + return &file->modseq_cache[0]; +} + +static struct modseq_cache * +modseq_cache_get_offset(struct mail_transaction_log_file *file, uoff_t offset) +{ + unsigned int i, best = UINT_MAX; + + for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) { + if (offset < file->modseq_cache[i].offset) + continue; + + if (file->modseq_cache[i].offset == 0) + return NULL; + + if (offset == file->modseq_cache[i].offset) { + /* exact cache hit */ + return modseq_cache_hit(file, i); + } + + if (best == UINT_MAX || + file->modseq_cache[i].offset < + file->modseq_cache[best].offset) + best = i; + } + if (best == UINT_MAX) + return NULL; + return &file->modseq_cache[best]; +} + +static struct modseq_cache * +modseq_cache_get_modseq(struct mail_transaction_log_file *file, uint64_t modseq) +{ + unsigned int i, best = UINT_MAX; + + for (i = 0; i < N_ELEMENTS(file->modseq_cache); i++) { + if (modseq < file->modseq_cache[i].highest_modseq) + continue; + + if (file->modseq_cache[i].offset == 0) + return NULL; + + if (modseq == file->modseq_cache[i].highest_modseq) { + /* exact cache hit */ + return modseq_cache_hit(file, i); + } + + if (best == UINT_MAX || + file->modseq_cache[i].highest_modseq < + file->modseq_cache[best].highest_modseq) + best = i; + } + if (best == UINT_MAX) + return NULL; + return &file->modseq_cache[best]; +} + +static int +log_get_synced_record(struct mail_transaction_log_file *file, uoff_t *offset, + const struct mail_transaction_header **hdr_r, + const char **error_r) +{ + const struct mail_transaction_header *hdr; + uint32_t trans_size; + + hdr = CONST_PTR_OFFSET(file->buffer->data, + *offset - file->buffer_offset); + + /* we've already synced this record at some point. it should + be valid. */ + trans_size = mail_index_offset_to_uint32(hdr->size); + if (trans_size < sizeof(*hdr) || + *offset - file->buffer_offset + trans_size > file->buffer->used) { + *error_r = t_strdup_printf( + "Transaction log corrupted unexpectedly at " + "%"PRIuUOFF_T": Invalid size %u (type=%x)", + *offset, trans_size, hdr->type); + mail_transaction_log_file_set_corrupted(file, "%s", *error_r); + return -1; + } + *offset += trans_size; + *hdr_r = hdr; + return 0; +} + +int mail_transaction_log_file_get_highest_modseq_at( + struct mail_transaction_log_file *file, + uoff_t offset, uint64_t *highest_modseq_r, + const char **error_r) +{ + const struct mail_transaction_header *hdr; + struct modseq_cache *cache; + uoff_t cur_offset; + uint64_t cur_modseq; + const char *reason; + int ret; + + i_assert(offset <= file->sync_offset); + + if (offset == file->sync_offset) { + *highest_modseq_r = file->sync_highest_modseq; + return 1; + } + + cache = modseq_cache_get_offset(file, offset); + if (cache == NULL) { + /* nothing usable in cache - scan from beginning */ + cur_offset = file->hdr.hdr_size; + cur_modseq = file->hdr.initial_modseq; + } else if (cache->offset == offset) { + /* exact cache hit */ + *highest_modseq_r = cache->highest_modseq; + return 1; + } else { + /* use cache to skip over some records */ + cur_offset = cache->offset; + cur_modseq = cache->highest_modseq; + } + + /* See if we can use the "modseq" header in dovecot.index to further + reduce how much we have to scan. */ + const struct mail_index_modseq_header *modseq_hdr = + file->log->index->map == NULL ? NULL : + &file->log->index->map->modseq_hdr_snapshot; + if (modseq_hdr != NULL && + modseq_hdr->log_seq == file->hdr.file_seq && + modseq_hdr->log_offset <= offset && + modseq_hdr->log_offset >= cur_offset) { + cur_offset = modseq_hdr->log_offset; + cur_modseq = modseq_hdr->highest_modseq; + } + + ret = mail_transaction_log_file_map(file, cur_offset, offset, &reason); + if (ret <= 0) { + *error_r = t_strdup_printf( + "Failed to map transaction log %s for getting modseq " + "at offset=%"PRIuUOFF_T" with start_offset=%"PRIuUOFF_T": %s", + file->filepath, offset, cur_offset, reason); + return ret; + } + + i_assert(cur_offset >= file->buffer_offset); + i_assert(cur_offset + file->buffer->used >= offset); + while (cur_offset < offset) { + if (log_get_synced_record(file, &cur_offset, &hdr, error_r) < 0) + return 0; + mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq, + MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr)); + } + + /* @UNSAFE: cache the value */ + memmove(file->modseq_cache + 1, file->modseq_cache, + sizeof(*file->modseq_cache) * + (N_ELEMENTS(file->modseq_cache) - 1)); + file->modseq_cache[0].offset = cur_offset; + file->modseq_cache[0].highest_modseq = cur_modseq; + + *highest_modseq_r = cur_modseq; + return 1; +} + +static int +get_modseq_next_offset_at(struct mail_transaction_log_file *file, + uint64_t modseq, bool use_highest, + uoff_t *cur_offset, uint64_t *cur_modseq, + uoff_t *next_offset_r) +{ + const struct mail_transaction_header *hdr; + const char *reason; + int ret; + + /* make sure we've read until end of file. this is especially important + with non-head logs which might only have been opened without being + synced. */ + ret = mail_transaction_log_file_map(file, *cur_offset, UOFF_T_MAX, &reason); + if (ret <= 0) { + mail_index_set_error(file->log->index, + "Failed to map transaction log %s for getting offset " + "for modseq=%"PRIu64" with start_offset=%"PRIuUOFF_T": %s", + file->filepath, modseq, *cur_offset, reason); + return -1; + } + + /* check sync_highest_modseq again in case sync_offset was updated */ + if (modseq >= file->sync_highest_modseq && use_highest) { + *next_offset_r = file->sync_offset; + return 0; + } + + i_assert(*cur_offset >= file->buffer_offset); + while (*cur_offset < file->sync_offset) { + if (log_get_synced_record(file, cur_offset, &hdr, &reason) < 0) { + mail_index_set_error(file->log->index, + "%s: %s", file->filepath, reason); + return -1; + } + mail_transaction_update_modseq(hdr, hdr + 1, cur_modseq, + MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr)); + if (*cur_modseq >= modseq) + break; + } + return 1; +} + +int mail_transaction_log_file_get_modseq_next_offset( + struct mail_transaction_log_file *file, + uint64_t modseq, uoff_t *next_offset_r) +{ + struct modseq_cache *cache; + uoff_t cur_offset; + uint64_t cur_modseq; + int ret; + + if (modseq == file->sync_highest_modseq) { + *next_offset_r = file->sync_offset; + return 0; + } + if (modseq == file->hdr.initial_modseq) { + *next_offset_r = file->hdr.hdr_size; + return 0; + } + + cache = modseq_cache_get_modseq(file, modseq); + if (cache == NULL) { + /* nothing usable in cache - scan from beginning */ + cur_offset = file->hdr.hdr_size; + cur_modseq = file->hdr.initial_modseq; + } else if (cache->highest_modseq == modseq) { + /* exact cache hit */ + *next_offset_r = cache->offset; + return 0; + } else { + /* use cache to skip over some records */ + cur_offset = cache->offset; + cur_modseq = cache->highest_modseq; + } + + if ((ret = get_modseq_next_offset_at(file, modseq, TRUE, &cur_offset, + &cur_modseq, next_offset_r)) <= 0) + return ret; + if (cur_offset == file->sync_offset) { + /* if we got to sync_offset, cur_modseq should be + sync_highest_modseq */ + mail_index_set_error(file->log->index, + "%s: Transaction log modseq tracking is corrupted - fixing", + file->filepath); + /* retry getting the offset by reading from the beginning + of the file */ + cur_offset = file->hdr.hdr_size; + cur_modseq = file->hdr.initial_modseq; + ret = get_modseq_next_offset_at(file, modseq, FALSE, + &cur_offset, &cur_modseq, + next_offset_r); + if (ret < 0) + return -1; + i_assert(ret != 0); + /* get it fixed on the next sync */ + if (file->log->index->need_recreate == NULL) { + file->log->index->need_recreate = + i_strdup("modseq tracking is corrupted"); + } + if (file->need_rotate == NULL) { + file->need_rotate = + i_strdup("modseq tracking is corrupted"); + } + /* clear cache, since it's unreliable */ + memset(file->modseq_cache, 0, sizeof(file->modseq_cache)); + } + + /* @UNSAFE: cache the value */ + memmove(file->modseq_cache + 1, file->modseq_cache, + sizeof(*file->modseq_cache) * + (N_ELEMENTS(file->modseq_cache) - 1)); + file->modseq_cache[0].offset = cur_offset; + file->modseq_cache[0].highest_modseq = cur_modseq; + + *next_offset_r = cur_offset; + return 0; +} diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h new file mode 100644 index 0000000..4961fe0 --- /dev/null +++ b/src/lib-index/mail-transaction-log-private.h @@ -0,0 +1,199 @@ +#ifndef MAIL_TRANSACTION_LOG_VIEW_H +#define MAIL_TRANSACTION_LOG_VIEW_H + +#include "buffer.h" +#include "mail-transaction-log.h" + +struct dotlock_settings; + +/* Synchronization can take a while sometimes, especially when copying lots of + mails. */ +#define MAIL_TRANSACTION_LOG_LOCK_TIMEOUT (3*60) +#define MAIL_TRANSACTION_LOG_DOTLOCK_CHANGE_TIMEOUT (3*60) + +#define MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file) ((file)->fd == -1) + +#define LOG_FILE_MODSEQ_CACHE_SIZE 10 + +struct modseq_cache { + uoff_t offset; + uint64_t highest_modseq; +}; + +struct mail_transaction_log_file { + struct mail_transaction_log *log; + /* Next file in the mail_transaction_log.files list. Sorted by + hdr.file_seq. */ + struct mail_transaction_log_file *next; + + /* refcount=0 is a valid state. files start that way, and they're + freed only when mail_transaction_logs_clean() is called. */ + int refcount; + + char *filepath; + int fd; + + /* Cached values for last stat()/fstat() */ + ino_t st_ino; + dev_t st_dev; + time_t last_mtime; + uoff_t last_size; + + /* Used to avoid logging mmap() errors too rapidly. */ + time_t last_mmap_error_time; + /* If non-NULL, the log file should be rotated. The string contains a + human-readable reason why the rotation was requested. */ + char *need_rotate; + + /* Copy of the log file header. Set when opened. */ + struct mail_transaction_log_header hdr; + /* Buffer that points to mmap_base */ + buffer_t mmap_buffer; + /* Buffer that can be used to access the log file contents. Either + points to mmap_buffer, or it's a copy of the file contents starting + from buffer_offset. */ + buffer_t *buffer; + /* Offset to log where the buffer starts from. 0 with mmaped log. */ + uoff_t buffer_offset; + /* If non-NULL, mmap()ed log file */ + void *mmap_base; + size_t mmap_size; + + /* Offset to log file how far it's been read. Usually it's the same + as the log file size. However, if the last multi-record transaction + wasn't fully written (or is in the middle of being written), this + points to the beginning of the MAIL_TRANSACTION_BOUNDARY record. */ + uoff_t sync_offset; + /* highest modseq at sync_offset */ + uint64_t sync_highest_modseq; + /* The last mail_index_header.log_file_tail_offset update that was + read from the log. */ + uoff_t last_read_hdr_tail_offset; + /* Update mail_index_header.log_file_tail_offset to this offset the + next time a transaction is written. Transaction log handling may + increase this automatically by making it skip external transactions + after last_read_hdr_tail_offset (to avoid re-reading them + needlessly). */ + uoff_t max_tail_offset; + + /* Last seen offsets for MAIL_TRANSACTION_INDEX_DELETED and + MAIL_TRANSACTION_INDEX_UNDELETED records. These are used to update + mail_index.index_delete* fields. */ + uoff_t index_deleted_offset, index_undeleted_offset; + + /* Cache to optimize mail_transaction_log_file_get_modseq_next_offset() + so it doesn't always have to start from the beginning of the log + file to find the wanted modseq. */ + struct modseq_cache modseq_cache[LOG_FILE_MODSEQ_CACHE_SIZE]; + + /* Lock for the log file fd. If dotlocking is used, this is NULL and + mail_transaction_log.dotlock is used instead. */ + struct file_lock *file_lock; + /* Time when the log was successfully locked */ + time_t lock_create_time; + + /* Log is currently locked. */ + bool locked:1; + /* TRUE if sync_offset has already been updated while this log was + locked. This can be used to optimize away unnecessary checks to see + whether there's more data written to log after sync_offset. */ + bool locked_sync_offset_updated:1; + /* Log file has found to be corrupted. Stop trying to read it. + The indexid is also usually overwritten to be 0 in the log header at + this time. */ + bool corrupted:1; +}; + +struct mail_transaction_log { + struct mail_index *index; + /* Linked list of all transaction log views */ + struct mail_transaction_log_view *views; + /* Paths to .log and .log.2 */ + char *filepath, *filepath2; + + /* Linked list of all the opened log files. The oldest files may have + already been unlinked. The list is sorted by the log file sequence + (oldest/lowest first), so that transaction views can use them + easily. */ + struct mail_transaction_log_file *files; + /* Latest log file (the last file in the files linked list) */ + struct mail_transaction_log_file *head; + /* open_file is used temporarily while opening the log file. + if mail_transaction_log_open() failed, it's left there for + mail_transaction_log_create(). */ + struct mail_transaction_log_file *open_file; + + /* Normally the .log locking is done via their file descriptors, so + e.g. rotating a log needs to lock both the old and the new files + at the same time. However, when FILE_LOCK_METHOD_DOTLOCK is used, + the lock isn't file-specific. There is just a single dotlock that + is created by the first log file lock. The second lock simply + increases the refcount. (It's not expected that there would be more + than 2 locks.) */ + int dotlock_refcount; + struct dotlock *dotlock; + + /* This session has already checked whether an old .log.2 should be + unlinked. */ + bool log_2_unlink_checked:1; +}; + +void +mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file, + const char *fmt, ...) + ATTR_FORMAT(2, 3) ATTR_COLD; + +void mail_transaction_log_get_dotlock_set(struct mail_transaction_log *log, + struct dotlock_settings *set_r); + +struct mail_transaction_log_file * +mail_transaction_log_file_alloc_in_memory(struct mail_transaction_log *log); +struct mail_transaction_log_file * +mail_transaction_log_file_alloc(struct mail_transaction_log *log, + const char *path); +void mail_transaction_log_file_free(struct mail_transaction_log_file **file); + +/* Returns 1 if log was opened, 0 if it didn't exist or was already open, + -1 if error. */ +int mail_transaction_log_file_open(struct mail_transaction_log_file *file, + const char **reason_r); +int mail_transaction_log_file_create(struct mail_transaction_log_file *file, + bool reset); +int mail_transaction_log_file_lock(struct mail_transaction_log_file *file); + +int mail_transaction_log_find_file(struct mail_transaction_log *log, + uint32_t file_seq, bool nfs_flush, + struct mail_transaction_log_file **file_r, + const char **reason_r); + +/* Returns 1 if ok, 0 if file is corrupted or offset range is invalid, + -1 if I/O error */ +int mail_transaction_log_file_map(struct mail_transaction_log_file *file, + uoff_t start_offset, uoff_t end_offset, + const char **reason_r); +int mail_transaction_log_file_move_to_memory(struct mail_transaction_log_file *file); + +void mail_transaction_logs_clean(struct mail_transaction_log *log); + +bool mail_transaction_log_want_rotate(struct mail_transaction_log *log, + const char **reason_r); +int mail_transaction_log_rotate(struct mail_transaction_log *log, bool reset); +int mail_transaction_log_lock_head(struct mail_transaction_log *log, + const char *lock_reason); +void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file, + const char *lock_reason); + +void mail_transaction_update_modseq(const struct mail_transaction_header *hdr, + const void *data, uint64_t *cur_modseq, + unsigned int version); +/* Returns 1 if ok, 0 if file is corrupted or offset range is invalid, + -1 if I/O error */ +int mail_transaction_log_file_get_highest_modseq_at( + struct mail_transaction_log_file *file, + uoff_t offset, uint64_t *highest_modseq_r, + const char **error_r); +int mail_transaction_log_file_get_modseq_next_offset( + struct mail_transaction_log_file *file, + uint64_t modseq, uoff_t *next_offset_r); + +#endif diff --git a/src/lib-index/mail-transaction-log-view-private.h b/src/lib-index/mail-transaction-log-view-private.h new file mode 100644 index 0000000..e739609 --- /dev/null +++ b/src/lib-index/mail-transaction-log-view-private.h @@ -0,0 +1,33 @@ +#ifndef MAIL_TRANSACTION_LOG_VIEW_PRIVATE_H +#define MAIL_TRANSACTION_LOG_VIEW_PRIVATE_H + +#include "mail-transaction-log-private.h" + +struct mail_transaction_log_view { + struct mail_transaction_log *log; + struct mail_transaction_log_view *next; + + uint32_t min_file_seq, max_file_seq; + uoff_t min_file_offset, max_file_offset; + + struct mail_transaction_header tmp_hdr; + + /* a list of log files we've referenced. we have to keep this list + explicitly because more files may be added into the linked list + at any time. */ + ARRAY(struct mail_transaction_log_file *) file_refs; + struct mail_transaction_log_file *cur, *head, *tail; + uoff_t cur_offset; + + uint64_t prev_modseq; + uint32_t prev_file_seq; + uoff_t prev_file_offset; + + struct mail_transaction_log_file *mark_file; + uoff_t mark_offset, mark_next_offset; + uint64_t mark_modseq; + + bool broken:1; +}; + +#endif diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c new file mode 100644 index 0000000..0113bfb --- /dev/null +++ b/src/lib-index/mail-transaction-log-view.c @@ -0,0 +1,909 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "mail-index-private.h" +#include "mail-transaction-log-view-private.h" + +struct mail_transaction_log_view * +mail_transaction_log_view_open(struct mail_transaction_log *log) +{ + struct mail_transaction_log_view *view; + + view = i_new(struct mail_transaction_log_view, 1); + view->log = log; + view->broken = TRUE; + + i_assert(view->log->head != NULL); + + view->head = view->tail = view->log->head; + view->head->refcount++; + i_array_init(&view->file_refs, 8); + array_push_back(&view->file_refs, &view->head); + + view->next = log->views; + log->views = view; + return view; +} + +static void +mail_transaction_log_view_unref_all(struct mail_transaction_log_view *view) +{ + struct mail_transaction_log_file *const *files; + unsigned int i, count; + + files = array_get(&view->file_refs, &count); + for (i = 0; i < count; i++) + files[i]->refcount--; + + array_clear(&view->file_refs); +} + +void mail_transaction_log_view_close(struct mail_transaction_log_view **_view) +{ + struct mail_transaction_log_view *view = *_view; + struct mail_transaction_log_view **p; + + *_view = NULL; + + for (p = &view->log->views; *p != NULL; p = &(*p)->next) { + if (*p == view) { + *p = view->next; + break; + } + } + + mail_transaction_log_view_unref_all(view); + mail_transaction_logs_clean(view->log); + + array_free(&view->file_refs); + i_free(view); +} + +static const char * +mail_transaction_log_get_file_seqs(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + string_t *str = t_str_new(32); + + if (log->files == NULL) + return ""; + + for (file = log->files; file != NULL; file = file->next) + str_printfa(str, ",%u", file->hdr.file_seq); + return str_c(str) + 1; +} + +static void +view_set_failed_unref(struct mail_transaction_log_file *head, + struct mail_transaction_log_file *tail) +{ + struct mail_transaction_log_file *file; + + if (tail == NULL) { + i_assert(head == NULL); + return; + } + + for (file = tail; file != head; file = file->next) { + i_assert(file != NULL); + i_assert(file->refcount > 0); + file->refcount--; + } + i_assert(file != NULL); + i_assert(file->refcount > 0); + file->refcount--; +} + +int mail_transaction_log_view_set(struct mail_transaction_log_view *view, + uint32_t min_file_seq, uoff_t min_file_offset, + uint32_t max_file_seq, uoff_t max_file_offset, + bool *reset_r, const char **reason_r) +{ + struct mail_transaction_log_file *tail, *head, *file, *const *files; + uoff_t start_offset, end_offset; + unsigned int i; + uint32_t seq; + int ret; + + *reset_r = FALSE; + *reason_r = NULL; + + if (view->log == NULL) { + /* transaction log is closed already. this log view shouldn't + be used anymore. */ + *reason_r = "Log already closed"; + return 0; + } + + if (min_file_seq == 0) { + /* index file doesn't exist yet. this transaction log should + start from the beginning */ + if (view->log->files->hdr.prev_file_seq != 0) { + /* but it doesn't */ + *reason_r = t_strdup_printf( + "Wanted log beginning, but found prev_file_seq=%u", + view->log->files->hdr.prev_file_seq); + return 0; + } + + min_file_seq = view->log->files->hdr.file_seq; + min_file_offset = 0; + + if (max_file_seq == 0) { + max_file_seq = min_file_seq; + max_file_offset = min_file_offset; + } + } + + for (file = view->log->files; file != NULL; file = file->next) { + if (file->hdr.prev_file_seq == min_file_seq) + break; + } + + if (file != NULL && min_file_offset == file->hdr.prev_file_offset) { + /* we can (and sometimes must) skip to the next file */ + min_file_seq = file->hdr.file_seq; + min_file_offset = file->hdr.hdr_size; + } + + for (file = view->log->files; file != NULL; file = file->next) { + if (file->hdr.prev_file_seq == max_file_seq) + break; + } + if (file != NULL && max_file_offset == file->hdr.prev_file_offset) { + /* we can skip to the next file. we've delayed checking for + min_file_seq <= max_file_seq until now, because it's not + really an error to specify the same position twice (even if + in "wrong" order) */ + i_assert(min_file_seq <= max_file_seq || + min_file_seq <= file->hdr.file_seq); + max_file_seq = file->hdr.file_seq; + max_file_offset = file->hdr.hdr_size; + } else { + i_assert(min_file_seq <= max_file_seq); + } + + if (min_file_seq == max_file_seq && min_file_offset > max_file_offset) { + /* log file offset is probably corrupted in the index file. */ + *reason_r = t_strdup_printf( + "Invalid offset: file_seq=%u, min_file_offset (%"PRIuUOFF_T + ") > max_file_offset (%"PRIuUOFF_T")", + min_file_seq, min_file_offset, max_file_offset); + mail_transaction_log_view_set_corrupted(view, "%s", *reason_r); + return 0; + } + + tail = head = file = NULL; + for (seq = min_file_seq; seq <= max_file_seq; seq++) { + const char *reason = NULL; + + if (file == NULL || file->hdr.file_seq != seq) { + /* see if we could find the missing file. if we know + the max. file sequence or we don't have the the min. + file, make sure NFS attribute cache gets flushed if + necessary. */ + bool nfs_flush = seq == min_file_seq || + max_file_seq != (uint32_t)-1; + + ret = mail_transaction_log_find_file(view->log, seq, + nfs_flush, &file, &reason); + if (ret <= 0) { + if (ret < 0) { + *reason_r = t_strdup_printf( + "Failed to find file seq=%u: %s", + seq, reason); + view_set_failed_unref(head, tail); + return -1; + } + + /* not found / corrupted */ + file = NULL; + } + } + + if (file == NULL || file->hdr.file_seq != seq) { + i_assert(reason != NULL); + if (file == NULL && max_file_seq == (uint32_t)-1 && + head == view->log->head) { + /* we just wanted to sync everything */ + i_assert(max_file_offset == UOFF_T_MAX); + max_file_seq = seq-1; + break; + } + /* if any of the found files reset the index, + ignore any missing files up to it */ + file = tail != NULL ? tail : view->log->files; + for (;; file = file->next) { + if (file == NULL || + file->hdr.file_seq > max_file_seq) { + /* missing files in the middle */ + *reason_r = t_strdup_printf( + "Missing middle file seq=%u (between %u..%u, we have seqs %s): %s", + seq, min_file_seq, max_file_seq, + mail_transaction_log_get_file_seqs(view->log), reason); + view_set_failed_unref(head, tail); + return 0; + } + + if (file->hdr.file_seq >= seq && + file->hdr.prev_file_seq == 0) { + /* we can ignore the missing file */ + break; + } + } + /* we're going to rebuild the head/tail. remove the old + references first. */ + view_set_failed_unref(head, tail); + seq = file->hdr.file_seq; + tail = NULL; + } + + if (tail == NULL) + tail = file; + head = file; + /* NOTE: we need to reference immediately or it could become + freed by mail_transaction_log_find_file() */ + file->refcount++; + file = file->next; + } + i_assert(tail != NULL); + + if (min_file_offset == 0) { + /* beginning of the file */ + min_file_offset = tail->hdr.hdr_size; + if (min_file_offset > max_file_offset && + min_file_seq == max_file_seq) { + /* we don't actually want to show anything */ + max_file_offset = min_file_offset; + } + } + + if (min_file_offset < tail->hdr.hdr_size) { + /* log file offset is probably corrupted in the index file. */ + *reason_r = t_strdup_printf( + "Invalid min_file_offset: file_seq=%u, min_file_offset (%"PRIuUOFF_T + ") < hdr_size (%u)", + min_file_seq, min_file_offset, tail->hdr.hdr_size); + mail_transaction_log_view_set_corrupted(view, "%s", *reason_r); + view_set_failed_unref(head, tail); + return 0; + } + if (max_file_offset < head->hdr.hdr_size) { + /* log file offset is probably corrupted in the index file. */ + *reason_r = t_strdup_printf( + "Invalid max_file_offset: file_seq=%u, min_file_offset (%"PRIuUOFF_T + ") < hdr_size (%u)", + max_file_seq, max_file_offset, head->hdr.hdr_size); + mail_transaction_log_view_set_corrupted(view, "%s", *reason_r); + view_set_failed_unref(head, tail); + return 0; + } + + /* we have all of them. update refcounts. */ + mail_transaction_log_view_unref_all(view); + + /* Reference all used files. */ + view->tail = tail; + view->head = head; + for (file = view->tail; ; file = file->next) { + array_push_back(&view->file_refs, &file); + + if (file == head) + break; + } + + view->cur = view->tail; + view->cur_offset = view->cur->hdr.file_seq == min_file_seq ? + min_file_offset : view->cur->hdr.hdr_size; + + /* Map the files only after we've found them all. Otherwise if we map + one file and then another file just happens to get rotated, we could + include both files in the view but skip the last transactions from + the first file. + + We're mapping the files in reverse order so that _log_file_map() + can verify that prev_file_offset matches how far it actually managed + to sync the file. */ + files = array_front(&view->file_refs); + for (i = array_count(&view->file_refs); i > 0; i--) { + file = files[i-1]; + start_offset = file->hdr.file_seq == min_file_seq ? + min_file_offset : file->hdr.hdr_size; + end_offset = file->hdr.file_seq == max_file_seq ? + max_file_offset : UOFF_T_MAX; + ret = mail_transaction_log_file_map(file, start_offset, + end_offset, reason_r); + if (ret <= 0) { + *reason_r = t_strdup_printf( + "Failed to map file seq=%u " + "offset=%"PRIuUOFF_T"..%"PRIuUOFF_T" (ret=%d): %s", + file->hdr.file_seq, start_offset, end_offset, ret, *reason_r); + return ret; + } + + if (file->hdr.prev_file_seq == 0) { + /* this file resets the index. + don't bother reading the others. */ + if (view->cur != file || + view->cur_offset == file->hdr.hdr_size) { + view->cur = file; + view->cur_offset = file->hdr.hdr_size; + *reset_r = TRUE; + break; + } + i_assert(i == 1); + } + } + + if (min_file_seq == view->head->hdr.file_seq && + min_file_offset > view->head->sync_offset) { + /* log file offset is probably corrupted in the index file. */ + *reason_r = t_strdup_printf( + "Invalid offset: file_seq=%u, min_file_offset (%"PRIuUOFF_T + ") > sync_offset (%"PRIuUOFF_T")", min_file_seq, + min_file_offset, view->head->sync_offset); + mail_transaction_log_view_set_corrupted(view, "%s", *reason_r); + return 0; + } + + i_assert(max_file_seq == (uint32_t)-1 || + max_file_seq == view->head->hdr.file_seq); + i_assert(max_file_offset == UOFF_T_MAX || + max_file_offset <= view->head->sync_offset); + i_assert(min_file_seq != max_file_seq || + max_file_seq != view->head->hdr.file_seq || + max_file_offset != UOFF_T_MAX || + min_file_offset <= view->head->sync_offset); + + view->prev_file_seq = view->cur->hdr.file_seq; + view->prev_file_offset = view->cur_offset; + + view->min_file_seq = min_file_seq; + view->min_file_offset = min_file_offset; + view->max_file_seq = max_file_seq; + view->max_file_offset = I_MIN(max_file_offset, view->head->sync_offset); + view->broken = FALSE; + + ret = mail_transaction_log_file_get_highest_modseq_at(view->cur, + view->cur_offset, &view->prev_modseq, reason_r); + if (ret <= 0) + return ret; + + i_assert(view->cur_offset <= view->cur->sync_offset); + return 1; +} + +int mail_transaction_log_view_set_all(struct mail_transaction_log_view *view) +{ + struct mail_transaction_log_file *file, *first; + const char *reason = NULL; + int ret; + + /* make sure .log.2 file is opened */ + (void)mail_transaction_log_find_file(view->log, 1, FALSE, &file, &reason); + + first = view->log->files; + i_assert(first != NULL); + + for (file = view->log->files; file != NULL; file = file->next) { + ret = mail_transaction_log_file_map(file, file->hdr.hdr_size, + UOFF_T_MAX, &reason); + if (ret < 0) { + first = NULL; + break; + } + if (ret == 0) { + /* corrupted */ + first = NULL; + } else if (file->hdr.prev_file_seq == 0) { + /* this file resets the index. skip the old ones. */ + first = file; + } + } + if (first == NULL) { + /* index wasn't reset after corruption was found */ + i_assert(reason != NULL); + mail_index_set_error(view->log->index, + "Failed to map transaction log %s for all-view: %s", + view->log->filepath, reason); + return -1; + } + + mail_transaction_log_view_unref_all(view); + for (file = first; file != NULL; file = file->next) { + array_push_back(&view->file_refs, &file); + file->refcount++; + } + + view->tail = first; + view->cur = view->tail; + view->cur_offset = view->tail->hdr.hdr_size; + + view->prev_file_seq = view->cur->hdr.file_seq; + view->prev_file_offset = view->cur_offset; + + view->min_file_seq = view->cur->hdr.file_seq; + view->min_file_offset = view->cur_offset; + view->max_file_seq = view->head->hdr.file_seq; + view->max_file_offset = view->head->sync_offset; + view->broken = FALSE; + + if (mail_transaction_log_file_get_highest_modseq_at(view->cur, + view->cur_offset, &view->prev_modseq, &reason) <= 0) { + mail_index_set_error(view->log->index, + "Failed to get modseq in %s for all-view: %s", + view->log->filepath, reason); + return -1; + } + return 0; +} + +void mail_transaction_log_view_clear(struct mail_transaction_log_view *view, + uint32_t oldest_file_seq) +{ + struct mail_transaction_log_file *file; + const char *reason; + + mail_transaction_log_view_unref_all(view); + if (oldest_file_seq != 0 && + mail_transaction_log_find_file(view->log, oldest_file_seq, FALSE, + &file, &reason) > 0) { + for (; file != NULL; file = file->next) { + array_push_back(&view->file_refs, &file); + file->refcount++; + } + } + + view->cur = view->head = view->tail = NULL; + + view->mark_file = NULL; + view->mark_offset = 0; + view->mark_modseq = 0; + + view->min_file_seq = view->max_file_seq = 0; + view->min_file_offset = view->max_file_offset = 0; + view->cur_offset = 0; + + view->prev_file_seq = 0; + view->prev_file_offset = 0; + view->prev_modseq = 0; +} + +void +mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view, + uint32_t *file_seq_r, + uoff_t *file_offset_r) +{ + *file_seq_r = view->prev_file_seq; + *file_offset_r = view->prev_file_offset; +} + +uint64_t +mail_transaction_log_view_get_prev_modseq(struct mail_transaction_log_view *view) +{ + return view->prev_modseq; +} + +static bool +mail_transaction_log_view_get_last(struct mail_transaction_log_view *view, + struct mail_transaction_log_file **last_r, + uoff_t *last_offset_r) +{ + struct mail_transaction_log_file *cur = view->cur; + uoff_t cur_offset = view->cur_offset; + bool last = FALSE; + + if (cur == NULL) { + *last_r = NULL; + return TRUE; + } + + for (;;) { + if (cur->hdr.file_seq == view->max_file_seq) { + /* last file */ + if (cur_offset == view->max_file_offset || + cur_offset == cur->sync_offset) { + /* we're all finished */ + last = TRUE; + } + } else if (cur_offset == cur->sync_offset) { + /* end of file, go to next one */ + if (cur->next == NULL) { + last = TRUE; + } else { + cur = cur->next; + cur_offset = cur->hdr.hdr_size; + continue; + } + } + + /* not EOF */ + break; + } + + *last_r = cur; + *last_offset_r = cur_offset; + return last; +} + +bool mail_transaction_log_view_is_last(struct mail_transaction_log_view *view) +{ + struct mail_transaction_log_file *cur; + uoff_t cur_offset; + + return mail_transaction_log_view_get_last(view, &cur, &cur_offset); +} + +void +mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view, + const char *fmt, ...) +{ + va_list va; + + view->broken = TRUE; + + va_start(va, fmt); + T_BEGIN { + mail_transaction_log_file_set_corrupted(view->log->head, "%s", + t_strdup_vprintf(fmt, va)); + } T_END; + va_end(va); +} + +bool +mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view) +{ + return view->broken; +} + +static bool +log_view_is_uid_range_valid(struct mail_transaction_log_file *file, + enum mail_transaction_type rec_type, + const ARRAY_TYPE(seq_range) *uids) +{ + const struct seq_range *rec, *prev = NULL; + unsigned int i, count = array_count(uids); + + if ((uids->arr.buffer->used % uids->arr.element_size) != 0) { + mail_transaction_log_file_set_corrupted(file, + "Invalid record size (type=0x%x)", rec_type); + return FALSE; + } else if (count == 0) { + mail_transaction_log_file_set_corrupted(file, + "No UID ranges (type=0x%x)", rec_type); + return FALSE; + } + + for (i = 0; i < count; i++, prev = rec) { + rec = array_idx(uids, i); + if (rec->seq1 > rec->seq2 || rec->seq1 == 0) { + mail_transaction_log_file_set_corrupted(file, + "Invalid UID range (%u .. %u, type=0x%x)", + rec->seq1, rec->seq2, rec_type); + return FALSE; + } + if (prev != NULL && rec->seq1 <= prev->seq2) { + mail_transaction_log_file_set_corrupted(file, + "Non-sorted UID ranges (type=0x%x)", rec_type); + return FALSE; + } + } + return TRUE; +} + +static bool +log_view_is_record_valid(struct mail_transaction_log_file *file, + const struct mail_transaction_header *hdr, + const void *data) +{ + enum mail_transaction_type rec_type; + ARRAY_TYPE(seq_range) uids = ARRAY_INIT; + buffer_t uid_buf; + uint32_t rec_size; + + rec_type = hdr->type & MAIL_TRANSACTION_TYPE_MASK; + rec_size = mail_index_offset_to_uint32(hdr->size) - sizeof(*hdr); + + /* we want to be extra careful with expunges */ + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) { + if (rec_type != (MAIL_TRANSACTION_EXPUNGE | + MAIL_TRANSACTION_EXPUNGE_PROT)) { + mail_transaction_log_file_set_corrupted(file, + "expunge record missing protection mask"); + return FALSE; + } + rec_type &= ENUM_NEGATE(MAIL_TRANSACTION_EXPUNGE_PROT); + } + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) { + if (rec_type != (MAIL_TRANSACTION_EXPUNGE_GUID | + MAIL_TRANSACTION_EXPUNGE_PROT)) { + mail_transaction_log_file_set_corrupted(file, + "expunge guid record missing protection mask"); + return FALSE; + } + rec_type &= ENUM_NEGATE(MAIL_TRANSACTION_EXPUNGE_PROT); + } + + if (rec_size == 0) { + mail_transaction_log_file_set_corrupted(file, + "Empty record contents (type=0x%x)", rec_type); + return FALSE; + } + + /* records that are exported by syncing and view syncing will be + checked here so that we don't have to implement the same validation + multiple times. other records are checked internally by + mail_index_sync_record(). */ + switch (rec_type) { + case MAIL_TRANSACTION_APPEND: + if ((rec_size % sizeof(struct mail_index_record)) != 0) { + mail_transaction_log_file_set_corrupted(file, + "Invalid append record size"); + return FALSE; + } + break; + case MAIL_TRANSACTION_EXPUNGE: + buffer_create_from_const_data(&uid_buf, data, rec_size); + array_create_from_buffer(&uids, &uid_buf, + sizeof(struct mail_transaction_expunge)); + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: { + const struct mail_transaction_expunge_guid *recs = data; + unsigned int i, count; + + if ((rec_size % sizeof(*recs)) != 0) { + mail_transaction_log_file_set_corrupted(file, + "Invalid expunge guid record size"); + return FALSE; + } + count = rec_size / sizeof(*recs); + for (i = 0; i < count; i++) { + if (recs[i].uid == 0) { + mail_transaction_log_file_set_corrupted(file, + "Expunge guid record with uid=0"); + return FALSE; + } + } + break; + } + case MAIL_TRANSACTION_FLAG_UPDATE: + buffer_create_from_const_data(&uid_buf, data, rec_size); + array_create_from_buffer(&uids, &uid_buf, + sizeof(struct mail_transaction_flag_update)); + break; + case MAIL_TRANSACTION_KEYWORD_UPDATE: { + const struct mail_transaction_keyword_update *rec = data; + unsigned int seqset_offset; + + seqset_offset = sizeof(*rec) + rec->name_size; + if ((seqset_offset % 4) != 0) + seqset_offset += 4 - (seqset_offset % 4); + + if (rec->name_size == 0) { + mail_transaction_log_file_set_corrupted(file, + "Trying to use empty keyword"); + return FALSE; + } + if (seqset_offset > rec_size) { + mail_transaction_log_file_set_corrupted(file, + "Invalid keyword update record size"); + return FALSE; + } + + buffer_create_from_const_data(&uid_buf, + CONST_PTR_OFFSET(data, seqset_offset), + rec_size - seqset_offset); + array_create_from_buffer(&uids, &uid_buf, + sizeof(uint32_t)*2); + break; + } + case MAIL_TRANSACTION_KEYWORD_RESET: + buffer_create_from_const_data(&uid_buf, data, rec_size); + array_create_from_buffer(&uids, &uid_buf, + sizeof(struct mail_transaction_keyword_reset)); + break; + case MAIL_TRANSACTION_EXT_INTRO: { + const struct mail_transaction_ext_intro *rec; + unsigned int i; + + for (i = 0; i < rec_size; ) { + if (i + sizeof(*rec) > rec_size) { + /* should be just extra padding */ + break; + } + + rec = CONST_PTR_OFFSET(data, i); + if (i + sizeof(*rec) + rec->name_size > rec_size) { + mail_transaction_log_file_set_corrupted(file, + "ext intro: name_size too large"); + return FALSE; + } + i += sizeof(*rec) + rec->name_size; + if ((i % 4) != 0) + i += 4 - (i % 4); + } + break; + } + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: { + const char *attr_changes = data; + unsigned int i; + + for (i = 0; i+2 < rec_size && attr_changes[i] != '\0'; ) { + if (attr_changes[i] != '+' && attr_changes[i] != '-') { + mail_transaction_log_file_set_corrupted(file, + "attribute update: Invalid prefix 0x%02x", + attr_changes[i]); + return FALSE; + } + i++; + if (attr_changes[i] != 'p' && attr_changes[i] != 's') { + mail_transaction_log_file_set_corrupted(file, + "attribute update: Invalid type 0x%02x", + attr_changes[i]); + return FALSE; + } + i++; + if (attr_changes[i] == '\0') { + mail_transaction_log_file_set_corrupted(file, + "attribute update: Empty key"); + return FALSE; + } + i += strlen(attr_changes+i) + 1; + } + if (i == 0 || (i < rec_size && attr_changes[i] != '\0')) { + mail_transaction_log_file_set_corrupted(file, + "attribute update doesn't end with NUL"); + return FALSE; + } + break; + } + default: + break; + } + + if (array_is_created(&uids)) { + if (!log_view_is_uid_range_valid(file, rec_type, &uids)) + return FALSE; + } + return TRUE; +} + +static int +log_view_get_next(struct mail_transaction_log_view *view, + const struct mail_transaction_header **hdr_r, + const void **data_r) +{ + const struct mail_transaction_header *hdr; + struct mail_transaction_log_file *file; + const void *data; + enum mail_transaction_type rec_type; + uint32_t full_size; + size_t file_size; + int ret; + + if (view->cur == NULL) + return 0; + + /* prev_file_offset should point to beginning of previous log record. + when we reach EOF, it should be left there, not to beginning of the + next file that's not included inside the view. */ + if (mail_transaction_log_view_get_last(view, &view->cur, + &view->cur_offset)) { + /* if the last file was the beginning of a file, we want to + move prev pointers there */ + view->prev_file_seq = view->cur->hdr.file_seq; + view->prev_file_offset = view->cur_offset; + view->cur = NULL; + return 0; + } + + view->prev_file_seq = view->cur->hdr.file_seq; + view->prev_file_offset = view->cur_offset; + + file = view->cur; + + data = file->buffer->data; + file_size = file->buffer->used + file->buffer_offset; + + if (view->cur_offset + sizeof(*hdr) > file_size) { + mail_transaction_log_file_set_corrupted(file, + "offset points outside file " + "(%"PRIuUOFF_T" + %zu > %zu)", + view->cur_offset, sizeof(*hdr), file_size); + return -1; + } + + i_assert(view->cur_offset >= file->buffer_offset); + hdr = CONST_PTR_OFFSET(data, view->cur_offset - file->buffer_offset); + data = CONST_PTR_OFFSET(hdr, sizeof(*hdr)); + + rec_type = hdr->type & MAIL_TRANSACTION_TYPE_MASK; + full_size = mail_index_offset_to_uint32(hdr->size); + if (full_size < sizeof(*hdr)) { + mail_transaction_log_file_set_corrupted(file, + "record size too small (type=0x%x, " + "offset=%"PRIuUOFF_T", size=%u)", + rec_type, view->cur_offset, full_size); + return -1; + } + + if (file_size - view->cur_offset < full_size) { + mail_transaction_log_file_set_corrupted(file, + "record size too large (type=0x%x, " + "offset=%"PRIuUOFF_T", size=%u, end=%zu)", + rec_type, view->cur_offset, full_size, file_size); + return -1; + } + + T_BEGIN { + ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1; + } T_END; + if (ret > 0) { + mail_transaction_update_modseq(hdr, data, &view->prev_modseq, + MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr)); + *hdr_r = hdr; + *data_r = data; + view->cur_offset += full_size; + } + return ret; +} + +int mail_transaction_log_view_next(struct mail_transaction_log_view *view, + const struct mail_transaction_header **hdr_r, + const void **data_r) +{ + const struct mail_transaction_header *hdr; + const void *data; + int ret = 0; + + if (view->broken) + return -1; + + ret = log_view_get_next(view, &hdr, &data); + if (ret <= 0) { + if (ret < 0) + view->cur_offset = view->cur->sync_offset; + return ret; + } + + /* drop expunge protection */ + if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) == + (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT) || + (hdr->type & MAIL_TRANSACTION_TYPE_MASK) == + (MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT)) + view->tmp_hdr.type = hdr->type & ENUM_NEGATE(MAIL_TRANSACTION_EXPUNGE_PROT); + else + view->tmp_hdr.type = hdr->type; + + /* return record's size */ + view->tmp_hdr.size = mail_index_offset_to_uint32(hdr->size); + i_assert(view->tmp_hdr.size > sizeof(*hdr)); + view->tmp_hdr.size -= sizeof(*hdr); + + *hdr_r = &view->tmp_hdr; + *data_r = data; + return 1; +} + +void mail_transaction_log_view_mark(struct mail_transaction_log_view *view) +{ + i_assert(view->cur->hdr.file_seq == view->prev_file_seq); + + view->mark_file = view->cur; + view->mark_offset = view->prev_file_offset; + view->mark_next_offset = view->cur_offset; + view->mark_modseq = view->prev_modseq; +} + +void mail_transaction_log_view_rewind(struct mail_transaction_log_view *view) +{ + i_assert(view->mark_file != NULL); + + view->cur = view->mark_file; + view->cur_offset = view->mark_next_offset; + view->prev_file_seq = view->cur->hdr.file_seq; + view->prev_file_offset = view->mark_offset; + view->prev_modseq = view->mark_modseq; +} diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c new file mode 100644 index 0000000..6e9b1eb --- /dev/null +++ b/src/lib-index/mail-transaction-log.c @@ -0,0 +1,664 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "buffer.h" +#include "file-dotlock.h" +#include "nfs-workarounds.h" +#include "mmap-util.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" + +#include <stddef.h> +#include <stdio.h> +#include <sys/stat.h> + +static void +mail_transaction_log_set_head(struct mail_transaction_log *log, + struct mail_transaction_log_file *file) +{ + i_assert(log->head != file); + + file->refcount++; + log->head = file; + + i_assert(log->files != NULL); + i_assert(log->files->next != NULL || log->files == file); +} + +struct mail_transaction_log * +mail_transaction_log_alloc(struct mail_index *index) +{ + struct mail_transaction_log *log; + + log = i_new(struct mail_transaction_log, 1); + log->index = index; + return log; +} + +static void mail_transaction_log_2_unlink_old(struct mail_transaction_log *log) +{ + struct stat st; + uint32_t log2_rotate_time = log->index->map->hdr.log2_rotate_time; + + if (MAIL_INDEX_IS_IN_MEMORY(log->index)) + return; + + if (log2_rotate_time == 0) { + if (nfs_safe_stat(log->filepath2, &st) == 0) + log2_rotate_time = st.st_mtime; + else if (errno == ENOENT) + log2_rotate_time = (uint32_t)-1; + else { + mail_index_set_error(log->index, + "stat(%s) failed: %m", log->filepath2); + return; + } + } + + if (log2_rotate_time != (uint32_t)-1 && + ioloop_time - (time_t)log2_rotate_time >= (time_t)log->index->optimization_set.log.log2_max_age_secs && + !log->index->readonly) { + i_unlink_if_exists(log->filepath2); + log2_rotate_time = (uint32_t)-1; + } + + if (log2_rotate_time != log->index->map->hdr.log2_rotate_time) { + /* Either the log2_rotate_time in header was missing, or we + just deleted the .log.2 and need to set it as nonexistent. + Either way we need to update the header. + + Write this as part of the next sync's transaction. We're + here because we're already opening a sync lock, so it'll + always happen. It's also required especially with mdbox map + index, which doesn't like changes done outside syncing. */ + log->index->hdr_log2_rotate_time_delayed_update = + log2_rotate_time; + } +} + +int mail_transaction_log_open(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + const char *reason; + int ret; + + i_free(log->filepath); + i_free(log->filepath2); + log->filepath = i_strconcat(log->index->filepath, + MAIL_TRANSACTION_LOG_SUFFIX, NULL); + log->filepath2 = i_strconcat(log->filepath, ".2", NULL); + + if (log->open_file != NULL) + mail_transaction_log_file_free(&log->open_file); + + if (MAIL_INDEX_IS_IN_MEMORY(log->index)) + return 0; + + file = mail_transaction_log_file_alloc(log, log->filepath); + if ((ret = mail_transaction_log_file_open(file, &reason)) <= 0) { + /* leave the file for _create() */ + log->open_file = file; + return ret; + } + mail_transaction_log_set_head(log, file); + return 1; +} + +int mail_transaction_log_create(struct mail_transaction_log *log, bool reset) +{ + struct mail_transaction_log_file *file; + + if (MAIL_INDEX_IS_IN_MEMORY(log->index)) { + file = mail_transaction_log_file_alloc_in_memory(log); + mail_transaction_log_set_head(log, file); + return 0; + } + + file = mail_transaction_log_file_alloc(log, log->filepath); + if (log->open_file != NULL) { + /* remember what file we tried to open. if someone else created + a new file, use it instead of recreating it */ + file->st_ino = log->open_file->st_ino; + file->st_dev = log->open_file->st_dev; + file->last_size = log->open_file->last_size; + file->last_mtime = log->open_file->last_mtime; + mail_transaction_log_file_free(&log->open_file); + } + + if (mail_transaction_log_file_create(file, reset) < 0) { + mail_transaction_log_file_free(&file); + return -1; + } + + mail_transaction_log_set_head(log, file); + return 1; +} + +void mail_transaction_log_close(struct mail_transaction_log *log) +{ + i_assert(log->views == NULL); + + if (log->open_file != NULL) + mail_transaction_log_file_free(&log->open_file); + if (log->head != NULL) + log->head->refcount--; + mail_transaction_logs_clean(log); + i_assert(log->files == NULL); +} + +void mail_transaction_log_free(struct mail_transaction_log **_log) +{ + struct mail_transaction_log *log = *_log; + + *_log = NULL; + + mail_transaction_log_close(log); + log->index->log = NULL; + i_free(log->filepath); + i_free(log->filepath2); + i_free(log); +} + +int mail_transaction_log_move_to_memory(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + + if (!log->index->initial_mapped && log->files != NULL && + log->files->hdr.prev_file_seq != 0) { + /* we couldn't read dovecot.index and we don't have the first + .log file, so just start from scratch */ + mail_transaction_log_close(log); + } + + i_free(log->filepath); + i_free(log->filepath2); + log->filepath = i_strconcat(log->index->filepath, + MAIL_TRANSACTION_LOG_SUFFIX, NULL); + log->filepath2 = i_strconcat(log->filepath, ".2", NULL); + + if (log->head != NULL) + return mail_transaction_log_file_move_to_memory(log->head); + else { + file = mail_transaction_log_file_alloc_in_memory(log); + mail_transaction_log_set_head(log, file); + return 0; + } +} + +void mail_transaction_log_indexid_changed(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + + mail_transaction_logs_clean(log); + + for (file = log->files; file != NULL; file = file->next) { + if (file->hdr.indexid != log->index->indexid) { + mail_transaction_log_file_set_corrupted(file, + "indexid changed: %u -> %u", + file->hdr.indexid, log->index->indexid); + } + } + + if (log->head != NULL && + log->head->hdr.indexid != log->index->indexid) { + struct mail_transaction_log_file *old_head = log->head; + + (void)mail_transaction_log_create(log, FALSE); + if (--old_head->refcount == 0) { + if (old_head == log->head) { + /* failed to create a new log */ + log->head = NULL; + } + mail_transaction_log_file_free(&old_head); + } + } +} + +void mail_transaction_logs_clean(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file, *next; + + /* remove only files from the beginning. this way if a view has + referenced an old file, it can still find the new files even if + there aren't any references to it currently. */ + for (file = log->files; file != NULL; file = next) { + next = file->next; + + i_assert(file->refcount >= 0); + if (file->refcount > 0) + break; + + mail_transaction_log_file_free(&file); + } + /* sanity check: we shouldn't have locked refcount=0 files */ + for (; file != NULL; file = file->next) { + i_assert(!file->locked || file->refcount > 0); + } + i_assert(log->head == NULL || log->files != NULL); +} + +bool mail_transaction_log_want_rotate(struct mail_transaction_log *log, + const char **reason_r) +{ + struct mail_transaction_log_file *file = log->head; + + if (file->need_rotate != NULL) { + *reason_r = t_strdup(file->need_rotate); + return TRUE; + } + + if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION || + (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION && + file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) { + /* upgrade immediately to a new log file format */ + *reason_r = t_strdup_printf( + ".log file format version %u.%u is too old", + file->hdr.major_version, file->hdr.minor_version); + return TRUE; + } + + if (file->sync_offset > log->index->optimization_set.log.max_size) { + /* file is too large, definitely rotate */ + *reason_r = t_strdup_printf( + ".log file size %"PRIuUOFF_T" > max_size %"PRIuUOFF_T, + file->sync_offset, log->index->optimization_set.log.max_size); + return TRUE; + } + if (file->sync_offset < log->index->optimization_set.log.min_size) { + /* file is still too small */ + return FALSE; + } + /* rotate if the timestamp is old enough */ + if (file->hdr.create_stamp < + ioloop_time - log->index->optimization_set.log.min_age_secs) { + *reason_r = t_strdup_printf( + ".log create_stamp %u is older than %u secs", + file->hdr.create_stamp, + log->index->optimization_set.log.min_age_secs); + return TRUE; + } + return FALSE; +} + +int mail_transaction_log_rotate(struct mail_transaction_log *log, bool reset) +{ + struct mail_transaction_log_file *file, *old_head; + const char *path = log->head->filepath; + struct stat st; + int ret; + + i_assert(log->head->locked); + + if (MAIL_INDEX_IS_IN_MEMORY(log->index)) { + file = mail_transaction_log_file_alloc_in_memory(log); + if (reset) { + file->hdr.prev_file_seq = 0; + file->hdr.prev_file_offset = 0; + } + } else { + /* we're locked, we shouldn't need to worry about ESTALE + problems in here. */ + if (fstat(log->head->fd, &st) < 0) { + mail_index_file_set_syscall_error(log->index, + log->head->filepath, "fstat()"); + return -1; + } + + file = mail_transaction_log_file_alloc(log, path); + + file->st_dev = st.st_dev; + file->st_ino = st.st_ino; + file->last_mtime = st.st_mtime; + file->last_size = st.st_size; + + if ((ret = mail_transaction_log_file_create(file, reset)) < 0) { + mail_transaction_log_file_free(&file); + return -1; + } + if (ret == 0) { + mail_index_set_error(log->index, + "Transaction log %s was recreated while we had it locked - " + "locking is broken (lock_method=%s)", path, + file_lock_method_to_str(log->index->set.lock_method)); + mail_transaction_log_file_free(&file); + return -1; + } + i_assert(file->locked); + } + + old_head = log->head; + mail_transaction_log_set_head(log, file); + + e_debug(log->index->event, "Rotated transaction log %s (seq=%u, reset=%s)", + file->filepath, file->hdr.file_seq, reset ? "yes" : "no"); + + /* the newly created log file is already locked */ + mail_transaction_log_file_unlock(old_head, + !log->index->log_sync_locked ? "rotating" : + "rotating while syncing"); + if (--old_head->refcount == 0) + mail_transaction_logs_clean(log); + return 0; +} + +static int +mail_transaction_log_refresh(struct mail_transaction_log *log, bool nfs_flush, + const char **reason_r) +{ + struct mail_transaction_log_file *file; + struct stat st; + + i_assert(log->head != NULL); + + if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(log->head)) { + *reason_r = "Log is in memory"; + return 0; + } + + if (nfs_flush && + (log->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) + nfs_flush_file_handle_cache(log->filepath); + if (nfs_safe_stat(log->filepath, &st) < 0) { + if (errno != ENOENT) { + mail_index_file_set_syscall_error(log->index, + log->filepath, + "stat()"); + *reason_r = t_strdup_printf("stat(%s) failed: %m", log->filepath); + return -1; + } + /* We shouldn't lose dovecot.index.log unless the mailbox was + deleted or renamed. Just fail this and let the mailbox + opening code figure out whether to create a new log file + or not. Anything else can cause unwanted behavior (e.g. + mailbox deletion not fully finishing due to .nfs* files and + an IDLEing IMAP process creating the index back here). */ + log->index->index_deleted = TRUE; + *reason_r = "Trasnaction log lost while it was open"; + return -1; + } else if (log->head->st_ino == st.st_ino && + CMP_DEV_T(log->head->st_dev, st.st_dev)) { + /* NFS: log files get rotated to .log.2 files instead + of being unlinked, so we don't bother checking if + the existing file has already been unlinked here + (in which case inodes could match but point to + different files) */ + *reason_r = "Log inode is unchanged"; + return 0; + } + + file = mail_transaction_log_file_alloc(log, log->filepath); + if (mail_transaction_log_file_open(file, reason_r) <= 0) { + *reason_r = t_strdup_printf( + "Failed to refresh main transaction log: %s", *reason_r); + mail_transaction_log_file_free(&file); + return -1; + } + + i_assert(!file->locked); + + struct mail_transaction_log_file *old_head = log->head; + mail_transaction_log_set_head(log, file); + if (--old_head->refcount == 0) + mail_transaction_logs_clean(log); + *reason_r = "Log reopened"; + return 0; +} + +void mail_transaction_log_get_mailbox_sync_pos(struct mail_transaction_log *log, + uint32_t *file_seq_r, + uoff_t *file_offset_r) +{ + *file_seq_r = log->head->hdr.file_seq; + *file_offset_r = log->head->max_tail_offset; +} + +void mail_transaction_log_set_mailbox_sync_pos(struct mail_transaction_log *log, + uint32_t file_seq, + uoff_t file_offset) +{ + i_assert(file_seq == log->head->hdr.file_seq); + i_assert(file_offset >= log->head->last_read_hdr_tail_offset); + + if (file_offset >= log->head->max_tail_offset) + log->head->max_tail_offset = file_offset; +} + +int mail_transaction_log_find_file(struct mail_transaction_log *log, + uint32_t file_seq, bool nfs_flush, + struct mail_transaction_log_file **file_r, + const char **reason_r) +{ + struct mail_transaction_log_file *file; + const char *reason; + int ret; + + if (file_seq > log->head->hdr.file_seq) { + /* see if the .log file has been recreated */ + if (log->head->locked) { + /* transaction log is locked. there's no way a newer + file exists. */ + *reason_r = "Log is locked - newer log can't exist"; + return 0; + } + + if (mail_transaction_log_refresh(log, FALSE, &reason) < 0) { + *reason_r = reason; + return -1; + } + if (file_seq > log->head->hdr.file_seq) { + if (!nfs_flush || + (log->index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) == 0) { + *reason_r = t_strdup_printf( + "Requested newer log than exists: %s", reason); + return 0; + } + /* try again, this time flush attribute cache */ + if (mail_transaction_log_refresh(log, TRUE, &reason) < 0) { + *reason_r = t_strdup_printf( + "Log refresh with NFS flush failed: %s", reason); + return -1; + } + if (file_seq > log->head->hdr.file_seq) { + *reason_r = t_strdup_printf( + "Requested newer log than exists - " + "still after NFS flush: %s", reason); + return 0; + } + } + } + + for (file = log->files; file != NULL; file = file->next) { + if (file->hdr.file_seq == file_seq) { + *file_r = file; + return 1; + } + if (file->hdr.file_seq > file_seq && + file->hdr.prev_file_seq == 0) { + /* Fail here mainly to avoid unnecessarily trying to + open .log.2 that most likely doesn't even exist. */ + *reason_r = "Log was reset after requested file_seq"; + return 0; + } + } + + if (MAIL_INDEX_IS_IN_MEMORY(log->index)) { + *reason_r = "Logs are only in memory"; + return 0; + } + + /* see if we have it in log.2 file */ + file = mail_transaction_log_file_alloc(log, log->filepath2); + if ((ret = mail_transaction_log_file_open(file, reason_r)) <= 0) { + *reason_r = t_strdup_printf( + "Not found from .log.2: %s", *reason_r); + mail_transaction_log_file_free(&file); + return ret; + } + + /* but is it what we expected? */ + if (file->hdr.file_seq != file_seq) { + *reason_r = t_strdup_printf(".log.2 contains file_seq=%u", + file->hdr.file_seq); + return 0; + } + + *file_r = file; + return 1; +} + +int mail_transaction_log_lock_head(struct mail_transaction_log *log, + const char *lock_reason) +{ + struct mail_transaction_log_file *file; + time_t lock_wait_started, lock_secs = 0; + const char *reason; + int ret = 0; + + /* we want to get the head file locked. this is a bit racy, + since by the time we have it locked a new log file may have been + created. + + creating new log file requires locking the head file, so if we + can lock it and don't see another file, we can be sure no-one is + creating a new log at the moment */ + + lock_wait_started = time(NULL); + for (;;) { + file = log->head; + if (mail_transaction_log_file_lock(file) < 0) + return -1; + + file->refcount++; + ret = mail_transaction_log_refresh(log, TRUE, &reason); + if (--file->refcount == 0) { + mail_transaction_log_file_unlock(file, t_strdup_printf( + "trying to lock head for %s", lock_reason)); + mail_transaction_logs_clean(log); + file = NULL; + } + + if (ret == 0 && log->head == file) { + /* success */ + i_assert(file != NULL); + lock_secs = file->lock_create_time - lock_wait_started; + break; + } + + if (file != NULL) { + mail_transaction_log_file_unlock(file, t_strdup_printf( + "trying to lock head for %s", lock_reason)); + } + if (ret < 0) + break; + + /* try again */ + } + if (lock_secs > MAIL_TRANSACTION_LOG_LOCK_WARN_SECS) { + i_warning("Locking transaction log file %s took %ld seconds (%s)", + log->head->filepath, (long)lock_secs, lock_reason); + } + + i_assert(ret < 0 || log->head != NULL); + return ret; +} + +int mail_transaction_log_sync_lock(struct mail_transaction_log *log, + const char *lock_reason, + uint32_t *file_seq_r, uoff_t *file_offset_r) +{ + const char *reason; + + i_assert(!log->index->log_sync_locked); + + if (!log->log_2_unlink_checked) { + /* we need to check once in a while if .log.2 should be deleted + to avoid wasting space on such old files. but we also don't + want to waste time on checking it when the same mailbox + gets opened over and over again rapidly (e.g. pop3). so + do this only when there have actually been some changes + to mailbox (i.e. when it's being locked here) */ + log->log_2_unlink_checked = TRUE; + mail_transaction_log_2_unlink_old(log); + } + + if (mail_transaction_log_lock_head(log, lock_reason) < 0) + return -1; + + /* update sync_offset */ + if (mail_transaction_log_file_map(log->head, log->head->sync_offset, + UOFF_T_MAX, &reason) <= 0) { + mail_index_set_error(log->index, + "Failed to map transaction log %s at " + "sync_offset=%"PRIuUOFF_T" after locking: %s", + log->head->filepath, log->head->sync_offset, reason); + mail_transaction_log_file_unlock(log->head, t_strdup_printf( + "%s - map failed", lock_reason)); + return -1; + } + + log->index->log_sync_locked = TRUE; + *file_seq_r = log->head->hdr.file_seq; + *file_offset_r = log->head->sync_offset; + return 0; +} + +void mail_transaction_log_sync_unlock(struct mail_transaction_log *log, + const char *lock_reason) +{ + i_assert(log->index->log_sync_locked); + + log->index->log_sync_locked = FALSE; + mail_transaction_log_file_unlock(log->head, lock_reason); +} + +void mail_transaction_log_get_head(struct mail_transaction_log *log, + uint32_t *file_seq_r, uoff_t *file_offset_r) +{ + *file_seq_r = log->head->hdr.file_seq; + *file_offset_r = log->head->sync_offset; +} + +void mail_transaction_log_get_tail(struct mail_transaction_log *log, + uint32_t *file_seq_r) +{ + struct mail_transaction_log_file *tail, *file = log->files; + + for (tail = file; file->next != NULL; file = file->next) { + if (file->hdr.file_seq + 1 != file->next->hdr.file_seq) + tail = file->next; + } + *file_seq_r = tail->hdr.file_seq; +} + +bool mail_transaction_log_is_head_prev(struct mail_transaction_log *log, + uint32_t file_seq, uoff_t file_offset) +{ + return log->head->hdr.prev_file_seq == file_seq && + log->head->hdr.prev_file_offset == file_offset; +} + +int mail_transaction_log_unlink(struct mail_transaction_log *log) +{ + if (unlink(log->filepath) < 0 && + errno != ENOENT && errno != ESTALE) { + mail_index_file_set_syscall_error(log->index, log->filepath, + "unlink()"); + return -1; + } + return 0; +} + +void mail_transaction_log_get_dotlock_set(struct mail_transaction_log *log, + struct dotlock_settings *set_r) +{ + struct mail_index *index = log->index; + + i_zero(set_r); + set_r->timeout = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT, + index->set.max_lock_timeout_secs); + set_r->stale_timeout = MAIL_TRANSACTION_LOG_DOTLOCK_CHANGE_TIMEOUT; + set_r->nfs_flush = (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0; + set_r->use_excl_lock = + (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0; +} diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h new file mode 100644 index 0000000..c19bb20 --- /dev/null +++ b/src/lib-index/mail-transaction-log.h @@ -0,0 +1,494 @@ +#ifndef MAIL_TRANSACTION_LOG_H +#define MAIL_TRANSACTION_LOG_H + +#include "mail-index.h" + +#define MAIL_TRANSACTION_LOG_SUFFIX ".log" + +#define MAIL_TRANSACTION_LOG_MAJOR_VERSION 1 +#define MAIL_TRANSACTION_LOG_MINOR_VERSION 3 +/* Minimum allowed mail_transaction_log_header.hdr_size. If it's smaller, + assume the file is corrupted. */ +#define MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE 24 + +/* Helper macro for other MAIL_TRANSACTION_LOG_VERSION_*() macros */ +#define MAIL_TRANSACTION_LOG_VERSION_FULL(major, minor) \ + ((major) << 8 | (minor)) +/* Returns TRUE if the transaction log version supports the given feature. + The wanted_feature is one of the MAIL_TRANSACTION_LOG_VERSION_FEATURE_* + macros without the macro prefix, e.g. just COMPAT_FLAGS. */ +#define MAIL_TRANSACTION_LOG_VERSION_HAVE(version, wanted_feature) \ + ((version) >= MAIL_TRANSACTION_LOG_VERSION_FEATURE_##wanted_feature) +/* Returns transaction log version from the given mail_transaction_log_header + which is compatible for the MAIL_TRANSACTION_LOG_VERSION_HAVE() macro. */ +#define MAIL_TRANSACTION_LOG_HDR_VERSION(hdr) \ + MAIL_TRANSACTION_LOG_VERSION_FULL((hdr)->major_version, (hdr)->minor_version) + +/* Log feature: mail_transaction_log_header.compat_flags is filled. */ +#define MAIL_TRANSACTION_LOG_VERSION_FEATURE_COMPAT_FLAGS \ + MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2) +/* Log feature: Don't increase modseq when reading internal flag updates + (because they're not client-visible anyway). + See MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(). */ +#define MAIL_TRANSACTION_LOG_VERSION_FEATURE_HIDE_INTERNAL_MODSEQS \ + MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3) + +struct mail_transaction_log_header { + /* Major version is increased only when you can't have backwards + compatibility. If the field doesn't match + MAIL_TRANSACTION_LOG_MAJOR_VERSION, don't even try to read it. */ + uint8_t major_version; + /* Minor version is increased when the file format changes in a + backwards compatible way. */ + uint8_t minor_version; + /* Size of the header. If it's larger than this struct, ignore any + unknown fields. If it's smaller, assume the rest of the fields + are 0. */ + uint16_t hdr_size; + + /* Unique index file ID, which must match the main index's indexid. + See mail_index_header.indexid. This is overwritten to be 0 if the + log file is marked as corrupted. */ + uint32_t indexid; + /* Log file sequence number. Increased every time the log is rotated + and a new log is created. Using (file_seq, offset) uniquely + identifies a position in the transaction log. */ + uint32_t file_seq; + /* The previous log file's sequence and offset when the log was + rotated. The offset should be the same as the previous log file's + size. If there was no previous log file, or if the index is being + reset, these are 0. + + These are mainly useful to optimize syncing when the start position + is (prev_file_seq, prev_file_offset). Then it's it's already known + that the syncing can be started from this log file wihtout having + to open the previous log file only to realize that there is nothing + to sync. (Which could have also lead to an error if the .log.2 was + already deleted.) */ + uint32_t prev_file_seq; + uint32_t prev_file_offset; + /* UNIX timestamp when this file was created. Used in determining when + to rotate the log file. */ + uint32_t create_stamp; + /* Modseq value at the beginning of this file. Some transaction records + increase the modseq value. (Only with log format v1.1+) */ + uint64_t initial_modseq; + + /* Same as enum mail_index_header_compat_flags. Needs + MAIL_TRANSACTION_LOG_VERSION_FEATURE_COMPAT_FLAGS. */ + uint8_t compat_flags; + /* Unused fields to make the struct 64bit aligned. These can be used + to add more fields to the header. */ + uint8_t unused[3]; + uint32_t unused2; +}; + +enum mail_transaction_type { + /* struct mail_transaction_expunge[] - Expunge the UIDs. + Must have MAIL_TRANSACTION_EXPUNGE_PROT ORed to this. Avoid using + this, use MAIL_TRANSACTION_EXPUNGE_GUID instead. */ + MAIL_TRANSACTION_EXPUNGE = 0x00000001, + /* struct mail_index_record[] - Save new mails with given flags. */ + MAIL_TRANSACTION_APPEND = 0x00000002, + /* struct mail_transaction_flag_update[] - Update message flags + (or just modseq). */ + MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004, + /* struct mail_transaction_header_update[] - Update the index's base + header (struct mail_index_header). */ + MAIL_TRANSACTION_HEADER_UPDATE = 0x00000020, + /* struct mail_transaction_ext_intro - Start operations for the given + extension. This can be used to create a new extension or resize an + existing extension, but usually it is just used in front of the + other MAIL_TRANSACTION_EXT_* records to specify which extension + they're working with. */ + MAIL_TRANSACTION_EXT_INTRO = 0x00000040, + /* struct mail_transaction_ext_reset - Reset the last intro extension + by changing its reset_id and optionally zeroing out its old data. */ + MAIL_TRANSACTION_EXT_RESET = 0x00000080, + /* struct mail_transaction_ext_hdr_update[] - Update the last intro + extension's header. This might later become deprecated in favor of + supporting only MAIL_TRANSACTION_EXT_HDR_UPDATE32, but for now + it's still used for <64kB headers. */ + MAIL_TRANSACTION_EXT_HDR_UPDATE = 0x00000100, + /* struct mail_transaction_ext_rec_update[] - Update the last intro + extension records for the given UIDs with given content. */ + MAIL_TRANSACTION_EXT_REC_UPDATE = 0x00000200, + /* struct mail_transaction_keyword_update - Add/remove the specified + keyword to messages. */ + MAIL_TRANSACTION_KEYWORD_UPDATE = 0x00000400, + /* struct mail_transaction_keyword_reset[] - Clear out all keywords + in specified messages. */ + MAIL_TRANSACTION_KEYWORD_RESET = 0x00000800, + /* struct mail_transaction_ext_atomic_inc[] - Atomically increase or + decrease the last intro extension record. The record must be 1, 2, + 4 or 8 bytes. This can be used e.g. for refcount extensions. */ + MAIL_TRANSACTION_EXT_ATOMIC_INC = 0x00001000, + /* struct mail_transaction_expunge_guid[] - Expunge given UID, but + first verify that it matches the given GUID. Must have + MAIL_TRANSACTION_EXPUNGE_PROT ORed to this. */ + MAIL_TRANSACTION_EXPUNGE_GUID = 0x00002000, + MAIL_TRANSACTION_MODSEQ_UPDATE = 0x00008000, + /* struct mail_transaction_ext_hdr_update32[] - Update the last intro + extension's header. Used for >=64kB headers. See also + MAIL_TRANSACTION_EXT_HDR_UPDATE. This was added in Dovecot v2.0. */ + MAIL_TRANSACTION_EXT_HDR_UPDATE32 = 0x00010000, + /* Index was marked as deleted using mail_index_set_deleted(). + There is no record content for this. */ + MAIL_TRANSACTION_INDEX_DELETED = 0x00020000, + /* Index was marked as undeleted using mail_index_set_undeleted(). + There is no record content for this. */ + MAIL_TRANSACTION_INDEX_UNDELETED = 0x00040000, + /* struct mail_transaction_boundary - Specifies a size of the following + records that must be treated as a single transaction. This works + so that the transaction log reading code stops if it finds that + there is a transaction whose size points outside the currently + existing file. An unfinished transaction is truncated away after the + next write to the log. FIXME: it would be better to rotate the + log instead of truncating it. */ + MAIL_TRANSACTION_BOUNDARY = 0x00080000, + /* Mailbox attribute update. This is a bit complicated format: + - [+-][p-s]<name><NUL> + - "+" means attribute is set, "-" means unset + - "p" means private attribute, "s" means shared + - <name> is the attribute name + - This can repeat multiple times + - <NUL> + - 0..3 bytes padding for 32bit alignment + - For each attribute update an array of uint32_t integers: + - Update timestamp + - For each "+" only: Length of the attribute value. + */ + MAIL_TRANSACTION_ATTRIBUTE_UPDATE = 0x00100000, + + /* Mask to get the attribute type only (excluding flags). */ + MAIL_TRANSACTION_TYPE_MASK = 0x0fffffff, + +#define MAIL_TRANSACTION_EXT_MASK \ + (MAIL_TRANSACTION_EXT_INTRO | MAIL_TRANSACTION_EXT_RESET | \ + MAIL_TRANSACTION_EXT_HDR_UPDATE | MAIL_TRANSACTION_EXT_HDR_UPDATE32 | \ + MAIL_TRANSACTION_EXT_REC_UPDATE | MAIL_TRANSACTION_EXT_ATOMIC_INC) + + /* Since we'll expunge mails based on data read from transaction log, + try to avoid the possibility of corrupted transaction log expunging + messages. This value is ORed to the actual MAIL_TRANSACTION_EXPUNGE* + flag. If it's not present, assume corrupted log. */ + MAIL_TRANSACTION_EXPUNGE_PROT = 0x0000cd90, + + /* External transactions have a bit different meanings depending on the + transaction type. Generally they mean to indicate changes that have + already occurred, instead of changes that are only being requested + to happen on next sync. For example expunges are first requested + to be done with internal transactions, and then there's a separate + external transaction to indicate that they were actually done. */ + MAIL_TRANSACTION_EXTERNAL = 0x10000000, + /* This change syncs the state with another mailbox (dsync), + i.e. the change isn't something that a user requested locally. */ + MAIL_TRANSACTION_SYNC = 0x20000000 +}; + +struct mail_transaction_header { + /* Size of this header and the following records. This size can be + used to calculate how many records there are. The size is written + via mail_index_uint32_to_offset(). */ + uint32_t size; + uint32_t type; /* enum mail_transaction_type */ + /* Header is followed by the type-specific records. */ +}; + +/* See MAIL_TRANSACTION_MODSEQ_UPDATE. */ +struct mail_transaction_modseq_update { + uint32_t uid; + /* don't use uint64_t here. it adds extra 32 bits of padding and also + causes problems with CPUs that require alignment */ + uint32_t modseq_low32; + uint32_t modseq_high32; +}; + +/* See MAIL_TRANSACTION_EXPUNGE. */ +struct mail_transaction_expunge { + /* Expunge all mails between uid1..uid2. */ + uint32_t uid1, uid2; +}; +/* See MAIL_TRANSACTION_EXPUNGE_GUID. */ +struct mail_transaction_expunge_guid { + /* Expunge uid, but only if it matches guid_128. */ + uint32_t uid; + /* GUID of the mail. If it's not 128 bit GUID, first pass it through + mail_generate_guid_128_hash() to get 128 bit SHA1 of it. */ + guid_128_t guid_128; +}; + +/* See MAIL_TRANSACTION_FLAG_UPDATE. */ +struct mail_transaction_flag_update { + /* Change the flags for all mails between uid1..uid2. */ + uint32_t uid1, uid2; + /* Add these flags to the mails. */ + uint8_t add_flags; + /* Remove these flags to the mails. To replace all existing flags, + just set this to 0xff and specify the wanted flags in add_flags. */ + uint8_t remove_flags; + /* If non-0, MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ was used to force + increasing modseq update to the mails even though no flags were + actually changed. This differs from MAIL_TRANSACTION_MODSEQ_UPDATE + in that the modseq is just wanted to be increased, doesn't matter + to which value specifically. */ + uint8_t modseq_inc_flag; + /* Unused padding */ + uint8_t padding; +}; + +/* See MAIL_TRANSACTION_KEYWORD_UPDATE. */ +struct mail_transaction_keyword_update { + /* enum modify_type : MODIFY_ADD / MODIFY_REMOVE */ + uint8_t modify_type; + uint8_t padding; + /* Size of name[] */ + uint16_t name_size; + /* unsigned char name[name_size]; */ + /* Update keywords for the given UIDs. The array's size is calculated + from mail_transaction_header.size. */ + /* array of { uint32_t uid1, uid2; } */ +}; + +/* See MAIL_TRANSACTION_KEYWORD_RESET. */ +struct mail_transaction_keyword_reset { + /* Clear out all keywords for uid1..uid2. */ + uint32_t uid1, uid2; +}; + +/* See MAIL_TRANSACTION_HEADER_UPDATE. */ +struct mail_transaction_header_update { + /* Update start offset. */ + uint16_t offset; + /* Size of the following data[] to update. */ + uint16_t size; + /* unsigned char data[size]; */ + /* 0..3 bytes of padding to get to 32bit alignment. */ + /* unsigned char padding[]; */ +}; + +enum { + /* Don't shrink hdr_size, record_size or record_align but grow them + if necessary. */ + MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK = 0x01 +}; + +/* See MAIL_TRANSACTION_EXT_INTRO. Also see struct mail_index_ext_header for + more explanations of these fields. */ +struct mail_transaction_ext_intro { + /* If extension is already known to exist in the index file, + set ext_id, but use empty name. If this is a new extension, set + name, but use ext_id=(uint32_t)-1. */ + uint32_t ext_id; + uint32_t reset_id; + /* Size of the extension header. When growing the header size, it's + initially filled with zeros. The header can be written to with + ext-hdr-update records. */ + uint32_t hdr_size; + uint16_t record_size; + uint16_t record_align; + uint16_t flags; + uint16_t name_size; + /* unsigned char name[]; */ +}; + +/* See MAIL_TRANSACTION_EXT_RESET. */ +struct mail_transaction_ext_reset { + /* New value for extension's reset_id */ + uint32_t new_reset_id; + /* Non-0 if the old extension header and record data should be + preserved. Normally all of it is zeroed out. */ + uint8_t preserve_data; + uint8_t unused_padding[3]; +}; + +/* See MAIL_TRANSACTION_EXT_HDR_UPDATE. */ +struct mail_transaction_ext_hdr_update { + /* Update start offset. */ + uint16_t offset; + /* Size of the following data[] to update. */ + uint16_t size; + /* unsigned char data[size]; */ + /* 0..3 bytes of padding to get to 32bit alignment. */ + /* unsigned char padding[]; */ +}; +/* See MAIL_TRANSACTION_EXT_HDR_UPDATE32. */ +struct mail_transaction_ext_hdr_update32 { + /* Update start offset. */ + uint32_t offset; + /* Size of the following data[] to update. */ + uint32_t size; + /* unsigned char data[size]; */ + /* 0..3 bytes of padding to get to 32bit alignment. */ + /* unsigned char padding[]; */ +}; + +/* See MAIL_TRANSACTION_EXT_REC_UPDATE. */ +struct mail_transaction_ext_rec_update { + uint32_t uid; + /* unsigned char data[mail_transaction_ext_intro.record_size]; */ + /* 0..3 bytes of padding to get to 32bit alignment. */ + /* unsigned char padding[]; */ +}; + +/* See MAIL_TRANSACTION_EXT_ATOMIC_INC. */ +struct mail_transaction_ext_atomic_inc { + uint32_t uid; + /* Add this value to the extension record data. Can be negative. */ + int32_t diff; +}; + +/* See MAIL_TRANSACTION_BOUNDARY. */ +struct mail_transaction_boundary { + /* Size of the whole transaction, including this record and header. */ + uint32_t size; +}; + +struct mail_transaction_log_append_ctx { + struct mail_transaction_log *log; + /* All the changes that will be written to the transaction log. */ + buffer_t *output; + + /* Transaction flags as given to mail_transaction_log_append_begin(). */ + enum mail_transaction_type trans_flags; + + /* Tracking the current highest_modseq after the changes. This will + be used to update mail_transaction_log_file.sync_highest_modseq. */ + uint64_t new_highest_modseq; + /* Number of transaction records added so far. */ + unsigned int transaction_count; + + /* Copied from mail_index_transaction.sync_transaction */ + bool index_sync_transaction:1; + /* Copied from mail_index_transaction.tail_offset_changed */ + bool tail_offset_changed:1; + /* TRUE if the mail_transaction_log_file has been synced up to the + current write offset, and we're writing a syncing transaction + (index_sync_transaction=TRUE). This means that the just written + transaction can be assumed to be synced already. */ + bool sync_includes_this:1; + /* fdatasync() after writing the transaction. */ + bool want_fsync:1; +}; + +#define LOG_IS_BEFORE(seq1, offset1, seq2, offset2) \ + (((offset1) < (offset2) && (seq1) == (seq2)) || (seq1) < (seq2)) + +struct mail_transaction_log * +mail_transaction_log_alloc(struct mail_index *index); +void mail_transaction_log_free(struct mail_transaction_log **log); + +/* Open the transaction log. Returns 1 if ok, 0 if file doesn't exist or it's + is corrupted, -1 if there was some I/O error. */ +int mail_transaction_log_open(struct mail_transaction_log *log); +/* Create, or recreate, the transaction log. Returns 0 if ok, -1 if error. */ +int mail_transaction_log_create(struct mail_transaction_log *log, bool reset); +/* Close all the open transactions log files. */ +void mail_transaction_log_close(struct mail_transaction_log *log); + +/* Notify of indexid change */ +void mail_transaction_log_indexid_changed(struct mail_transaction_log *log); + +/* Returns the file seq/offset where the mailbox is currently synced at. + Since the log is rotated only when mailbox is fully synced, the sequence + points always to the latest file. This function doesn't actually find the + latest sync position, so you'll need to use eg. log_view_set() before + calling this. */ +void mail_transaction_log_get_mailbox_sync_pos(struct mail_transaction_log *log, + uint32_t *file_seq_r, + uoff_t *file_offset_r); +/* Set the current mailbox sync position. file_seq must always be the latest + log file's sequence. The offset written automatically to the log when + other transactions are being written. */ +void mail_transaction_log_set_mailbox_sync_pos(struct mail_transaction_log *log, + uint32_t file_seq, + uoff_t file_offset); + +struct mail_transaction_log_view * +mail_transaction_log_view_open(struct mail_transaction_log *log); +void mail_transaction_log_view_close(struct mail_transaction_log_view **view); + +/* Set view boundaries. Returns 1 if ok, 0 if files are lost, corrupted or the + offsets are broken, -1 if I/O error. reset_r=TRUE if the whole index should + be reset before applying any changes. */ +int mail_transaction_log_view_set(struct mail_transaction_log_view *view, + uint32_t min_file_seq, uoff_t min_file_offset, + uint32_t max_file_seq, uoff_t max_file_offset, + bool *reset_r, const char **reason_r); +/* Scan through all of the log files that we can find. + Returns -1 if error, 0 if ok. */ +int mail_transaction_log_view_set_all(struct mail_transaction_log_view *view); +/* Clear the view. If oldest_file_seq > 0, keep it and newer log files + referenced so we don't get desynced. */ +void mail_transaction_log_view_clear(struct mail_transaction_log_view *view, + uint32_t oldest_file_seq); + +/* Read next transaction record from current position. The position is updated. + Returns -1 if error, 0 if we're at end of the view, 1 if ok. */ +int mail_transaction_log_view_next(struct mail_transaction_log_view *view, + const struct mail_transaction_header **hdr_r, + const void **data_r); +/* Mark the current view's position to the record returned previously with + _log_view_next(). */ +void mail_transaction_log_view_mark(struct mail_transaction_log_view *view); +/* Seek to previously marked position. */ +void mail_transaction_log_view_rewind(struct mail_transaction_log_view *view); + +/* Returns the position of the record returned previously with + mail_transaction_log_view_next() */ +void +mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view, + uint32_t *file_seq_r, + uoff_t *file_offset_r); +/* Return the modseq of the change returned previously with _view_next(). */ +uint64_t +mail_transaction_log_view_get_prev_modseq(struct mail_transaction_log_view *view); +/* Returns TRUE if we're at the end of the view window. */ +bool mail_transaction_log_view_is_last(struct mail_transaction_log_view *view); + +/* Marks the log file in current position to be corrupted. */ +void +mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view, + const char *fmt, ...) + ATTR_FORMAT(2, 3) ATTR_COLD; +bool +mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view); + +int mail_transaction_log_append_begin(struct mail_index *index, + enum mail_transaction_type flags, + struct mail_transaction_log_append_ctx **ctx_r); +void mail_transaction_log_append_add(struct mail_transaction_log_append_ctx *ctx, + enum mail_transaction_type type, + const void *data, size_t size); +int mail_transaction_log_append_commit(struct mail_transaction_log_append_ctx **ctx); + +/* Lock transaction log for index synchronization. This is used as the main + exclusive lock for index changes. The index/log can still be read since they + don't use locking, but the log can't be written to while it's locked. + Returns 0 on success, -1 if locking failed for any reason. + + After successfully locking the transaction log, the log file is also fully + mapped into memory and its sync_offset updated. The locked file's sequence + and sync_offset are returned. */ +int mail_transaction_log_sync_lock(struct mail_transaction_log *log, + const char *lock_reason, + uint32_t *file_seq_r, uoff_t *file_offset_r); +void mail_transaction_log_sync_unlock(struct mail_transaction_log *log, + const char *lock_reason); +/* Returns the current head. Works only when log is locked. */ +void mail_transaction_log_get_head(struct mail_transaction_log *log, + uint32_t *file_seq_r, uoff_t *file_offset_r); +/* Returns the current tail from which all files are open to head. */ +void mail_transaction_log_get_tail(struct mail_transaction_log *log, + uint32_t *file_seq_r); +/* Returns TRUE if given seq/offset is current head log's rotate point. */ +bool mail_transaction_log_is_head_prev(struct mail_transaction_log *log, + uint32_t file_seq, uoff_t file_offset); + +/* Move currently opened log head file to memory (called by + mail_index_move_to_memory()) */ +int mail_transaction_log_move_to_memory(struct mail_transaction_log *log); +/* Unlink transaction log files */ +int mail_transaction_log_unlink(struct mail_transaction_log *log); + +#endif diff --git a/src/lib-index/mailbox-log.c b/src/lib-index/mailbox-log.c new file mode 100644 index 0000000..433ba29 --- /dev/null +++ b/src/lib-index/mailbox-log.c @@ -0,0 +1,292 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "eacces-error.h" +#include "mailbox-log.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> + +/* How often to reopen the log file to make sure that the changes are written + to the latest file. The main problem here is if the value is too high the + changes could be written to a file that was already rotated and deleted. + That wouldn't happen in any real world situations though, since the file + rotation time is probably measured in months or years. Still, each session + rarely writes anything here, so the value can just as well be a pretty small + one without any performance problems. */ +#define MAILBOX_LOG_REOPEN_SECS (60) +#define MAILBOX_LOG_ROTATE_SIZE (1024*4) + +struct mailbox_log { + char *filepath, *filepath2; + int fd; + struct event *event; + time_t open_timestamp; + + mode_t mode; + gid_t gid; + char *gid_origin; +}; + +struct mailbox_log_iter { + struct mailbox_log *log; + + int fd; + const char *filepath; + + struct mailbox_log_record buf[128]; + unsigned int idx, count; + uoff_t offset; + bool failed; +}; + +static void mailbox_log_close(struct mailbox_log *log); + +struct mailbox_log * +mailbox_log_alloc(struct event *parent_event, const char *path) +{ + struct mailbox_log *log; + + log = i_new(struct mailbox_log, 1); + log->event = event_create(parent_event); + log->filepath = i_strdup(path); + log->filepath2 = i_strconcat(path, ".2", NULL); + log->mode = 0644; + log->gid = (gid_t)-1; + log->fd = -1; + return log; +} + +void mailbox_log_free(struct mailbox_log **_log) +{ + struct mailbox_log *log = *_log; + + *_log = NULL; + + mailbox_log_close(log); + event_unref(&log->event); + i_free(log->gid_origin); + i_free(log->filepath); + i_free(log->filepath2); + i_free(log); +} + +static void mailbox_log_close(struct mailbox_log *log) +{ + i_close_fd_path(&log->fd, log->filepath); +} + +void mailbox_log_set_permissions(struct mailbox_log *log, mode_t mode, + gid_t gid, const char *gid_origin) +{ + log->mode = mode; + log->gid = gid; + i_free(log->gid_origin); + log->gid_origin = i_strdup(gid_origin); +} + +static int mailbox_log_open(struct mailbox_log *log) +{ + mode_t old_mode; + + i_assert(log->fd == -1); + + log->open_timestamp = ioloop_time; + log->fd = open(log->filepath, O_RDWR | O_APPEND); + if (log->fd != -1) + return 0; + + /* try to create it */ + old_mode = umask(0666 ^ log->mode); + log->fd = open(log->filepath, O_RDWR | O_APPEND | O_CREAT, 0666); + umask(old_mode); + + if (log->fd == -1) { + if (errno != EACCES) + e_error(log->event, "creat(%s) failed: %m", + log->filepath); + else + e_error(log->event, "%s", + eacces_error_get("creat", log->filepath)); + return -1; + } + if (fchown(log->fd, (uid_t)-1, log->gid) < 0) { + if (errno != EPERM) + e_error(log->event, "fchown(%s) failed: %m", + log->filepath); + else { + e_error(log->event, "%s", + eperm_error_get_chgrp("fchown", + log->filepath, log->gid, + log->gid_origin)); + } + } + return 0; +} + +static int mailbox_log_rotate_if_needed(struct mailbox_log *log) +{ + struct stat st; + + if (fstat(log->fd, &st) < 0) { + e_error(log->event, "fstat(%s) failed: %m", log->filepath); + return -1; + } + + if (st.st_size < MAILBOX_LOG_ROTATE_SIZE) + return 0; + + if (rename(log->filepath, log->filepath2) < 0 && errno != ENOENT) { + e_error(log->event, "rename(%s, %s) failed: %m", + log->filepath, log->filepath2); + return -1; + } + return 0; +} + +void mailbox_log_record_set_timestamp(struct mailbox_log_record *rec, + time_t stamp) +{ + cpu32_to_be_unaligned(stamp, rec->timestamp); +} + +time_t mailbox_log_record_get_timestamp(const struct mailbox_log_record *rec) +{ + return (time_t) be32_to_cpu_unaligned(rec->timestamp); +} + +int mailbox_log_append(struct mailbox_log *log, + const struct mailbox_log_record *rec) +{ + struct stat st; + ssize_t ret; + + /* we don't have to be too strict about appending to the latest log + file. the records' ordering doesn't matter and iteration goes + through both logs anyway. still, if there's a long running session + it shouldn't keep writing to a rotated log forever. */ + if (log->open_timestamp/MAILBOX_LOG_REOPEN_SECS != + ioloop_time/MAILBOX_LOG_REOPEN_SECS) + mailbox_log_close(log); + if (log->fd == -1) { + if (mailbox_log_open(log) < 0) + return -1; + i_assert(log->fd != -1); + } + + /* We don't bother with locking, atomic appends will protect us. + If they don't (NFS), the worst that can happen is that a few + records get overwritten (because they're all the same size). + This whole log isn't supposed to be super-reliable anyway. */ + ret = write(log->fd, rec, sizeof(*rec)); + if (ret < 0) { + e_error(log->event, "write(%s) failed: %m", log->filepath); + return -1; + } else if (ret != sizeof(*rec)) { + e_error(log->event, "write(%s) wrote %d/%u bytes", log->filepath, + (int)ret, (unsigned int)sizeof(*rec)); + if (fstat(log->fd, &st) == 0) { + if (ftruncate(log->fd, st.st_size - ret) < 0) { + e_error(log->event, "ftruncate(%s) failed: %m", + log->filepath); + } + } + return -1; + } + + (void)mailbox_log_rotate_if_needed(log); + return 0; +} + +static bool mailbox_log_iter_open_next(struct mailbox_log_iter *iter) +{ + i_close_fd_path(&iter->fd, iter->filepath); + if (iter->filepath == NULL) + iter->filepath = iter->log->filepath2; + else if (iter->filepath == iter->log->filepath2) + iter->filepath = iter->log->filepath; + else + return FALSE; + + iter->fd = open(iter->filepath, O_RDONLY | O_APPEND); + if (iter->fd != -1) + return TRUE; + else if (errno == ENOENT) { + if (iter->filepath == iter->log->filepath2) + return mailbox_log_iter_open_next(iter); + } else { + e_error(iter->log->event, "open(%s) failed: %m", iter->filepath); + iter->failed = TRUE; + } + return FALSE; +} + +struct mailbox_log_iter *mailbox_log_iter_init(struct mailbox_log *log) +{ + struct mailbox_log_iter *iter; + + iter = i_new(struct mailbox_log_iter, 1); + iter->log = log; + iter->fd = -1; + (void)mailbox_log_iter_open_next(iter); + return iter; +} + +const struct mailbox_log_record * +mailbox_log_iter_next(struct mailbox_log_iter *iter) +{ + const struct mailbox_log_record *rec; + uoff_t offset; + ssize_t ret; + + if (iter->idx == iter->count) { + if (iter->fd == -1) + return NULL; + + ret = pread(iter->fd, iter->buf, sizeof(iter->buf), + iter->offset); + if (ret < 0) { + e_error(iter->log->event, "pread(%s) failed: %m", + iter->filepath); + iter->failed = TRUE; + return NULL; + } + if (ret == 0) { + if (!mailbox_log_iter_open_next(iter)) + return NULL; + iter->idx = iter->count = 0; + iter->offset = 0; + return mailbox_log_iter_next(iter); + } + iter->idx = 0; + iter->count = ret / sizeof(iter->buf[0]); + iter->offset += iter->count * sizeof(iter->buf[0]); + } + rec = &iter->buf[iter->idx++]; + if (rec->type < MAILBOX_LOG_RECORD_DELETE_MAILBOX || + rec->type > MAILBOX_LOG_RECORD_UNSUBSCRIBE) { + offset = iter->offset - + (iter->count - iter->idx) * sizeof(iter->buf[0]); + e_error(iter->log->event, + "Corrupted mailbox log %s at offset %"PRIuUOFF_T": " + "type=%d", iter->filepath, offset, rec->type); + i_unlink(iter->filepath); + return NULL; + } + return rec; +} + +int mailbox_log_iter_deinit(struct mailbox_log_iter **_iter) +{ + struct mailbox_log_iter *iter = *_iter; + int ret = iter->failed ? -1 : 0; + + *_iter = NULL; + + i_close_fd_path(&iter->fd, iter->filepath); + i_free(iter); + return ret; +} diff --git a/src/lib-index/mailbox-log.h b/src/lib-index/mailbox-log.h new file mode 100644 index 0000000..ef19fc2 --- /dev/null +++ b/src/lib-index/mailbox-log.h @@ -0,0 +1,44 @@ +#ifndef MAILBOX_LOG_H +#define MAILBOX_LOG_H + +#include "guid.h" + +enum mailbox_log_record_type { + MAILBOX_LOG_RECORD_DELETE_MAILBOX = 1, + MAILBOX_LOG_RECORD_DELETE_DIR, + MAILBOX_LOG_RECORD_RENAME, + MAILBOX_LOG_RECORD_SUBSCRIBE, + MAILBOX_LOG_RECORD_UNSUBSCRIBE, + MAILBOX_LOG_RECORD_CREATE_DIR +}; + +struct mailbox_log_record { + uint8_t type; + uint8_t padding[3]; + guid_128_t mailbox_guid; + uint8_t timestamp[4]; +}; + +struct mailbox_log * +mailbox_log_alloc(struct event *parent_event, const char *path); +void mailbox_log_free(struct mailbox_log **log); + +void mailbox_log_set_permissions(struct mailbox_log *log, mode_t mode, + gid_t gid, const char *gid_origin); + +void mailbox_log_record_set_timestamp(struct mailbox_log_record *rec, + time_t stamp); +time_t mailbox_log_record_get_timestamp(const struct mailbox_log_record *rec); + +/* Append a new record to mailbox log. Returns 0 if ok, -1 if error. */ +int mailbox_log_append(struct mailbox_log *log, + const struct mailbox_log_record *rec); + +/* Iterate through all records in mailbox log. */ +struct mailbox_log_iter *mailbox_log_iter_init(struct mailbox_log *log); +const struct mailbox_log_record * +mailbox_log_iter_next(struct mailbox_log_iter *iter); +/* Returns 0 if ok, -1 if I/O error. */ +int mailbox_log_iter_deinit(struct mailbox_log_iter **iter); + +#endif diff --git a/src/lib-index/test-mail-cache-common.c b/src/lib-index/test-mail-cache-common.c new file mode 100644 index 0000000..ee34c04 --- /dev/null +++ b/src/lib-index/test-mail-cache-common.c @@ -0,0 +1,166 @@ +/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "test-common.h" +#include "test-mail-cache.h" + +static const struct mail_cache_field cache_field_foo = { + .name = "foo", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES, +}; +static const struct mail_cache_field cache_field_bar = { + .name = "bar", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES, +}; +static const struct mail_cache_field cache_field_baz = { + .name = "baz", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES, +}; + +void test_mail_cache_init(struct mail_index *index, + struct test_mail_cache_ctx *ctx_r) +{ + i_zero(ctx_r); + ctx_r->index = index; + ctx_r->cache = index->cache; + ctx_r->view = mail_index_view_open(index); + + ctx_r->cache_field = cache_field_foo; + ctx_r->cache_field2 = cache_field_bar; + ctx_r->cache_field3 = cache_field_baz; + /* Try to use different file_field_maps for different index instances + by randomizing the registration order. This only works for the 2nd + index that is opened, because the initial cache is always created + with all cache fields in the same order. */ + if (i_rand_limit(2) == 0) { + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1); + } else { + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1); + } +} + +void test_mail_cache_deinit(struct test_mail_cache_ctx *ctx) +{ + if (ctx->view != NULL) + mail_index_view_close(&ctx->view); + test_mail_index_close(&ctx->index); +} + +unsigned int test_mail_cache_get_purge_count(struct test_mail_cache_ctx *ctx) +{ + const struct mail_cache_header *hdr = ctx->cache->hdr; + + return hdr->file_seq - hdr->indexid; +} + +void test_mail_cache_index_sync(struct test_mail_cache_ctx *ctx) +{ + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + struct mail_index_sync_rec sync_rec; + + test_assert(mail_index_sync_begin(ctx->index, &sync_ctx, + &view, &trans, 0) == 1); + while (mail_index_sync_next(sync_ctx, &sync_rec)) { + if (sync_rec.type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + /* we're a bit kludgily assuming that there's only + one UID and also that uid==seq */ + mail_index_expunge(trans, sync_rec.uid1); + } + } + test_assert(mail_index_sync_commit(&sync_ctx) == 0); +} + +void test_mail_cache_view_sync(struct test_mail_cache_ctx *ctx) +{ + struct mail_index_view_sync_ctx *sync_ctx; + struct mail_index_view_sync_rec sync_rec; + bool delayed_expunges; + + sync_ctx = mail_index_view_sync_begin(ctx->view, MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + while (mail_index_view_sync_next(sync_ctx, &sync_rec)) ; + test_assert(mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) == 0); +} + +void test_mail_cache_purge(void) +{ + struct test_mail_cache_ctx ctx; + + test_mail_cache_init(test_mail_index_open(), &ctx); + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + test_mail_cache_deinit(&ctx); +} + +void test_mail_cache_add_mail(struct test_mail_cache_ctx *ctx, + unsigned int cache_field_idx, + const char *cache_data) +{ + const struct mail_index_header *hdr = mail_index_get_header(ctx->view); + struct mail_index_transaction *trans; + struct mail_index_view *updated_view; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + uint32_t seq, uid_validity = 12345; + + trans = mail_index_transaction_begin(ctx->view, 0); + updated_view = mail_index_transaction_open_updated_view(trans); + cache_view = mail_cache_view_open(ctx->cache, updated_view); + cache_trans = mail_cache_get_transaction(cache_view, trans); + + if (hdr->uid_validity == 0) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + mail_index_append(trans, hdr->next_uid, &seq); + if (cache_field_idx != UINT_MAX) { + mail_cache_add(cache_trans, seq, cache_field_idx, + cache_data, strlen(cache_data)); + } + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&updated_view); + mail_cache_view_close(&cache_view); + + /* View needs to have the latest changes before purge transaction + is created. */ + test_mail_cache_view_sync(ctx); +} + +void test_mail_cache_add_field(struct test_mail_cache_ctx *ctx, uint32_t seq, + unsigned int cache_field_idx, + const char *cache_data) +{ + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + + cache_view = mail_cache_view_open(ctx->cache, ctx->view); + trans = mail_index_transaction_begin(ctx->view, 0); + cache_trans = mail_cache_get_transaction(cache_view, trans); + mail_cache_add(cache_trans, seq, cache_field_idx, + cache_data, strlen(cache_data)); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_cache_view_close(&cache_view); +} + +void test_mail_cache_update_day_first_uid7(struct test_mail_cache_ctx *ctx, + uint32_t first_new_uid) +{ + struct mail_index_transaction *trans; + + trans = mail_index_transaction_begin(ctx->view, 0); + mail_index_update_header(trans, + offsetof(struct mail_index_header, day_first_uid[7]), + &first_new_uid, sizeof(first_new_uid), FALSE); + test_assert(mail_index_transaction_commit(&trans) == 0); + test_mail_cache_view_sync(ctx); +} diff --git a/src/lib-index/test-mail-cache-fields.c b/src/lib-index/test-mail-cache-fields.c new file mode 100644 index 0000000..18b6a33 --- /dev/null +++ b/src/lib-index/test-mail-cache-fields.c @@ -0,0 +1,112 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "test-common.h" +#include "test-mail-cache.h" + +static void test_mail_cache_fields_read_write(void) +{ + struct mail_cache_field cache_field = { + .name = "testfield", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_NO, + .last_used = 0x12345678, + }; + struct mail_cache_field cache_field2 = { + .name = "testfield2", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_NO, + .last_used = 0xaabbccdd, + }; + struct test_mail_cache_ctx ctx; + + test_begin("mail cache fields read-write"); + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_cache_register_fields(ctx.cache, &cache_field, 1); + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + /* after writing the initial cache file, register another cache field + that doesn't exist in it. */ + mail_cache_register_fields(ctx.cache, &cache_field2, 1); + + struct mail_cache_field_private *priv = + &ctx.cache->fields[cache_field.idx]; + struct mail_cache_field_private *priv2 = + &ctx.cache->fields[cache_field2.idx]; + + /* No changes */ + test_assert(mail_cache_header_fields_update(ctx.cache) == 0); + test_assert(mail_cache_header_fields_read(ctx.cache) == 0); + test_assert(cache_field.last_used == priv->field.last_used && + cache_field.decision == priv->field.decision); + test_assert(cache_field2.last_used == priv2->field.last_used && + cache_field2.decision == priv2->field.decision); + + /* Replace decision without marking it dirty. Make sure reading + overwrites it. Also make sure an old last_used is overwritten. */ + priv->field.decision = MAIL_CACHE_DECISION_YES; + priv->field.last_used = cache_field.last_used - 1; + test_assert(mail_cache_header_fields_read(ctx.cache) == 0); + test_assert(cache_field.last_used == priv->field.last_used && + cache_field.decision == priv->field.decision); + test_assert(cache_field2.last_used == priv2->field.last_used && + cache_field2.decision == priv2->field.decision); + + /* Replace decision and set it dirty. Make sure reading doesn't + overwrite it. Also make sure an old last_used is overwritten. */ + priv->decision_dirty = TRUE; + priv2->decision_dirty = TRUE; + priv->field.last_used = cache_field.last_used - 1; + priv->field.decision = MAIL_CACHE_DECISION_YES; + cache_field.decision = MAIL_CACHE_DECISION_YES; + priv2->field.decision = MAIL_CACHE_DECISION_YES; + cache_field2.decision = MAIL_CACHE_DECISION_YES; + test_assert(mail_cache_header_fields_read(ctx.cache) == 0); + test_assert(cache_field.last_used == priv->field.last_used && + cache_field.decision == priv->field.decision); + test_assert(cache_field2.last_used == priv2->field.last_used && + cache_field2.decision == priv2->field.decision); + test_assert(priv->decision_dirty); + test_assert(priv2->decision_dirty); + + /* Make sure a new last_used won't get overwritten by read. */ + priv->field.last_used = ++cache_field.last_used; + priv2->field.last_used = ++cache_field2.last_used; + test_assert(mail_cache_header_fields_read(ctx.cache) == 0); + test_assert(cache_field.last_used == priv->field.last_used && + cache_field.decision == priv->field.decision); + test_assert(cache_field2.last_used == priv2->field.last_used && + cache_field2.decision == priv2->field.decision); + + /* Write the new decision and last_used. Note that cache_field2 + isn't written, because it doesn't exist in the cache file. */ + test_assert(mail_cache_header_fields_update(ctx.cache) == 0); + test_assert(!priv->decision_dirty); + test_assert(priv2->decision_dirty); + /* make sure reading reads them back, even if they're changed */ + priv->field.decision = MAIL_CACHE_DECISION_NO; + priv->field.last_used = 1; + priv2->field.decision = MAIL_CACHE_DECISION_TEMP; + priv2->field.last_used = 2; + cache_field2.decision = MAIL_CACHE_DECISION_TEMP; + cache_field2.last_used = 2; + test_assert(mail_cache_header_fields_read(ctx.cache) == 0); + test_assert(cache_field.last_used == priv->field.last_used && + cache_field.decision == priv->field.decision); + test_assert(cache_field2.last_used == priv2->field.last_used && + cache_field2.decision == priv2->field.decision); + + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_cache_fields_read_write, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-cache-purge.c b/src/lib-index/test-mail-cache-purge.c new file mode 100644 index 0000000..525754b --- /dev/null +++ b/src/lib-index/test-mail-cache-purge.c @@ -0,0 +1,1076 @@ +/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "array.h" +#include "test-common.h" +#include "test-mail-cache.h" + +#include <stdio.h> +#include <sys/wait.h> + +static void test_mail_cache_read_during_purge2(void) +{ + struct test_mail_cache_ctx ctx; + struct mail_cache_view *cache_view; + string_t *str = t_str_new(16); + + i_set_failure_prefix("index2: "); + + /* read from cache via 2nd index */ + test_mail_cache_init(test_mail_index_open(), &ctx); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 1); + test_assert(strcmp(str_c(str), "foo1") == 0); + mail_cache_view_close(&cache_view); + + test_mail_cache_deinit(&ctx); +} + +static void test_mail_cache_read_during_purge(void) +{ + struct test_mail_cache_ctx ctx; + struct mail_index_transaction *trans; + int status; + + test_begin("mail cache read during purge"); + test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + + /* lock the index for cache purge */ + uint32_t log_seq; + uoff_t log_offset; + test_assert(mail_transaction_log_sync_lock(ctx.index->log, "purge", &log_seq, &log_offset) == 0); + + /* start purging cache using the 1st index, but don't commit yet */ + trans = mail_index_transaction_begin(ctx.view, 0); + test_assert(mail_cache_purge_with_trans(ctx.cache, trans, (uint32_t)-1, "test") == 0); + + switch (fork()) { + case (pid_t)-1: + i_fatal("fork() failed: %m"); + case 0: + test_mail_cache_read_during_purge2(); + /* cleanup so valgrind doesn't complain about memory leaks */ + mail_index_transaction_rollback(&trans); + mail_transaction_log_sync_unlock(ctx.index->log, "purge"); + test_mail_cache_deinit(&ctx); + test_exit(test_has_failed() ? 10 : 0); + default: + break; + } + + /* Wait a bit to make sure the child function has had a chance to run. + It's supposed to be waiting on the locked .log file. */ + usleep(100000); + /* finish cache purging */ + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_transaction_log_sync_unlock(ctx.index->log, "purge"); + mail_index_view_close(&ctx.view); + + /* wait for child to finish execution */ + if (wait(&status) == -1) + i_error("wait() failed: %m"); + test_assert(status == 0); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static void test_mail_cache_write_during_purge2(void) +{ + struct test_mail_cache_ctx ctx; + + i_set_failure_prefix("index2: "); + + /* add to cache via 2nd index */ + test_mail_cache_init(test_mail_index_open(), &ctx); + test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar2"); + test_mail_cache_deinit(&ctx); +} + +static void test_mail_cache_write_during_purge(void) +{ + struct test_mail_cache_ctx ctx; + struct mail_index_view *view; + struct mail_cache_view *cache_view; + struct mail_index_transaction *trans; + string_t *str = t_str_new(16); + int status; + + test_begin("mail cache write during purge"); + test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + + /* lock the index for cache purge */ + uint32_t log_seq; + uoff_t log_offset; + test_assert(mail_transaction_log_sync_lock(ctx.index->log, "purge", &log_seq, &log_offset) == 0); + + /* start purging cache using the 1st index, but don't commit yet */ + trans = mail_index_transaction_begin(ctx.view, 0); + test_assert(mail_cache_purge_with_trans(ctx.cache, trans, (uint32_t)-1, "test") == 0); + + switch (fork()) { + case (pid_t)-1: + i_fatal("fork() failed: %m"); + case 0: + test_mail_cache_write_during_purge2(); + /* cleanup so valgrind doesn't complain about memory leaks */ + mail_index_transaction_rollback(&trans); + mail_transaction_log_sync_unlock(ctx.index->log, "purge"); + test_mail_cache_deinit(&ctx); + test_exit(test_has_failed() ? 10 : 0); + default: + break; + } + + /* Wait a bit to make sure the child function has had a chance to run. + It's supposed to be waiting on the locked .log file. */ + usleep(100000); + /* finish cache purge */ + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_transaction_log_sync_unlock(ctx.index->log, "purge"); + mail_index_view_close(&ctx.view); + + /* wait for child to finish execution */ + if (wait(&status) == -1) + i_error("wait() failed: %m"); + test_assert(status == 0); + + /* make sure both cache fields are visible */ + test_assert(mail_index_refresh(ctx.index) == 0); + + view = mail_index_view_open(ctx.index); + cache_view = mail_cache_view_open(ctx.cache, view); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 1); + test_assert(strcmp(str_c(str), "foo1") == 0); + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field2.idx) == 1); + test_assert(strcmp(str_c(str), "bar2") == 0); + mail_cache_view_close(&cache_view); + mail_index_view_close(&view); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static void test_mail_cache_purge_while_cache_locked(void) +{ + struct test_mail_cache_ctx ctx; + struct mail_cache_view *cache_view; + string_t *str = t_str_new(16); + int status; + + test_begin("mail cache purge while cache locked"); + test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + + /* lock the cache */ + test_assert(mail_cache_lock(ctx.cache) == 1); + + /* purge the cache in another process */ + switch (fork()) { + case (pid_t)-1: + i_fatal("fork() failed: %m"); + case 0: + test_mail_cache_purge(); + test_mail_cache_deinit(&ctx); + test_exit(test_has_failed() ? 10 : 0); + default: + break; + } + + /* Wait a bit to make sure the child function has had a chance to run. + It should start purging, which would wait for our cache lock. */ + usleep(100000); + + mail_cache_unlock(ctx.cache); + + /* wait for child to finish execution */ + if (wait(&status) == -1) + i_error("wait() failed: %m"); + test_assert(status == 0); + + /* make sure the cache is still usable */ + test_assert(mail_index_refresh(ctx.index) == 0); + test_mail_cache_view_sync(&ctx); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 1); + test_assert(strcmp(str_c(str), "foo1") == 0); + mail_cache_view_close(&cache_view); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static bool cache_equals(struct mail_cache_view *cache_view, uint32_t seq, + unsigned int field_idx, const char *value) +{ + string_t *str = str_new(default_pool, 128); + int ret = mail_cache_lookup_field(cache_view, str, seq, field_idx); + bool match; + + if (value != NULL) { + test_assert_idx(ret == 1, seq); + match = strcmp(str_c(str), value) == 0; + test_assert_idx(match, seq); + } else { + test_assert_idx(ret == 0, seq); + match = ret == 0; + } + str_free(&str); + return match; +} + +static void test_mail_cache_purge_during_write_n(unsigned int num_mails, + bool commit_saves) +{ + const struct mail_index_optimization_settings optimization_set = { + .cache = { + .record_max_size = 1024*1024, + }, + }; + struct test_mail_cache_ctx ctx; + struct mail_index_view *updated_view; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + struct mail_index_transaction *trans; + uint32_t seq; + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + + /* Add mails */ + test_mail_cache_add_mail(&ctx, UINT_MAX, ""); + trans = mail_index_transaction_begin(ctx.view, 0); + for (seq = 2; seq <= num_mails; seq++) + mail_index_append(trans, seq, &seq); + + if (commit_saves) { + test_assert(mail_index_transaction_commit(&trans) == 0); + test_mail_cache_view_sync(&ctx); + trans = mail_index_transaction_begin(ctx.view, 0); + } + /* start adding a small cached field to mail1 */ + updated_view = mail_index_transaction_open_updated_view(trans); + cache_view = mail_cache_view_open(ctx.cache, updated_view); + cache_trans = mail_cache_get_transaction(cache_view, trans); + mail_cache_add(cache_trans, 1, ctx.cache_field.idx, "foo1", 4); + + /* add a huge field to mail2, which triggers flushing */ + size_t huge_field_size = MAIL_CACHE_MAX_WRITE_BUFFER + 1024; + char *huge_field = i_malloc(huge_field_size + 1); + memset(huge_field, 'x', huge_field_size); + mail_cache_add(cache_trans, 2, ctx.cache_field.idx, + huge_field, huge_field_size); + + /* verify that cached fields are still accessible */ + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1")); + test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field)); + + /* purge using a 2nd index */ + test_mail_cache_purge(); + + if (num_mails == 2) { + /* the mails are still accessible after purge */ + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1")); + test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field)); + } else if (!commit_saves) { + /* add 3rd mail, which attempts to flush 2nd mail and finds + that the first mail is already lost */ + test_expect_error_string("Purging lost 1 written cache records"); + mail_cache_add(cache_trans, 3, ctx.cache_field.idx, "foo3", 4); + test_expect_no_more_errors(); + + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL)); + test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field)); + test_assert(cache_equals(cache_view, 3, ctx.cache_field.idx, "foo3")); + } else { + /* add 3rd mail, which commits the first two mails */ + mail_cache_add(cache_trans, 3, ctx.cache_field.idx, "foo3", 4); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1")); + test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field)); + test_assert(cache_equals(cache_view, 3, ctx.cache_field.idx, "foo3")); + } + + /* finish committing cached fields */ + if (num_mails == 2 && !commit_saves) + test_expect_error_string("Purging lost 1 written cache records"); + test_assert(mail_index_transaction_commit(&trans) == 0); + test_expect_no_more_errors(); + mail_index_view_close(&updated_view); + mail_cache_view_close(&cache_view); + + /* see that we lost the first flush without commit_saves, but not the others */ + test_mail_cache_view_sync(&ctx); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + if (commit_saves) + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1")); + else + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL)); + test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field)); + if (num_mails >= 3) + test_assert(cache_equals(cache_view, 3, ctx.cache_field.idx, "foo3")); + mail_cache_view_close(&cache_view); + + mail_index_view_close(&ctx.view); + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + i_free(huge_field); +} + +static void test_mail_cache_write_lost_during_purge(void) +{ + test_begin("mail cache write lost during purge"); + test_mail_cache_purge_during_write_n(2, FALSE); + test_end(); +} + +static void test_mail_cache_write_lost_during_purge2(void) +{ + test_begin("mail cache write lost during purge (2)"); + test_mail_cache_purge_during_write_n(3, FALSE); + test_end(); +} + +static void test_mail_cache_write_autocommit(void) +{ + test_begin("mail cache write autocommit"); + test_mail_cache_purge_during_write_n(2, TRUE); + test_end(); +} + +static void test_mail_cache_write_autocommit2(void) +{ + test_begin("mail cache write autocommit"); + test_mail_cache_purge_during_write_n(3, TRUE); + test_end(); +} + +static size_t max_field_size(size_t max_size, size_t current_size) +{ + return max_size - current_size + - sizeof(struct mail_cache_record) + - sizeof(uint32_t) /* field_idx */ + - sizeof(uint32_t); /* data_size */ +} + +static void test_mail_cache_delete_too_large_int(bool exceed_on_first_write) +{ + const struct mail_index_optimization_settings optimization_set = { + .cache = { + .max_size = 1024, + }, + }; + struct test_mail_cache_ctx ctx; + struct stat st; + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo2"); + + test_assert(stat(ctx.index->cache->filepath, &st) == 0); + + /* create cache file that is exactly max_size */ + size_t field_size = + max_field_size(optimization_set.cache.max_size, st.st_size); + if (exceed_on_first_write) { + test_expect_error_string("Cache file too large"); + field_size++; + } + char *field = i_malloc(field_size + 1); + memset(field, 'x', field_size); + test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, field); + test_expect_no_more_errors(); + i_free(field); + + if (!exceed_on_first_write) { + test_assert(stat(ctx.index->cache->filepath, &st) == 0); + test_assert(st.st_size == 1024); + + /* adding anything more will delete the cache. */ + test_expect_error_string("Cache file too large"); + test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar1"); + test_expect_no_more_errors(); + } + test_assert(stat(ctx.index->cache->filepath, &st) < 0 && errno == ENOENT); + + mail_index_view_close(&ctx.view); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_delete_too_large(void) +{ + test_begin("mail cache delete too large"); + test_mail_cache_delete_too_large_int(FALSE); + test_end(); +} + +static void test_mail_cache_delete_too_large2(void) +{ + test_begin("mail cache delete too large (2)"); + test_mail_cache_delete_too_large_int(TRUE); + test_end(); +} + +static void test_mail_cache_purge_too_large_int(bool exceed_size) +{ + const struct mail_index_optimization_settings optimization_set = { + .cache = { + .max_size = 1024, + }, + }; + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + struct test_mail_cache_ctx ctx; + struct stat st; + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + + /* add two mails with some cache field and expunge the first mail */ + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "bar2"); + trans = mail_index_transaction_begin(ctx.view, 0); + mail_index_expunge(trans, 1); + test_assert(mail_index_transaction_commit(&trans) == 0); + test_mail_cache_index_sync(&ctx); + + /* Add a second mail whose cache field size is exactly the + max_size [+1 if exceed_size] */ + test_assert(stat(ctx.index->cache->filepath, &st) == 0); + size_t field_size = (exceed_size ? 1 : 0) + + max_field_size(optimization_set.cache.max_size, st.st_size); + char *field = i_malloc(field_size + 1); + memset(field, 'x', field_size); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, field); + + test_assert(stat(ctx.index->cache->filepath, &st) == 0); + if (exceed_size) + test_assert((uoff_t)st.st_size < optimization_set.cache.max_size); + else + test_assert((uoff_t)st.st_size == optimization_set.cache.max_size); + + /* make sure we still find the cache fields */ + test_mail_cache_view_sync(&ctx); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "bar2")); + test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, field)); + mail_cache_view_close(&cache_view); + + i_free(field); + if (exceed_size) + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + else + test_assert(test_mail_cache_get_purge_count(&ctx) == 0); + mail_index_view_close(&ctx.view); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_purge_too_large(void) +{ + test_begin("mail cache purge too large"); + test_mail_cache_purge_too_large_int(FALSE); + test_end(); +} + +static void test_mail_cache_purge_too_large2(void) +{ + test_begin("mail cache purge too large (2)"); + test_mail_cache_purge_too_large_int(TRUE); + test_end(); +} + +static void test_mail_cache_unexpectedly_lost_int(bool read_first) +{ + struct test_mail_cache_ctx ctx; + struct mail_cache_view *cache_view; + + test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + + test_mail_cache_purge(); + + /* Unexpectedly delete the cache file under us */ + i_unlink(ctx.cache->filepath); + + if (read_first) { + /* the cache file is already open, so initial reading should + work without errors */ + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1")); + mail_cache_view_close(&cache_view); + + /* if we refresh the index we get new reset_id, which requires + reopening the cache and that fails */ + test_assert(mail_index_refresh(ctx.index) == 0); + test_mail_cache_view_sync(&ctx); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_expect_error_string("test.dovecot.index.cache: No such file or directory"); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL)); + test_expect_no_more_errors(); + mail_cache_view_close(&cache_view); + } else { + test_expect_error_string("test.dovecot.index.cache: No such file or directory"); + } + + /* writing after losing the cache should still work */ + test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar1"); + test_expect_no_more_errors(); + + /* verify that the second cache field is found, but first is lost */ + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL)); + test_assert(cache_equals(cache_view, 1, ctx.cache_field2.idx, "bar1")); + mail_cache_view_close(&cache_view); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 2); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_unexpectedly_lost(void) +{ + test_begin("mail cache unexpectedly lost"); + test_mail_cache_unexpectedly_lost_int(FALSE); + test_end(); +} + +static void test_mail_cache_unexpectedly_lost2(void) +{ + test_begin("mail cache unexpectedly lost (2)"); + test_mail_cache_unexpectedly_lost_int(TRUE); + test_end(); +} + +static void test_mail_cache_resetid_mismatch_int(bool read_first) +{ + struct test_mail_cache_ctx ctx; + struct mail_cache_view *cache_view; + const char *temp_cache_path; + + test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + + /* make a copy of the first cache file */ + temp_cache_path = t_strdup_printf("%s.test", ctx.cache->filepath); + test_assert(link(ctx.cache->filepath, temp_cache_path) == 0); + + if (read_first) { + /* use a secondary index to purge the cache */ + test_mail_cache_purge(); + + /* Replace the new cache file with an old one */ + test_assert(rename(temp_cache_path, ctx.cache->filepath) == 0); + + /* the cache file is already open, so initial reading should + work without errors */ + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1")); + mail_cache_view_close(&cache_view); + + /* if we refresh the index we get new reset_id, which requires + reopening the cache and that fails */ + test_assert(mail_index_refresh(ctx.index) == 0); + test_mail_cache_view_sync(&ctx); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + + test_expect_error_string("reset_id mismatch even after locking"); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL)); + test_expect_no_more_errors(); + mail_cache_view_close(&cache_view); + } else { + /* purge cache to update reset_id in index */ + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + + /* Replace the new cache file with an old one */ + test_assert(rename(temp_cache_path, ctx.cache->filepath) == 0); + + test_expect_error_string("reset_id mismatch even after locking"); + } + + /* writing should automatically fix the reset_id mismatch */ + test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar1"); + test_expect_no_more_errors(); + + /* verify that the second cache field is found, but first is lost */ + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL)); + test_assert(cache_equals(cache_view, 1, ctx.cache_field2.idx, "bar1")); + mail_cache_view_close(&cache_view); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 2); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_resetid_mismatch(void) +{ + test_begin("mail cache resetid mismatch"); + test_mail_cache_resetid_mismatch_int(FALSE); + test_end(); +} + +static void test_mail_cache_resetid_mismatch2(void) +{ + test_begin("mail cache resetid mismatch (2)"); + test_mail_cache_resetid_mismatch_int(TRUE); + test_end(); +} + +enum test_drop { + TEST_DROP_NOTHING, + TEST_DROP_YES_TO_TEMP_FIRST, + TEST_DROP_YES_TO_TEMP_LAST, + TEST_DROP_TEMP_TO_NO, +}; + +static void test_mail_cache_purge_field_changes_int(enum test_drop drop) +{ + enum { + TEST_FIELD_NO, + TEST_FIELD_NO_FORCED, + TEST_FIELD_TEMP, + TEST_FIELD_TEMP_FORCED, + TEST_FIELD_YES, + TEST_FIELD_YES_FORCED, + }; + struct mail_cache_field cache_fields[] = { + { + .name = "no", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_NO, + }, + { + .name = "no-forced", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED, + }, + { + .name = "temp", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_TEMP, + }, + { + .name = "temp-forced", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_TEMP | MAIL_CACHE_DECISION_FORCED, + }, + { + .name = "yes", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES, + }, + { + .name = "yes-forced", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES | MAIL_CACHE_DECISION_FORCED, + }, + }; + const struct mail_index_optimization_settings optimization_set = { + .cache = { + .unaccessed_field_drop_secs = 61, + }, + }; + struct test_mail_cache_ctx ctx; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + struct mail_index_transaction *trans; + unsigned int i; + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + + /* add two mails with all of the cache fields */ + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + + /* Create the cache file before registering any of the cache_fields + that we're testing. Otherwise our caching decisions are messed up + by purging (which is called to auto-create the cache). */ + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + mail_cache_register_fields(ctx.cache, cache_fields, + N_ELEMENTS(cache_fields)); + + trans = mail_index_transaction_begin(ctx.view, 0); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + cache_trans = mail_cache_get_transaction(cache_view, trans); + for (i = 0; i < N_ELEMENTS(cache_fields); i++) { + const char *value = t_strdup_printf("%s-value", + cache_fields[i].name); + if ((cache_fields[i].decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) != + MAIL_CACHE_DECISION_NO) { + mail_cache_add(cache_trans, 1, cache_fields[i].idx, + value, strlen(value)); + mail_cache_add(cache_trans, 2, cache_fields[i].idx, + value, strlen(value)); + } + } + + /* day_stamp in index is used for deciding when a cache field needs to + be dropped. */ + uint32_t day_stamp = 123456789; + mail_index_update_header(trans, + offsetof(struct mail_index_header, day_stamp), + &day_stamp, sizeof(day_stamp), FALSE); + /* day_first_uid[7] is used to determine which mails are "old" and + which mails are "new". [7] is the first "new" mail. */ + uint32_t first_new_uid = 2; + mail_index_update_header(trans, + offsetof(struct mail_index_header, day_first_uid[7]), + &first_new_uid, sizeof(first_new_uid), FALSE); + test_assert(mail_index_transaction_commit(&trans) == 0); + + /* set the last_used time just at the boundary of being dropped or + being kept */ + for (i = 0; i < ctx.cache->fields_count; i++) { + unsigned int secs = optimization_set.cache.unaccessed_field_drop_secs; + switch (drop) { + case TEST_DROP_NOTHING: + break; + case TEST_DROP_YES_TO_TEMP_FIRST: + secs++; + break; + case TEST_DROP_YES_TO_TEMP_LAST: + secs *= 2; + break; + case TEST_DROP_TEMP_TO_NO: + secs *= 2; + secs++; + break; + } + ctx.cache->fields[i].field.last_used = day_stamp - secs; + } + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + test_mail_cache_view_sync(&ctx); + + /* verify that caching decisions are as expected after purging */ + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO].idx].field.decision == + MAIL_CACHE_DECISION_NO); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO_FORCED].idx].field.decision == + (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED)); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP_FORCED].idx].field.decision == + (MAIL_CACHE_DECISION_TEMP | MAIL_CACHE_DECISION_FORCED)); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES_FORCED].idx].field.decision == + (MAIL_CACHE_DECISION_YES | MAIL_CACHE_DECISION_FORCED)); + + switch (drop) { + case TEST_DROP_NOTHING: + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP].idx].field.decision == + MAIL_CACHE_DECISION_TEMP); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES].idx].field.decision == + MAIL_CACHE_DECISION_YES); + break; + case TEST_DROP_YES_TO_TEMP_FIRST: + case TEST_DROP_YES_TO_TEMP_LAST: + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP].idx].field.decision == + MAIL_CACHE_DECISION_TEMP); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES].idx].field.decision == + MAIL_CACHE_DECISION_TEMP); + break; + case TEST_DROP_TEMP_TO_NO: + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP].idx].field.decision == + MAIL_CACHE_DECISION_NO); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES].idx].field.decision == + MAIL_CACHE_DECISION_NO); + break; + } + + /* verify that cache fields exist as expected after purging */ + test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_NO].idx, NULL)); + test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_NO].idx, NULL)); + test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_NO_FORCED].idx, NULL)); + test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_NO_FORCED].idx, NULL)); + test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_TEMP].idx, NULL)); + if (drop == TEST_DROP_TEMP_TO_NO) + test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_TEMP].idx, NULL)); + else + test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_TEMP].idx, "temp-value")); + test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_TEMP_FORCED].idx, NULL)); + test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_TEMP_FORCED].idx, "temp-forced-value")); + if (drop != TEST_DROP_NOTHING) + test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_YES].idx, NULL)); + else + test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_YES].idx, "yes-value")); + test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_YES_FORCED].idx, "yes-forced-value")); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + mail_cache_view_close(&cache_view); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_purge_field_changes(void) +{ + test_begin("mail cache purge field changes (nothing)"); + test_mail_cache_purge_field_changes_int(TEST_DROP_NOTHING); + test_end(); +} + +static void test_mail_cache_purge_field_changes2(void) +{ + test_begin("mail cache purge field changes (yes -> temp, first)"); + test_mail_cache_purge_field_changes_int(TEST_DROP_YES_TO_TEMP_FIRST); + test_end(); +} + +static void test_mail_cache_purge_field_changes3(void) +{ + test_begin("mail cache purge field changes (yes -> temp, last)"); + test_mail_cache_purge_field_changes_int(TEST_DROP_YES_TO_TEMP_LAST); + test_end(); +} + +static void test_mail_cache_purge_field_changes4(void) +{ + test_begin("mail cache purge field changes (temp -> no)"); + test_mail_cache_purge_field_changes_int(TEST_DROP_TEMP_TO_NO); + test_end(); +} + +static void test_mail_cache_purge_already_done(void) +{ + struct test_mail_cache_ctx ctx; + + test_begin("mail cache purge already done"); + test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); + + test_mail_cache_purge(); + test_assert(mail_cache_purge(ctx.cache, 1, "test") == 0); + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + + test_assert(mail_cache_purge(ctx.cache, 2, "test") == 0); + test_assert(test_mail_cache_get_purge_count(&ctx) == 2); + + test_assert(mail_cache_purge(ctx.cache, 2, "test") == 0); + test_assert(test_mail_cache_get_purge_count(&ctx) == 2); + + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static void test_mail_cache_purge_bitmask(void) +{ + struct mail_index_optimization_settings optimization_set = { + .cache = { + .unaccessed_field_drop_secs = 60, + }, + }; + struct mail_cache_field bitmask_field = { + .name = "bitmask", + .type = MAIL_CACHE_FIELD_BITMASK, + .field_size = 1, + .decision = MAIL_CACHE_DECISION_TEMP, + }; + struct test_mail_cache_ctx ctx; + struct mail_cache_view *cache_view; + + test_begin("mail cache purge bitmask"); + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + ioloop_time = 1000000; + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + mail_cache_register_fields(ctx.cache, &bitmask_field, 1); + + test_mail_cache_update_day_first_uid7(&ctx, 3); + + test_mail_cache_add_field(&ctx, 1, bitmask_field.idx, "\x01"); + test_mail_cache_add_field(&ctx, 1, bitmask_field.idx, "\x02"); + test_mail_cache_add_field(&ctx, 1, bitmask_field.idx, "\x04"); + test_mail_cache_add_field(&ctx, 2, bitmask_field.idx, "\x01"); + test_mail_cache_add_field(&ctx, 2, bitmask_field.idx, "\x02"); + test_mail_cache_add_field(&ctx, 2, bitmask_field.idx, "\x04"); + + /* avoid dropping the field */ + ctx.cache->fields[bitmask_field.idx].field.last_used = ioloop_time; + + /* purge with TEMP decision, which causes the bitmask to be dropped */ + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(cache_equals(cache_view, 1, bitmask_field.idx, NULL)); + test_assert(cache_equals(cache_view, 2, bitmask_field.idx, NULL)); + mail_cache_view_close(&cache_view); + + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + + +static void +test_mail_cache_update_need_purge_continued_records_int(bool big_min_size) +{ + struct mail_index_optimization_settings optimization_set = { + .cache = { + .purge_min_size = big_min_size ? 1024*1024 : 1, + .purge_continued_percentage = 30, + }, + }; + char value[30]; + struct test_mail_cache_ctx ctx; + uint32_t seq; + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + + for (seq = 1; seq <= 100; seq++) { + i_snprintf(value, sizeof(value), "foo%d", seq); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, value); + } + + /* up to 29% no need to purge */ + for (seq = 1; seq <= 29; seq++) { + i_snprintf(value, sizeof(value), "bar%d", seq); + test_mail_cache_add_field(&ctx, seq, ctx.cache_field2.idx, value); + } + test_assert(ctx.cache->need_purge_file_seq == 0); + + /* at 30% need to purge */ + test_mail_cache_add_field(&ctx, 30, ctx.cache_field2.idx, "bar30"); + if (big_min_size) + test_assert(ctx.cache->need_purge_file_seq == 0); + else + test_assert(ctx.cache->need_purge_file_seq == ctx.cache->hdr->file_seq); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 0); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_update_need_purge_continued_records(void) +{ + test_begin("mail cache update need purge continued records"); + test_mail_cache_update_need_purge_continued_records_int(FALSE); + test_end(); +} + +static void test_mail_cache_update_need_purge_continued_records2(void) +{ + test_begin("mail cache update need purge continued records (2)"); + test_mail_cache_update_need_purge_continued_records_int(TRUE); + test_end(); +} + +static void +test_mail_cache_update_need_purge_deleted_records_int(bool big_min_size) +{ + struct mail_index_optimization_settings optimization_set = { + .cache = { + .purge_min_size = big_min_size ? 1024*1024 : 1, + .purge_delete_percentage = 30, + }, + }; + char value[30]; + struct mail_index_transaction *trans; + struct test_mail_cache_ctx ctx; + uint32_t seq; + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + + for (seq = 1; seq <= 100; seq++) { + i_snprintf(value, sizeof(value), "foo%d", seq); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, value); + } + + /* up to 29% no need to purge */ + trans = mail_index_transaction_begin(ctx.view, 0); + for (seq = 1; seq <= 29; seq++) { + i_snprintf(value, sizeof(value), "bar%d", seq); + mail_index_expunge(trans, seq); + } + test_assert(mail_index_transaction_commit(&trans) == 0); + test_mail_cache_index_sync(&ctx); + + test_assert(ctx.cache->need_purge_file_seq == 0); + test_assert(mail_cache_reopen(ctx.cache) == 1); + test_assert(ctx.cache->need_purge_file_seq == 0); + test_assert(test_mail_cache_get_purge_count(&ctx) == 0); + + /* at 30% need to purge */ + trans = mail_index_transaction_begin(ctx.view, 0); + mail_index_expunge(trans, 1); + test_assert(mail_index_transaction_commit(&trans) == 0); + /* syncing will internally purge if !big_min_size */ + test_mail_cache_index_sync(&ctx); + + test_assert(ctx.cache->need_purge_file_seq == 0); + test_assert(mail_cache_reopen(ctx.cache) == 1); + test_assert(ctx.cache->need_purge_file_seq == 0); + if (big_min_size) + test_assert(test_mail_cache_get_purge_count(&ctx) == 0); + else + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_update_need_purge_deleted_records(void) +{ + test_begin("mail cache update need purge deleted records"); + test_mail_cache_update_need_purge_deleted_records_int(FALSE); + test_end(); +} + +static void test_mail_cache_update_need_purge_deleted_records2(void) +{ + test_begin("mail cache update need purge deleted records (2)"); + test_mail_cache_update_need_purge_deleted_records_int(TRUE); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_cache_read_during_purge, + test_mail_cache_write_during_purge, + test_mail_cache_purge_while_cache_locked, + test_mail_cache_write_lost_during_purge, + test_mail_cache_write_lost_during_purge2, + test_mail_cache_write_autocommit, + test_mail_cache_write_autocommit2, + test_mail_cache_delete_too_large, + test_mail_cache_delete_too_large2, + test_mail_cache_purge_too_large, + test_mail_cache_purge_too_large2, + test_mail_cache_unexpectedly_lost, + test_mail_cache_unexpectedly_lost2, + test_mail_cache_resetid_mismatch, + test_mail_cache_resetid_mismatch2, + test_mail_cache_purge_field_changes, + test_mail_cache_purge_field_changes2, + test_mail_cache_purge_field_changes3, + test_mail_cache_purge_field_changes4, + test_mail_cache_purge_already_done, + test_mail_cache_purge_bitmask, + test_mail_cache_update_need_purge_continued_records, + test_mail_cache_update_need_purge_continued_records2, + test_mail_cache_update_need_purge_deleted_records, + test_mail_cache_update_need_purge_deleted_records2, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-cache.c b/src/lib-index/test-mail-cache.c new file mode 100644 index 0000000..14b3fb6 --- /dev/null +++ b/src/lib-index/test-mail-cache.c @@ -0,0 +1,764 @@ +/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "write-full.h" +#include "test-common.h" +#include "test-mail-cache.h" + +struct test_header_data { + uint32_t line1, line2; + uint32_t end_of_lines; + char headers[8]; +}; + +enum { + TEST_FIELD_NO, + TEST_FIELD_NO_FORCED, + TEST_FIELD_TEMP, + TEST_FIELD_TEMP_FORCED, + TEST_FIELD_YES, + TEST_FIELD_YES_FORCED, + TEST_FIELD_COUNT, +}; +static const struct mail_cache_field decision_cache_fields[TEST_FIELD_COUNT] = { + { + .name = "no", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_NO, + }, + { + .name = "no-forced", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED, + }, + { + .name = "temp", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_TEMP, + }, + { + .name = "temp-forced", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_TEMP | MAIL_CACHE_DECISION_FORCED, + }, + { + .name = "yes", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES, + }, + { + .name = "yes-forced", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES | MAIL_CACHE_DECISION_FORCED, + }, +}; + +static void test_mail_cache_fields(void) +{ + enum { + TEST_FIELD_FIXED, + TEST_FIELD_VARIABLE, + TEST_FIELD_STRING, + TEST_FIELD_BITMASK, + TEST_FIELD_HEADER1, + TEST_FIELD_HEADER2, + }; + struct mail_cache_field cache_fields[] = { + { + .name = "fixed", + .type = MAIL_CACHE_FIELD_FIXED_SIZE, + .field_size = 4, + .decision = MAIL_CACHE_DECISION_YES, + }, + { + .name = "variable", + .type = MAIL_CACHE_FIELD_VARIABLE_SIZE, + .decision = MAIL_CACHE_DECISION_YES, + }, + { + .name = "string", + .type = MAIL_CACHE_FIELD_STRING, + .decision = MAIL_CACHE_DECISION_YES, + }, + { + .name = "bitmask", + .type = MAIL_CACHE_FIELD_BITMASK, + .field_size = 4, + .decision = MAIL_CACHE_DECISION_YES, + }, + { + .name = "header1", + .type = MAIL_CACHE_FIELD_HEADER, + .decision = MAIL_CACHE_DECISION_YES, + }, + { + .name = "header2", + .type = MAIL_CACHE_FIELD_HEADER, + .decision = MAIL_CACHE_DECISION_YES, + }, + }; + struct test_mail_cache_ctx ctx; + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + string_t *str = t_str_new(16); + + test_begin("mail cache uncommitted lookups"); + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_cache_register_fields(ctx.cache, cache_fields, + N_ELEMENTS(cache_fields)); + + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + + /* add the cache fields */ + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + trans = mail_index_transaction_begin(ctx.view, 0); + cache_trans = mail_cache_get_transaction(cache_view, trans); + + const uint8_t fixed_data[] = { 0x12, 0x34, 0x56, 0x78 }; + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_FIXED].idx, + fixed_data, sizeof(fixed_data)); + const uint8_t variable_data[] = { 0xab, 0xcd, 0xef }; + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_VARIABLE].idx, + variable_data, sizeof(variable_data)); + const char string_data[] = { 's', 't', 'r' }; + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_STRING].idx, + string_data, sizeof(string_data)); + uint8_t bitmask_data[] = { 0x00, 0x01, 0x10, 0x11 }; + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_BITMASK].idx, + bitmask_data, sizeof(bitmask_data)); + struct test_header_data header_data1 = { + .line1 = 15, + .line2 = 30, + .headers = "foo\nbar\n", + }; + struct test_header_data header_data2 = { + .line1 = 10, + .line2 = 20, + .headers = "123\n456\n", + }; + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_HEADER1].idx, + &header_data1, sizeof(header_data1)); + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_HEADER2].idx, + &header_data2, sizeof(header_data2)); + + /* make sure the fields can be looked up even though they're + not committed */ + for (int i = 0;; i++) { + str_truncate(str, 0); + test_assert_idx(mail_cache_lookup_field(cache_view, str, 1, + cache_fields[TEST_FIELD_FIXED].idx) == 1, i); + test_assert_idx(str_len(str) == sizeof(fixed_data) && + memcmp(str_data(str), fixed_data, str_len(str)) == 0, i); + str_truncate(str, 0); + test_assert_idx(mail_cache_lookup_field(cache_view, str, 1, + cache_fields[TEST_FIELD_VARIABLE].idx) == 1, i); + test_assert_idx(str_len(str) == sizeof(variable_data) && + memcmp(str_data(str), variable_data, str_len(str)) == 0, i); + str_truncate(str, 0); + test_assert_idx(mail_cache_lookup_field(cache_view, str, 1, + cache_fields[TEST_FIELD_STRING].idx) == 1, i); + test_assert_idx(str_len(str) == sizeof(string_data) && + memcmp(str_data(str), string_data, str_len(str)) == 0, i); + str_truncate(str, 0); + test_assert_idx(mail_cache_lookup_field(cache_view, str, 1, + cache_fields[TEST_FIELD_BITMASK].idx) == 1, i); + test_assert_idx(str_len(str) == sizeof(bitmask_data) && + memcmp(str_data(str), bitmask_data, str_len(str)) == 0, i); + const unsigned int lookup_header_fields[] = { + cache_fields[TEST_FIELD_HEADER2].idx, + cache_fields[TEST_FIELD_HEADER1].idx, + }; + str_truncate(str, 0); + test_assert_idx(mail_cache_lookup_headers(cache_view, str, 1, + lookup_header_fields, + N_ELEMENTS(lookup_header_fields)) == 1, i); + test_assert_strcmp(str_c(str), "123\nfoo\n456\nbar\n"); + + if (trans == NULL) + break; + + /* add more bitmask data within the same transaction */ + uint8_t bitmask_add[4] = { 0x20, 0x20, 0x20, 0x20 }; + for (unsigned int j = 0; j < sizeof(bitmask_data); j++) + bitmask_data[j] |= bitmask_add[j]; + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_BITMASK].idx, + bitmask_add, sizeof(bitmask_add)); + /* check that we can still read it */ + str_truncate(str, 0); + test_assert_idx(mail_cache_lookup_field(cache_view, str, 1, + cache_fields[TEST_FIELD_BITMASK].idx) == 1, i); + test_assert_idx(str_len(str) == sizeof(bitmask_data) && + memcmp(str_data(str), bitmask_data, str_len(str)) == 0, i); + + /* commit the transaction and lookup the fields again */ + test_assert(mail_index_transaction_commit(&trans) == 0); + } + + /* add more bitmask data in separate transactions */ + for (unsigned int i = 0; i < 4; i++) { + uint8_t bitmask_add[4] = { 0, 0, 0, 0 }; + bitmask_add[i] = 0x40; + bitmask_data[i] |= 0x40; + + trans = mail_index_transaction_begin(ctx.view, 0); + cache_trans = mail_cache_get_transaction(cache_view, trans); + mail_cache_add(cache_trans, 1, cache_fields[TEST_FIELD_BITMASK].idx, + bitmask_add, sizeof(bitmask_add)); + test_assert(mail_index_transaction_commit(&trans) == 0); + } + + /* verify that bitmask is still as expected */ + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + cache_fields[TEST_FIELD_BITMASK].idx) == 1); + test_assert(str_len(str) == sizeof(bitmask_data) && + memcmp(str_data(str), bitmask_data, str_len(str)) == 0); + + /* verify that bitmask is still as expected after purging */ + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + test_mail_cache_view_sync(&ctx); + + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + cache_fields[TEST_FIELD_BITMASK].idx) == 1); + test_assert(str_len(str) == sizeof(bitmask_data) && + memcmp(str_data(str), bitmask_data, str_len(str)) == 0); + + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + mail_cache_view_close(&cache_view); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static void test_mail_cache_record_max_size_int(unsigned int field3_size) +{ + const struct mail_index_optimization_settings optimization_set = { + .cache = { + /* lets assume we can write 2 cache fields, + each containing 8 bytes */ + .record_max_size = sizeof(struct mail_cache_record) + + 2 * (sizeof(uint32_t) + /* field_idx */ + sizeof(uint32_t) + /* data_size */ + 8), /* content max length */ + }, + }; + struct test_mail_cache_ctx ctx; + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + string_t *str = t_str_new(16); + + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + + /* Add the first cache field. In a chain of cache records each one + has independent max size. Although this isn't really ideal, because + purging merges them and drops the records entirely if the combined + length is too large. But for now test least test what is + implemented. */ + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "12345678"); + + /* add the other field(s) */ + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + trans = mail_index_transaction_begin(ctx.view, 0); + cache_trans = mail_cache_get_transaction(cache_view, trans); + mail_cache_add(cache_trans, 1, ctx.cache_field2.idx, "abcdefgh", 8); + if (field3_size > 0) + mail_cache_add(cache_trans, 1, ctx.cache_field3.idx, "ijklmnopq", field3_size); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_cache_view_close(&cache_view); + + /* make sure all the fields are visible */ + test_mail_cache_view_sync(&ctx); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 1); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field2.idx) == 1); + if (field3_size == 8) { + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field3.idx) == 1); + test_assert_strcmp(str_c(str), "12345678abcdefghijklmnop"); + } else { + test_assert_strcmp(str_c(str), "12345678abcdefgh"); + } + mail_cache_view_close(&cache_view); + test_assert(test_mail_cache_get_purge_count(&ctx) == 0); + + /* if there are 3 fields, purging realizes that the record is too + large and drops it */ + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + test_assert(mail_cache_reopen(ctx.cache) == 1); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + if (field3_size == 8) { + /* test that none of the fields are in cache */ + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field2.idx) == 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field3.idx) == 0); + } else { + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 1); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field2.idx) == 1); + test_assert_strcmp(str_c(str), "12345678abcdefgh"); + } + mail_cache_view_close(&cache_view); + test_assert(test_mail_cache_get_purge_count(&ctx) == 1); + + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_record_max_size(void) +{ + test_begin("mail cache record max size"); + test_mail_cache_record_max_size_int(0); + test_end(); +} + +static void test_mail_cache_record_max_size2(void) +{ + test_begin("mail cache record max size (2)"); + test_mail_cache_record_max_size_int(8); + test_end(); +} + +static void test_mail_cache_record_max_size3(void) +{ + test_begin("mail cache record max size (3)"); + test_mail_cache_record_max_size_int(9); + test_end(); +} + +static void test_mail_cache_record_max_size4(void) +{ + const struct mail_index_optimization_settings optimization_set = { + .cache = { + .record_max_size = sizeof(struct mail_cache_record) + + sizeof(uint32_t) + /* field_idx */ + sizeof(uint32_t) + /* data_size */ + 8, /* content max length */ + }, + }; + struct test_mail_cache_ctx ctx; + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + string_t *str = t_str_new(16); + + test_begin("mail cache record max size (4)"); + test_mail_cache_init(test_mail_index_init(), &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + trans = mail_index_transaction_begin(ctx.view, 0); + cache_trans = mail_cache_get_transaction(cache_view, trans); + mail_cache_add(cache_trans, 1, ctx.cache_field.idx, "123456789", 9); + mail_cache_add(cache_trans, 2, ctx.cache_field.idx, "123456789", 9); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_cache_view_close(&cache_view); + + /* make sure none of the fields are visible */ + test_mail_cache_view_sync(&ctx); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 0); + test_assert(mail_cache_lookup_field(cache_view, str, 2, + ctx.cache_field.idx) == 0); + mail_cache_view_close(&cache_view); + test_assert(ctx.cache->hdr == NULL); /* never created */ + + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static void test_mail_cache_add_decisions(void) +{ + struct mail_cache_field cache_fields[TEST_FIELD_COUNT]; + enum mail_cache_decision_type expected_decisions[TEST_FIELD_COUNT]; + struct test_mail_cache_ctx ctx; + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + unsigned int i; + + test_begin("mail cache add decisions"); + + test_mail_cache_init(test_mail_index_init(), &ctx); + memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields)); + mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT); + for (i = 0; i < TEST_FIELD_COUNT; i++) + expected_decisions[i] = cache_fields[i].decision; + + /* create the initial cache file */ + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + + /* check that decisions haven't changed */ + for (i = 0; i < TEST_FIELD_COUNT; i++) + test_assert_idx(ctx.cache->fields[cache_fields[i].idx].field.decision == expected_decisions[i], i); + + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + trans = mail_index_transaction_begin(ctx.view, 0); + cache_trans = mail_cache_get_transaction(cache_view, trans); + + /* test that when cache decisions are disabled, it doesn't affect the + NO state change */ + mail_cache_view_update_cache_decisions(cache_view, FALSE); + mail_cache_add(cache_trans, 2, cache_fields[TEST_FIELD_NO].idx, "bar", 3); + mail_cache_view_update_cache_decisions(cache_view, TRUE); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO].idx].field.decision == MAIL_CACHE_DECISION_NO); + + /* add a cache field of each type */ + for (i = 0; i < TEST_FIELD_COUNT; i++) + mail_cache_add(cache_trans, 1, cache_fields[i].idx, "foo", 3); + /* quick check before commit that the state is as expected */ + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO].idx].field.decision == MAIL_CACHE_DECISION_TEMP); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO].idx].decision_dirty); + test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO].idx].uid_highwater == 1); + + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_cache_view_close(&cache_view); + + /* verify the state: NO state becomes TEMP, others are unchanged */ + expected_decisions[TEST_FIELD_NO] = MAIL_CACHE_DECISION_TEMP; + test_assert(!ctx.cache->field_header_write_pending); + for (i = 0; i < TEST_FIELD_COUNT; i++) { + const struct mail_cache_field_private *priv = + &ctx.cache->fields[cache_fields[i].idx]; + test_assert_idx(priv->field.decision == expected_decisions[i], i); + test_assert_idx(!priv->decision_dirty, i); + uint32_t uid_highwater = priv->uid_highwater; + if (i != TEST_FIELD_NO_FORCED) + test_assert_idx(uid_highwater == 1, i); + else + test_assert_idx(uid_highwater == 0, i); + } + + test_assert(test_mail_cache_get_purge_count(&ctx) == 0); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static void test_mail_cache_lookup_decisions_int(bool header_lookups) +{ + struct mail_cache_field cache_fields[TEST_FIELD_COUNT]; + enum mail_cache_decision_type expected_decisions[TEST_FIELD_COUNT]; + uint32_t expected_uid_highwater[TEST_FIELD_COUNT]; + time_t expected_last_used[TEST_FIELD_COUNT]; + struct test_mail_cache_ctx ctx; + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + unsigned int i; + string_t *str = t_str_new(16); + + test_mail_cache_init(test_mail_index_init(), &ctx); + /* create the initial mails and the cache file */ + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); + + /* register fields after the initial purge create the cache */ + memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields)); + mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT); + for (i = 0; i < TEST_FIELD_COUNT; i++) { + expected_decisions[i] = cache_fields[i].decision; + expected_uid_highwater[i] = 0; + } + + /* day_first_uid[7] is used to determine which mails are "old" and + which mails are "new". [7] is the first "new" mail. */ + test_mail_cache_update_day_first_uid7(&ctx, 2); + + trans = mail_index_transaction_begin(ctx.view, 0); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + + /* test that nothing changes when cache decision updates are disabled */ + mail_cache_view_update_cache_decisions(cache_view, FALSE); + for (i = 0; i < TEST_FIELD_COUNT; i++) T_BEGIN { + const struct mail_cache_field_private *priv = + &ctx.cache->fields[cache_fields[i].idx]; + if (!header_lookups) { + test_assert_idx(mail_cache_lookup_field(cache_view, + str, 1, cache_fields[i].idx) == 0, i); + } else { + /* it's a bit wrong to lookup headers using a STRING + type cache field, but this is simpler and at least + currently there's no assert for it.. */ + test_assert_idx(mail_cache_lookup_headers(cache_view, + str, 2, &cache_fields[i].idx, 1) == 0, i); + } + test_assert_idx(priv->field.decision == expected_decisions[i], i); + test_assert_idx(!priv->decision_dirty, i); + test_assert_idx(priv->uid_highwater == 0, i); + test_assert_idx(priv->field.last_used == 0, i); + } T_END; + test_assert(!ctx.cache->field_header_write_pending); + mail_cache_view_update_cache_decisions(cache_view, TRUE); + + /* set cache fields for the first "new" mail (seq/UID 2) */ + ioloop_time = 123456789; + for (i = 0; i < TEST_FIELD_COUNT; i++) T_BEGIN { + const struct mail_cache_field_private *priv = + &ctx.cache->fields[cache_fields[i].idx]; + + time_t prev_last_used = priv->field.last_used; + ioloop_time++; + if (!header_lookups) { + test_assert_idx(mail_cache_lookup_field(cache_view, + str, 2, cache_fields[i].idx) == 0, i); + } else { + test_assert_idx(mail_cache_lookup_headers(cache_view, + str, 2, &cache_fields[i].idx, 1) == 0, i); + } + expected_last_used[i] = ioloop_time; + switch (i) { + case TEST_FIELD_NO_FORCED: + expected_last_used[i] = 0; + /* fall through */ + case TEST_FIELD_NO: + /* Note that just doing a cache lookup won't change + caching decision. Higher level code needs to figure + out itself if it wants the field to become cached. + This happens only by calling mail_cache_add(). */ + break; + case TEST_FIELD_TEMP: + /* Note that uid_highwater isn't permanently saved to + the cache file. It's used only within a single + session. */ + expected_uid_highwater[i] = 2; + break; + case TEST_FIELD_YES: + /* YES decision doesn't change last_used until the + cache decision has been confirmed again. */ + expected_last_used[i] = prev_last_used; + expected_uid_highwater[i] = 2; + break; + } + test_assert_idx(priv->field.decision == expected_decisions[i], i); + test_assert_idx(priv->uid_highwater == expected_uid_highwater[i], i); + test_assert_idx(priv->field.last_used == expected_last_used[i], i); + test_assert_idx(!priv->decision_dirty, i); + } T_END; + test_assert(!ctx.cache->field_header_write_pending); + + test_assert(mail_index_transaction_commit(&trans) == 0); + + /* test that after commit and reopening the decisions are still the + same. */ + test_assert(mail_cache_reopen(ctx.cache) == 1); + for (i = 0; i < TEST_FIELD_COUNT; i++) { + const struct mail_cache_field_private *priv = + &ctx.cache->fields[cache_fields[i].idx]; + test_assert_idx(priv->field.decision == expected_decisions[i], i); + test_assert_idx(priv->uid_highwater == expected_uid_highwater[i], i); + test_assert_idx(priv->field.last_used == expected_last_used[i], i); + test_assert_idx(!priv->decision_dirty, i); + } + + /* update the day_first_uid so all mails are now "old" */ + test_mail_cache_update_day_first_uid7(&ctx, 4); + + for (uint32_t seq = 2; seq >= 1; seq--) { + /* Reading a 3rd mail, which is also now "old". It causes + TEMP -> YES cache decision (don't read backwards yet, + that's a separate test). */ + expected_decisions[TEST_FIELD_TEMP] = MAIL_CACHE_DECISION_YES; + for (i = 0; i < TEST_FIELD_COUNT; i++) T_BEGIN { + const struct mail_cache_field_private *priv = + &ctx.cache->fields[cache_fields[i].idx]; + + /* Keep increasing ioloop_time just to make sure that + last_used doesn't change. (It changes only once per + 24h) */ + ioloop_time++; + if (!header_lookups) { + test_assert_idx(mail_cache_lookup_field( + cache_view, str, seq, + cache_fields[i].idx) == 0, i); + } else { + test_assert_idx(mail_cache_lookup_headers( + cache_view, str, seq, + &cache_fields[i].idx, 1) == 0, i); + } + if (i == TEST_FIELD_YES && seq == 2) { + /* YES decision is confirmed now. The last_used + timestamp was updated for the first old + mail. */ + expected_last_used[i] = ioloop_time; + } + test_assert_idx(priv->field.decision == expected_decisions[i], i); + test_assert_idx(priv->uid_highwater == expected_uid_highwater[i], i); + test_assert_idx(priv->field.last_used == expected_last_used[i], i); + test_assert_idx(priv->decision_dirty == (i == TEST_FIELD_TEMP), i); + } T_END; + /* restore caching decision */ + ctx.cache->fields[cache_fields[TEST_FIELD_TEMP].idx].field.decision = + MAIL_CACHE_DECISION_TEMP; + /* reading mails backwards also causes TEMP -> YES cache + decision, even if all mails are "new" */ + test_mail_cache_update_day_first_uid7(&ctx, 1); + } + + test_assert(test_mail_cache_get_purge_count(&ctx) == 0); + mail_cache_view_close(&cache_view); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); +} + +static void test_mail_cache_lookup_decisions(void) +{ + test_begin("mail cache lookup decisions"); + test_mail_cache_lookup_decisions_int(FALSE); + test_end(); +} + +static void test_mail_cache_lookup_decisions2(void) +{ + test_begin("mail cache lookup decisions (2)"); + test_mail_cache_lookup_decisions_int(TRUE); + test_end(); +} + +static void test_mail_cache_in_memory(void) +{ + const struct mail_index_optimization_settings optimization_set = { + .cache = { + .record_max_size = MAIL_CACHE_MAX_WRITE_BUFFER*2, + }, + }; + struct test_mail_cache_ctx ctx; + struct mail_index *index; + struct mail_index_transaction *trans; + struct mail_cache_view *cache_view; + struct mail_cache_transaction_ctx *cache_trans; + + test_begin("mail cache add in-memory"); + + index = mail_index_alloc(NULL, NULL, "(in-memory)"); + test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0); + test_mail_cache_init(index, &ctx); + mail_index_set_optimization_settings(ctx.index, &optimization_set); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); + + trans = mail_index_transaction_begin(ctx.view, 0); + cache_trans = mail_cache_get_transaction(cache_view, trans); + + size_t blob_size = 1024*130; + char *blob = i_malloc(blob_size); + memset(blob, 'x', blob_size); + mail_cache_add(cache_trans, 1, ctx.cache_field.idx, blob, blob_size); + mail_cache_add(cache_trans, 1, ctx.cache_field2.idx, "foo1", 4); + mail_cache_add(cache_trans, 2, ctx.cache_field2.idx, "foo2", 4); + + /* all fields are still available */ + string_t *str = str_new(default_pool, blob_size + 1024); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 1); + test_assert(str_len(str) == blob_size); + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field2.idx) == 1); + test_assert_strcmp(str_c(str), "foo1"); + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 2, + ctx.cache_field2.idx) == 1); + test_assert_strcmp(str_c(str), "foo2"); + + /* adding a second blob grows memory usage beyond + MAIL_CACHE_MAX_WRITE_BUFFER and frees the first cached mail + entirely (although in theory it could drop just the big blob) */ + mail_cache_add(cache_trans, 2, ctx.cache_field.idx, blob, blob_size); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field.idx) == 0); + test_assert(mail_cache_lookup_field(cache_view, str, 1, + ctx.cache_field2.idx) == 0); + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 2, + ctx.cache_field.idx) == 1); + test_assert(str_len(str) == blob_size); + str_truncate(str, 0); + test_assert(mail_cache_lookup_field(cache_view, str, 2, + ctx.cache_field2.idx) == 1); + test_assert_strcmp(str_c(str), "foo2"); + + test_assert(mail_index_transaction_commit(&trans) == 0); + + str_free(&str); + i_free(blob); + + mail_cache_view_close(&cache_view); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +static void test_mail_cache_size_corruption(void) +{ + struct test_mail_cache_ctx ctx; + struct mail_cache_view *cache_view; + struct mail_cache_lookup_iterate_ctx iter; + struct mail_cache_iterate_field field; + + test_begin("mail cache size corruption"); + + test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "12345678"); + cache_view = mail_cache_view_open(ctx.cache, ctx.view); + + /* lookup the added cache field */ + mail_cache_lookup_iter_init(cache_view, 1, &iter); + test_assert(iter.offset > 0); + + uoff_t size_offset = iter.offset + + offsetof(struct mail_cache_record, size); + uint32_t new_size = 0x10000000; + test_assert(pwrite_full(ctx.cache->fd, &new_size, sizeof(new_size), + size_offset) == 0); + test_expect_error_string("record points outside file"); + test_assert(mail_cache_lookup_iter_next(&iter, &field) == -1); + test_expect_no_more_errors(); + + mail_cache_view_close(&cache_view); + test_mail_cache_deinit(&ctx); + test_mail_index_delete(); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_cache_fields, + test_mail_cache_record_max_size, + test_mail_cache_record_max_size2, + test_mail_cache_record_max_size3, + test_mail_cache_record_max_size4, + test_mail_cache_add_decisions, + test_mail_cache_lookup_decisions, + test_mail_cache_lookup_decisions2, + test_mail_cache_in_memory, + test_mail_cache_size_corruption, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-cache.h b/src/lib-index/test-mail-cache.h new file mode 100644 index 0000000..2a274b5 --- /dev/null +++ b/src/lib-index/test-mail-cache.h @@ -0,0 +1,32 @@ +#ifndef TEST_MAIL_CACHE_H +#define TEST_MAIL_CACHE_H + +#include "test-mail-index.h" +#include "mail-cache-private.h" + +struct test_mail_cache_ctx { + struct mail_index *index; + struct mail_cache *cache; + struct mail_index_view *view; + + struct mail_cache_field cache_field, cache_field2, cache_field3; +}; + +void test_mail_cache_init(struct mail_index *index, + struct test_mail_cache_ctx *ctx_r); +void test_mail_cache_deinit(struct test_mail_cache_ctx *ctx); + +unsigned int test_mail_cache_get_purge_count(struct test_mail_cache_ctx *ctx); +void test_mail_cache_index_sync(struct test_mail_cache_ctx *ctx); +void test_mail_cache_view_sync(struct test_mail_cache_ctx *ctx); +void test_mail_cache_purge(void); +void test_mail_cache_add_mail(struct test_mail_cache_ctx *ctx, + unsigned int cache_field_idx, + const char *cache_data); +void test_mail_cache_add_field(struct test_mail_cache_ctx *ctx, uint32_t seq, + unsigned int cache_field_idx, + const char *cache_data); +void test_mail_cache_update_day_first_uid7(struct test_mail_cache_ctx *ctx, + uint32_t first_new_uid); + +#endif diff --git a/src/lib-index/test-mail-index-map.c b/src/lib-index/test-mail-index-map.c new file mode 100644 index 0000000..0b0b3ad --- /dev/null +++ b/src/lib-index/test-mail-index-map.c @@ -0,0 +1,57 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "test-common.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-index-transaction-private.h" + +static void test_mail_index_map_lookup_seq_range_count(unsigned int messages_count) +{ + struct mail_index_record_map rec_map; + struct mail_index_map map; + uint32_t seq, first_uid, last_uid, first_seq, last_seq, max_uid; + + i_zero(&map); + i_zero(&rec_map); + map.rec_map = &rec_map; + map.hdr.messages_count = messages_count; + map.hdr.record_size = sizeof(struct mail_index_record); + rec_map.records_count = map.hdr.messages_count; + rec_map.records = i_new(struct mail_index_record, map.hdr.messages_count); + + for (seq = 1; seq <= map.hdr.messages_count; seq++) + MAIL_INDEX_REC_AT_SEQ(&map, seq)->uid = seq*2; + max_uid = (seq-1)*2; + map.hdr.next_uid = max_uid + 1; + + for (first_uid = 2; first_uid <= max_uid; first_uid++) { + for (last_uid = first_uid; last_uid <= max_uid; last_uid++) { + if (first_uid == last_uid && first_uid%2 != 0) + continue; + mail_index_map_lookup_seq_range(&map, first_uid, last_uid, &first_seq, &last_seq); + test_assert((first_uid+1)/2 == first_seq && last_uid/2 == last_seq); + } + } + i_free(rec_map.records); +} + +static void test_mail_index_map_lookup_seq_range(void) +{ + unsigned int i; + + test_begin("mail index map lookup seq range"); + for (i = 1; i < 20; i++) + test_mail_index_map_lookup_seq_range_count(i); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_index_map_lookup_seq_range, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-index-modseq.c b/src/lib-index/test-mail-index-modseq.c new file mode 100644 index 0000000..4a2d524 --- /dev/null +++ b/src/lib-index/test-mail-index-modseq.c @@ -0,0 +1,77 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "test-common.h" +#include "test-mail-index.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log-private.h" + +static void test_mail_index_modseq_get_next_log_offset(void) +{ + static const struct { + uint32_t log_seq; + uoff_t log_offset; + } tests[] = { + { 0, 0 }, + { 2, 40 }, + { 2, 148 }, + { 2, 164 }, + { 3, 40 }, + { 3, 56 }, + { 3, 72 }, + { 3, 88 }, + }; + struct mail_index *index; + struct mail_index_view *view, *view2; + struct mail_index_transaction *trans; + uint32_t seq, uid; + + test_begin("mail_transaction_log_file_get_modseq_next_offset()"); + index = test_mail_index_init(); + view = mail_index_view_open(index); + mail_index_modseq_enable(index); + + trans = mail_index_transaction_begin(view, 0); + uid = 1234; + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid, sizeof(uid), TRUE); + test_assert(mail_index_transaction_commit(&trans) == 0); + + for (uid = 1; uid <= 3; uid++) { + trans = mail_index_transaction_begin(view, 0); + mail_index_append(trans, uid, &seq); + test_assert(mail_index_transaction_commit(&trans) == 0); + } + test_assert(mail_transaction_log_file_lock(index->log->head) == 0); + test_assert(mail_transaction_log_rotate(index->log, FALSE) == 0); + mail_transaction_log_file_unlock(index->log->head, "rotating"); + for (uid = 4; uid <= 6; uid++) { + trans = mail_index_transaction_begin(view, 0); + mail_index_append(trans, uid, &seq); + test_assert(mail_index_transaction_commit(&trans) == 0); + } + + view2 = mail_index_view_open(index); + for (uint64_t modseq = 1; modseq <= 7; modseq++) { + uint32_t log_seq = 0; + uoff_t log_offset; + + test_assert_idx(mail_index_modseq_get_next_log_offset(view2, modseq, &log_seq, &log_offset) == (tests[modseq].log_seq != 0), modseq); + test_assert_idx(tests[modseq].log_seq == log_seq && tests[modseq].log_offset == log_offset, modseq); + } + + mail_index_view_close(&view); + mail_index_view_close(&view2); + test_mail_index_deinit(&index); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_index_modseq_get_next_log_offset, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-index-sync-ext.c b/src/lib-index/test-mail-index-sync-ext.c new file mode 100644 index 0000000..156d50f --- /dev/null +++ b/src/lib-index/test-mail-index-sync-ext.c @@ -0,0 +1,86 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "test-common.h" +#include "mail-transaction-log-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-index-modseq.h" + +static void test_lookup_seq_range(struct mail_index_view *view ATTR_UNUSED, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r) +{ + *first_seq_r = first_uid; + *last_seq_r = last_uid; +} + +static void test_mail_index_sync_ext_atomic_inc(void) +{ + struct mail_index_sync_map_ctx ctx; + struct mail_transaction_ext_atomic_inc u; + struct mail_index_ext *ext; + void *ptr; + + test_begin("mail index sync ext atomic inc"); + + i_zero(&ctx); + ctx.view = t_new(struct mail_index_view, 1); + ctx.view->log_view = t_new(struct mail_transaction_log_view, 1); + ctx.view->index = t_new(struct mail_index, 1); + ctx.view->index->fsck_log_head_file_seq = 10; /* silence errors */ + ctx.view->v.lookup_seq_range = test_lookup_seq_range; + ctx.view->map = t_new(struct mail_index_map, 1); + ctx.view->map->hdr.next_uid = 2; + ctx.view->map->hdr.record_size = sizeof(struct mail_index_record) + 16; + ctx.view->map->rec_map = t_new(struct mail_index_record_map, 1); + ctx.view->map->rec_map->records = + t_malloc0(ctx.view->map->hdr.record_size); + t_array_init(&ctx.view->map->extensions, 4); + ext = array_append_space(&ctx.view->map->extensions); + ext->record_offset = sizeof(struct mail_index_record); + ptr = PTR_OFFSET(ctx.view->map->rec_map->records, ext->record_offset); + + i_zero(&u); + test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == -1); + + u.uid = 2; + test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == -1); + + u.uid = 1; +#define TEST_ATOMIC(_type, _value, _diff, _ret) \ + { _type *n = ptr; *n = _value; } \ + ctx.cur_ext_record_size = sizeof(_type); \ + u.diff = _diff; \ + test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == _ret); + +#define TEST_ATOMIC_BLOCK(_type, _max) \ + TEST_ATOMIC(_type, 1, -1, 1); \ + TEST_ATOMIC(_type, 1, -2, -1); \ + TEST_ATOMIC(_type, 0, -1, -1); \ + TEST_ATOMIC(_type, 0, _max, 1); \ + TEST_ATOMIC(_type, 1, _max, -1); \ + TEST_ATOMIC(_type, 0, (_max+1), -1); \ + TEST_ATOMIC(_type, _max, 1, -1); \ + TEST_ATOMIC(_type, _max, -_max, 1); \ + TEST_ATOMIC(_type, _max, -(_max+1), -1); + + TEST_ATOMIC_BLOCK(uint8_t, 255); + TEST_ATOMIC_BLOCK(uint16_t, 65535); + + ctx.cur_ext_record_size = 5; + u.diff = 0; + test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == -1); + + i_free(ctx.view->index->need_recreate); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_index_sync_ext_atomic_inc, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-index-transaction-finish.c b/src/lib-index/test-mail-index-transaction-finish.c new file mode 100644 index 0000000..e32467e --- /dev/null +++ b/src/lib-index/test-mail-index-transaction-finish.c @@ -0,0 +1,297 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "test-common.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-index-transaction-private.h" + + +static struct mail_index_record recs[20]; +static uint64_t modseqs[N_ELEMENTS(recs)]; + +bool mail_index_map_get_ext_idx(struct mail_index_map *map ATTR_UNUSED, + uint32_t ext_id ATTR_UNUSED, + uint32_t *idx_r ATTR_UNUSED) { return FALSE; } +void mail_index_ext_set_reset_id(struct mail_index_transaction *t ATTR_UNUSED, + uint32_t ext_id ATTR_UNUSED, + uint32_t reset_id ATTR_UNUSED) { } +void mail_index_transaction_set_log_updates(struct mail_index_transaction *t ATTR_UNUSED) { } +void mail_index_update_day_headers(struct mail_index_transaction *t ATTR_UNUSED, time_t day_stamp ATTR_UNUSED) {} +bool mail_index_cancel_flag_updates(struct mail_index_transaction *t ATTR_UNUSED, + uint32_t seq ATTR_UNUSED) { return TRUE; } +bool mail_index_cancel_keyword_updates(struct mail_index_transaction *t ATTR_UNUSED, + uint32_t seq ATTR_UNUSED) { return TRUE; } +void mail_index_transaction_sort_appends(struct mail_index_transaction *t ATTR_UNUSED) {} +int mail_index_map(struct mail_index *index ATTR_UNUSED, + enum mail_index_sync_handler_type type ATTR_UNUSED) { return 1; } +void mail_index_update_modseq(struct mail_index_transaction *t ATTR_UNUSED, uint32_t seq ATTR_UNUSED, + uint64_t min_modseq ATTR_UNUSED) {} + +const struct mail_index_record * +mail_index_lookup(struct mail_index_view *view ATTR_UNUSED, uint32_t seq) +{ + i_assert(seq < N_ELEMENTS(recs)); + return &recs[seq]; +} + +struct mail_index_record * +mail_index_transaction_lookup(struct mail_index_transaction *t ATTR_UNUSED, + uint32_t seq) +{ + i_assert(seq < N_ELEMENTS(recs)); + return &recs[seq]; +} + +uint64_t mail_index_modseq_lookup(struct mail_index_view *view ATTR_UNUSED, + uint32_t seq) +{ + i_assert(seq < N_ELEMENTS(modseqs)); + return modseqs[seq]; +} + +uint64_t mail_index_modseq_get_highest(struct mail_index_view *view ATTR_UNUSED) +{ + return modseqs[0]; +} + +#define MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far) \ + for (unsigned int sofar = 0; sofar < n_so_far; sofar++) \ + mail_index_transaction_finish_so_far(t); \ + mail_index_transaction_finish(t); + +static void +test_mail_index_transaction_finish_flag_updates(unsigned int n_so_far) +{ + struct mail_index_transaction *t; + const struct mail_index_flag_update *updates; + struct mail_index_flag_update u; + unsigned int count; + + t = t_new(struct mail_index_transaction, 1); + t->drop_unnecessary_flag_updates = TRUE; + + i_zero(&u); + u.add_flags = MAIL_SEEN; u.remove_flags = MAIL_DRAFT; + + test_begin(t_strdup_printf("mail index transaction finish flag updates n_so_far=%u", n_so_far)); + + /* test fast path: all changed */ + t_array_init(&t->updates, 10); + u.uid1 = 1; u.uid2 = 2; + array_push_back(&t->updates, &u); + u.uid1 = 4; u.uid2 = 5; + array_push_back(&t->updates, &u); + MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far); + + updates = array_get(&t->updates, &count); + test_assert(count == 4); + test_assert(updates[0].uid1 == 1*2 && updates[0].uid2 == 1*2); + test_assert(updates[1].uid1 == 2*2 && updates[1].uid2 == 2*2); + test_assert(updates[2].uid1 == 4*2 && updates[2].uid2 == 4*2); + test_assert(updates[3].uid1 == 5*2 && updates[3].uid2 == 5*2); + + /* nothing changed */ + t_array_init(&t->updates, 10); + u.uid1 = 1; u.uid2 = 2; + array_push_back(&t->updates, &u); + u.uid1 = 4; u.uid2 = 5; + array_push_back(&t->updates, &u); + recs[1].flags = MAIL_SEEN; + recs[2].flags = MAIL_SEEN; + recs[4].flags = MAIL_SEEN; + recs[5].flags = MAIL_SEEN; + MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far); + test_assert(!array_is_created(&t->updates)); + + /* some changes */ + t_array_init(&t->updates, 10); + u.uid1 = 2; u.uid2 = 3; + array_push_back(&t->updates, &u); + u.uid1 = 5; u.uid2 = 6; + array_push_back(&t->updates, &u); + MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far); + + updates = array_get(&t->updates, &count); + test_assert(count == 2); + test_assert(updates[0].uid1 == 3*2 && updates[0].uid2 == 3*2); + test_assert(updates[1].uid1 == 6*2 && updates[1].uid2 == 6*2); + + test_end(); +} + +static void +test_mail_index_transaction_finish_check_conflicts(unsigned int n_so_far) +{ + struct mail_index_transaction *t; + const struct seq_range *conflicts; + ARRAY_TYPE(seq_range) conflict_seqs = ARRAY_INIT; + unsigned int count; + + t = t_new(struct mail_index_transaction, 1); + t->view = t_new(struct mail_index_view, 1); + t->min_flagupdate_seq = 5; + t->max_flagupdate_seq = 8; + t->conflict_seqs = &conflict_seqs; + + modseqs[0] = 1234; + modseqs[5] = 5; + modseqs[6] = 8; + modseqs[7] = 6; + modseqs[8] = 7; + + test_begin(t_strdup_printf("mail index transaction finish check conflicts n_so_far=%u", n_so_far)); + + /* fast path: no conflicts */ + t->max_modseq = 1234; + MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far); + test_assert(!array_is_created(&conflict_seqs)); + + /* try some conflicts */ + t->max_modseq = 6; + MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far); + + i_assert(array_is_created(&conflict_seqs)); + + conflicts = array_get(&conflict_seqs, &count); + test_assert(count == 2); + test_assert(conflicts[0].seq1 == 6 && conflicts[0].seq2 == 6); + test_assert(conflicts[1].seq1 == 8 && conflicts[1].seq2 == 8); + + test_end(); + array_free(t->conflict_seqs); +} + +static void +test_mail_index_transaction_finish_modseq_updates(unsigned int n_so_far) +{ + struct mail_index_transaction *t; + const struct mail_transaction_modseq_update *ups; + struct mail_transaction_modseq_update u; + unsigned int count; + + t = t_new(struct mail_index_transaction, 1); + + test_begin(t_strdup_printf("mail index transaction finish modseq updates n_so_far=%u", n_so_far)); + + t_array_init(&t->modseq_updates, 10); + u.modseq_low32 = 1234567890; + u.modseq_high32 = 987654321; + u.uid = 1; array_push_back(&t->modseq_updates, &u); + u.modseq_low32++; + u.modseq_high32++; + u.uid = 2; array_push_back(&t->modseq_updates, &u); + u.modseq_low32++; + u.modseq_high32++; + u.uid = 5; array_push_back(&t->modseq_updates, &u); + u.modseq_low32 = 1234; + u.modseq_high32 = 0; + u.uid = 2; array_push_back(&t->modseq_updates, &u); + + MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far); + + ups = array_get(&t->modseq_updates, &count); + test_assert(count == 4); + + test_assert(ups[0].uid == 1*2); + test_assert(ups[0].modseq_low32 == 1234567890 && + ups[0].modseq_high32 == 987654321); + test_assert(ups[1].uid == 2*2); + test_assert(ups[1].modseq_low32 == 1234567891 && + ups[1].modseq_high32 == 987654322); + test_assert(ups[2].uid == 5*2); + test_assert(ups[2].modseq_low32 == 1234567892 && + ups[2].modseq_high32 == 987654323); + test_assert(ups[3].uid == 2*2); + test_assert(ups[3].modseq_low32 == 1234 && + ups[3].modseq_high32 == 0); + test_end(); +} + +static void +test_mail_index_transaction_finish_expunges(unsigned int n_so_far) +{ + struct mail_index_transaction *t; + guid_128_t guid1, guid2, guid3; + const struct mail_transaction_expunge_guid *expunges; + struct mail_transaction_expunge_guid expunge; + unsigned int i, count; + + for (i = 0; i < sizeof(guid2); i++) { + guid1[i] = i + 1; + guid2[i] = i ^ 0xff; + guid3[i] = i + 0x80; + } + + recs[1].uid = 12; + recs[2].uid = 15; + recs[3].uid = 18; + + t = t_new(struct mail_index_transaction, 1); + t->expunges_nonsorted = TRUE; + + test_begin(t_strdup_printf("mail index transaction finish expunges n_so_far=%u", n_so_far)); + + t_array_init(&t->expunges, 3); + expunge.uid = 2; + memcpy(expunge.guid_128, guid2, sizeof(expunge.guid_128)); + array_push_back(&t->expunges, &expunge); + array_push_back(&t->expunges, &expunge); + expunge.uid = 1; + memcpy(expunge.guid_128, guid1, sizeof(expunge.guid_128)); + array_push_back(&t->expunges, &expunge); + array_push_back(&t->expunges, &expunge); + expunge.uid = 3; + memcpy(expunge.guid_128, guid3, sizeof(expunge.guid_128)); + array_push_back(&t->expunges, &expunge); + array_push_back(&t->expunges, &expunge); + + MAIL_INDEX_TRANSACTION_FINISH(t, n_so_far); + + expunges = array_get(&t->expunges, &count); + test_assert(count == 3); + test_assert(expunges[0].uid == 12); + test_assert(memcmp(expunges[0].guid_128, guid1, sizeof(guid1)) == 0); + test_assert(expunges[1].uid == 15); + test_assert(memcmp(expunges[1].guid_128, guid2, sizeof(guid2)) == 0); + test_assert(expunges[2].uid == 18); + test_assert(memcmp(expunges[2].guid_128, guid3, sizeof(guid3)) == 0); + test_end(); +} + +static void test_state_reset(void) +{ + memset(recs, 0, sizeof(recs)); + memset(modseqs, 0, sizeof(modseqs)); + for (unsigned int n = 1; n < N_ELEMENTS(recs); n++) + recs[n].uid = n*2; +} + +static void test_mail_index_transaction_finish(void) +{ + void (*const test_finish_functions[])(unsigned int) = { + test_mail_index_transaction_finish_flag_updates, + test_mail_index_transaction_finish_check_conflicts, + test_mail_index_transaction_finish_modseq_updates, + test_mail_index_transaction_finish_expunges, + }; + unsigned int i, j; + + for (i = 0; i < N_ELEMENTS(test_finish_functions); i++) { + for (j = 0; j < 3; j++) { + test_state_reset(); + test_finish_functions[i](j); + } + } +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_index_transaction_finish, + NULL + }; + + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-index-transaction-update.c b/src/lib-index/test-mail-index-transaction-update.c new file mode 100644 index 0000000..cdfa951 --- /dev/null +++ b/src/lib-index/test-mail-index-transaction-update.c @@ -0,0 +1,683 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "env-util.h" +#include "test-common.h" +#include "mail-index-private.h" +#include "mail-index-transaction-private.h" + +#include <time.h> + +static struct mail_index_header hdr; +static struct mail_index_record rec; + +const struct mail_index_header * +mail_index_get_header(struct mail_index_view *view ATTR_UNUSED) +{ + return &hdr; +} + +const struct mail_index_record * +mail_index_lookup(struct mail_index_view *view ATTR_UNUSED, + uint32_t seq ATTR_UNUSED) +{ + return &rec; +} + +void mail_index_lookup_keywords(struct mail_index_view *view ATTR_UNUSED, + uint32_t seq ATTR_UNUSED, + ARRAY_TYPE(keyword_indexes) *keyword_idx ATTR_UNUSED) +{ + array_clear(keyword_idx); +} + +bool mail_index_map_get_ext_idx(struct mail_index_map *map ATTR_UNUSED, + uint32_t ext_id ATTR_UNUSED, + uint32_t *idx_r ATTR_UNUSED) +{ + return FALSE; +} + +uint32_t mail_index_view_get_messages_count(struct mail_index_view *view ATTR_UNUSED) +{ + return hdr.messages_count; +} + +void mail_index_transaction_lookup_latest_keywords(struct mail_index_transaction *t ATTR_UNUSED, + uint32_t seq ATTR_UNUSED, + ARRAY_TYPE(keyword_indexes) *keywords ATTR_UNUSED) +{ +} + +struct mail_keywords * +mail_index_keywords_create_from_indexes(struct mail_index *index ATTR_UNUSED, + const ARRAY_TYPE(keyword_indexes) + *keyword_indexes ATTR_UNUSED) +{ + return NULL; +} + +void mail_index_keywords_unref(struct mail_keywords **keywords ATTR_UNUSED) +{ +} + +static struct mail_index_transaction * +mail_index_transaction_new(void) +{ + struct mail_index_transaction *t; + + t = t_new(struct mail_index_transaction, 1); + t->first_new_seq = hdr.messages_count + 1; + return t; +} +static void mail_index_transaction_cleanup(struct mail_index_transaction *t) +{ + if (array_is_created(&t->appends)) + array_free(&t->appends); + if (array_is_created(&t->updates)) + array_free(&t->updates); + if (array_is_created(&t->modseq_updates)) + array_free(&t->modseq_updates); + if (array_is_created(&t->expunges)) + array_free(&t->expunges); +} + +static void test_mail_index_append(void) +{ + struct mail_index_transaction *t; + const struct mail_index_record *appends; + ARRAY_TYPE(seq_range) saved_uids_arr; + const struct seq_range *saved_uids; + unsigned int count; + uint32_t seq; + + hdr.messages_count = 4; + t = mail_index_transaction_new(); + + test_begin("mail index append"); + mail_index_append(t, 0, &seq); + test_assert(t->log_updates); + test_assert(seq == 5); + mail_index_append(t, 0, &seq); + test_assert(seq == 6); + test_assert(!t->appends_nonsorted); + + t_array_init(&saved_uids_arr, 128); + mail_index_append_finish_uids(t, 123, &saved_uids_arr); + saved_uids = array_get(&saved_uids_arr, &count); + test_assert(count == 1); + test_assert(saved_uids[0].seq1 == 123 && saved_uids[0].seq2 == 124); + + appends = array_get(&t->appends, &count); + test_assert(appends[0].uid == 123); + test_assert(appends[0].flags == 0); + test_assert(appends[1].uid == 124); + test_assert(appends[1].flags == 0); + test_end(); + mail_index_transaction_cleanup(t); + + /* test with some uids */ + t = mail_index_transaction_new(); + + test_begin("mail index append with uids"); + mail_index_append(t, 0, &seq); + test_assert(seq == 5); + mail_index_append(t, 126, &seq); + test_assert(seq == 6); + test_assert(!t->appends_nonsorted); + mail_index_append(t, 124, &seq); + test_assert(seq == 7); + test_assert(t->appends_nonsorted); + mail_index_append(t, 0, &seq); + test_assert(seq == 8); + mail_index_append(t, 128, &seq); + test_assert(seq == 9); + test_assert(t->highest_append_uid == 128); + + mail_index_append_finish_uids(t, 125, &saved_uids_arr); + saved_uids = array_get(&saved_uids_arr, &count); + test_assert(count == 4); + test_assert(saved_uids[0].seq1 == 129 && saved_uids[0].seq2 == 129); + test_assert(saved_uids[1].seq1 == 126 && saved_uids[1].seq2 == 126); + test_assert(saved_uids[2].seq1 == 130 && saved_uids[2].seq2 == 131); + test_assert(saved_uids[3].seq1 == 128 && saved_uids[3].seq2 == 128); + + appends = array_get(&t->appends, &count); + test_assert(count == 5); + test_assert(appends[0].uid == 129); + test_assert(appends[1].uid == 126); + test_assert(appends[2].uid == 130); + test_assert(appends[3].uid == 131); + test_assert(appends[4].uid == 128); + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_flag_update_fastpath(void) +{ + struct mail_index_transaction *t; + const struct mail_index_flag_update *updates; + unsigned int count; + + hdr.messages_count = 20; + t = mail_index_transaction_new(); + + test_begin("mail index flag update fast paths"); + + mail_index_update_flags_range(t, 13, 14, MODIFY_REPLACE, + MAIL_DELETED); + test_assert(t->last_update_idx == 0); + test_assert(array_count(&t->updates) == 1); + + mail_index_update_flags_range(t, 15, 15, MODIFY_REPLACE, + MAIL_DELETED); + test_assert(t->last_update_idx == 0); + test_assert(array_count(&t->updates) == 1); + + mail_index_update_flags_range(t, 16, 16, MODIFY_ADD, + MAIL_DELETED); + test_assert(t->last_update_idx == 1); + test_assert(array_count(&t->updates) == 2); + + updates = array_get(&t->updates, &count); + test_assert(updates[0].uid1 == 13); + test_assert(updates[0].uid2 == 15); + test_assert(updates[0].add_flags == MAIL_DELETED); + test_assert(updates[0].remove_flags == + (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_SEEN | MAIL_DRAFT)); + test_assert(updates[1].uid1 == 16); + test_assert(updates[1].uid2 == 16); + test_assert(updates[1].add_flags == MAIL_DELETED); + test_assert(updates[1].remove_flags == 0); + test_assert(!t->log_updates); + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_flag_update_simple_merges(void) +{ + struct mail_index_transaction *t; + const struct mail_index_flag_update *updates; + unsigned int count; + + hdr.messages_count = 20; + t = mail_index_transaction_new(); + + test_begin("mail index flag update simple merges"); + + mail_index_update_flags_range(t, 6, 8, MODIFY_ADD, + MAIL_FLAGGED); + test_assert(t->last_update_idx == 0); + mail_index_update_flags_range(t, 5, 6, MODIFY_ADD, + MAIL_FLAGGED); + test_assert(t->last_update_idx == 0); + mail_index_update_flags_range(t, 4, 4, MODIFY_ADD, + MAIL_FLAGGED); + test_assert(t->last_update_idx == 0); + mail_index_update_flags_range(t, 7, 9, MODIFY_ADD, + MAIL_FLAGGED); + test_assert(t->last_update_idx == 0); + mail_index_update_flags_range(t, 10, 10, MODIFY_ADD, + MAIL_FLAGGED); + updates = array_get(&t->updates, &count); + test_assert(count == 1); + test_assert(updates[0].uid1 == 4); + test_assert(updates[0].uid2 == 10); + test_assert(updates[0].add_flags == MAIL_FLAGGED); + test_assert(updates[0].remove_flags == 0); + + mail_index_update_flags_range(t, 12, 12, MODIFY_ADD, + MAIL_FLAGGED); + mail_index_update_flags_range(t, 11, 11, MODIFY_ADD, + MAIL_FLAGGED); + updates = array_get(&t->updates, &count); + test_assert(count == 1); + test_assert(updates[0].uid1 == 4); + test_assert(updates[0].uid2 == 12); + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_flag_update_complex_merges(void) +{ + struct mail_index_transaction *t; + const struct mail_index_flag_update *updates; + unsigned int count; + + hdr.messages_count = 20; + t = mail_index_transaction_new(); + + test_begin("mail index flag update complex merges"); + + mail_index_update_flags_range(t, 6, 8, MODIFY_REPLACE, + MAIL_SEEN); + mail_index_update_flags_range(t, 3, 6, MODIFY_ADD, + MAIL_FLAGGED); + mail_index_update_flags_range(t, 5, 7, MODIFY_ADD, + MAIL_DRAFT); + mail_index_update_flags_range(t, 6, 6, MODIFY_REPLACE, + MAIL_SEEN | MAIL_ANSWERED); + mail_index_update_flags_range(t, 5, 10, MODIFY_REMOVE, + MAIL_ANSWERED); + mail_index_update_flags_range(t, 7, 12, MODIFY_ADD, + MAIL_DELETED); + + updates = array_get(&t->updates, &count); + test_assert(count == 7); + test_assert(updates[0].uid1 == 3); + test_assert(updates[0].uid2 == 4); + test_assert(updates[0].add_flags == MAIL_FLAGGED); + test_assert(updates[0].remove_flags == 0); + test_assert(updates[1].uid1 == 5); + test_assert(updates[1].uid2 == 5); + test_assert(updates[1].add_flags == (MAIL_DRAFT | MAIL_FLAGGED)); + test_assert(updates[1].remove_flags == MAIL_ANSWERED); + test_assert(updates[2].uid1 == 6); + test_assert(updates[2].uid2 == 6); + test_assert(updates[2].add_flags == MAIL_SEEN); + test_assert(updates[2].remove_flags == (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_DELETED | MAIL_DRAFT)); + test_assert(updates[3].uid1 == 7); + test_assert(updates[3].uid2 == 7); + test_assert(updates[3].add_flags == (MAIL_SEEN | MAIL_DRAFT | MAIL_DELETED)); + test_assert(updates[3].remove_flags == (MAIL_ANSWERED | MAIL_FLAGGED)); + test_assert(updates[4].uid1 == 8); + test_assert(updates[4].uid2 == 8); + test_assert(updates[4].add_flags == (MAIL_SEEN | MAIL_DELETED)); + test_assert(updates[4].remove_flags == (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_DRAFT)); + test_assert(updates[5].uid1 == 9); + test_assert(updates[5].uid2 == 10); + test_assert(updates[5].add_flags == MAIL_DELETED); + test_assert(updates[5].remove_flags == MAIL_ANSWERED); + test_assert(updates[6].uid1 == 11); + test_assert(updates[6].uid2 == 12); + test_assert(updates[6].add_flags == MAIL_DELETED); + test_assert(updates[6].remove_flags == 0); + + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void +flags_array_check(struct mail_index_transaction *t, + const enum mail_flags *flags, unsigned int msg_count) +{ + const struct mail_index_flag_update *updates; + unsigned int i, count, seq; + + if (array_is_created(&t->updates)) + updates = array_get(&t->updates, &count); + else { + updates = NULL; + count = 0; + } + for (seq = 1, i = 0; i < count; i++) { + if (i > 0) { + test_assert(updates[i-1].uid2 < updates[i].uid1); + test_assert(updates[i-1].uid2 + 1 != updates[i].uid1 || + updates[i-1].add_flags != updates[i].add_flags || + updates[i-1].remove_flags != updates[i].remove_flags); + } + for (; seq != updates[i].uid1; seq++) + test_assert(flags[seq] == 0); + for (; seq <= updates[i].uid2; seq++) + test_assert(flags[seq] == updates[i].add_flags); + } + for (; seq <= msg_count; seq++) + test_assert(flags[seq] == 0); +} + +static void test_mail_index_flag_update_random(void) +{ + struct mail_index_transaction *t; + unsigned int r, seq1, seq2, seq; + enum mail_flags *flags, change; + enum modify_type modify_type; + + hdr.messages_count = 20; + t = mail_index_transaction_new(); + + test_begin("mail index flag update random"); + + flags = t_new(enum mail_flags, hdr.messages_count + 1); + for (r = 0; r < 1000; r++) { + change = i_rand_limit(MAIL_FLAGS_NONRECENT + 1); + seq1 = i_rand_minmax(1, hdr.messages_count); + seq2 = seq1 == hdr.messages_count ? seq1 : + i_rand_minmax(seq1, hdr.messages_count); + + switch (i_rand_limit(3)) { + case 0: + modify_type = MODIFY_ADD; + for (seq = seq1; seq <= seq2; seq++) + flags[seq] |= change; + break; + case 1: + modify_type = MODIFY_REMOVE; + for (seq = seq1; seq <= seq2; seq++) + flags[seq] &= ENUM_NEGATE(change); + break; + case 2: + modify_type = MODIFY_REPLACE; + for (seq = seq1; seq <= seq2; seq++) + flags[seq] = change; + break; + default: + i_unreached(); + } + mail_index_update_flags_range(t, seq1, seq2, modify_type, + change); + flags_array_check(t, flags, hdr.messages_count); + } + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_cancel_flag_updates(void) +{ + struct mail_index_transaction *t; + const struct mail_index_flag_update *updates; + unsigned int count; + + hdr.messages_count = 20; + t = mail_index_transaction_new(); + + test_begin("mail index cancel flag updates"); + + mail_index_update_flags_range(t, 5, 7, MODIFY_REPLACE, 0); + updates = array_get(&t->updates, &count); + test_assert(count == 1); + test_assert(updates[0].uid1 == 5 && updates[0].uid2 == 7); + test_assert(mail_index_cancel_flag_updates(t, 5)); + test_assert(updates[0].uid1 == 6 && updates[0].uid2 == 7); + test_assert(mail_index_cancel_flag_updates(t, 7)); + test_assert(updates[0].uid1 == 6 && updates[0].uid2 == 6); + test_assert(mail_index_cancel_flag_updates(t, 6)); + test_assert(!array_is_created(&t->updates)); + + mail_index_update_flags_range(t, 5, 7, MODIFY_REPLACE, 0); + test_assert(mail_index_cancel_flag_updates(t, 6)); + updates = array_get(&t->updates, &count); + test_assert(count == 2); + test_assert(updates[0].uid1 == 5 && updates[0].uid2 == 5); + test_assert(updates[1].uid1 == 7 && updates[1].uid2 == 7); + + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_flag_update_appends(void) +{ + struct mail_index_transaction *t; + const struct mail_index_record *appends; + const struct mail_index_flag_update *updates; + unsigned int count; + uint32_t seq; + + hdr.messages_count = 4; + t = mail_index_transaction_new(); + + test_begin("mail index flag update appends"); + mail_index_append(t, 0, &seq); + test_assert(seq == 5); + mail_index_append(t, 0, &seq); + test_assert(seq == 6); + mail_index_append(t, 0, &seq); + test_assert(seq == 7); + + mail_index_update_flags_range(t, 5, 6, MODIFY_REPLACE, + MAIL_SEEN | MAIL_FLAGGED); + mail_index_update_flags_range(t, 6, 7, MODIFY_ADD, + MAIL_DRAFT | MAIL_FLAGGED); + mail_index_update_flags_range(t, 5, 7, MODIFY_REMOVE, + MAIL_FLAGGED); + + appends = array_get(&t->appends, &count); + test_assert(count == 3); + test_assert(appends[0].flags == MAIL_SEEN); + test_assert(appends[1].flags == (MAIL_SEEN | MAIL_DRAFT)); + test_assert(appends[2].flags == MAIL_DRAFT); + + /* mixed existing/appends */ + mail_index_update_flags_range(t, 4, 5, MODIFY_ADD, + MAIL_ANSWERED); + test_assert(appends[0].flags == (MAIL_SEEN | MAIL_ANSWERED)); + + updates = array_get(&t->updates, &count); + test_assert(count == 1); + test_assert(updates[0].uid1 == 4); + test_assert(updates[0].uid2 == 4); + test_assert(updates[0].add_flags == MAIL_ANSWERED); + test_end(); + + mail_index_transaction_cleanup(t); +} + +static bool test_flag_update_pos(struct mail_index_transaction *t, + uint32_t seq, unsigned int idx) +{ + unsigned int i, j, count; + + count = array_count(&t->updates); + for (i = 0; i < idx; i++) { + for (j = idx + 1; j <= count; j++) { + if (mail_index_transaction_get_flag_update_pos(t, i, j, seq) != idx) { + test_assert(FALSE); + return FALSE; + } + } + } + return TRUE; +} + +static void test_mail_index_transaction_get_flag_update_pos(void) +{ + struct mail_index_transaction *t; + + test_begin("mail index transaction get flag update pos"); + + hdr.messages_count = 10; + t = mail_index_transaction_new(); + mail_index_update_flags_range(t, 1, 1, MODIFY_REPLACE, 0); + mail_index_update_flags_range(t, 3, 4, MODIFY_REPLACE, 0); + mail_index_update_flags_range(t, 6, 7, MODIFY_REPLACE, 0); + mail_index_update_flags_range(t, 9, 10, MODIFY_REPLACE, 0); + + test_assert(test_flag_update_pos(t, 1, 0)); + test_assert(test_flag_update_pos(t, 2, 1)); + test_assert(test_flag_update_pos(t, 3, 1)); + test_assert(test_flag_update_pos(t, 4, 1)); + test_assert(test_flag_update_pos(t, 5, 2)); + test_assert(test_flag_update_pos(t, 6, 2)); + test_assert(test_flag_update_pos(t, 7, 2)); + test_assert(test_flag_update_pos(t, 8, 3)); + test_assert(test_flag_update_pos(t, 9, 3)); + test_assert(test_flag_update_pos(t, 10, 3)); + test_assert(test_flag_update_pos(t, 11, 4)); + test_assert(test_flag_update_pos(t, 12, 4)); + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_modseq_update(void) +{ + struct mail_index_transaction *t; + const struct mail_transaction_modseq_update *ups; + unsigned int count; + + test_begin("mail index modseq update"); + + hdr.messages_count = 10; + t = mail_index_transaction_new(); + + mail_index_update_modseq(t, 4, 0x8234fefa02747429ULL); + mail_index_update_modseq(t, 6, 0x1234567890abcdefULL); + mail_index_update_modseq(t, 2, 0xfeed); + mail_index_update_modseq(t, 4, 2); + /* modseq=1 updates are ignored: */ + mail_index_update_modseq(t, 5, 1); + mail_index_update_modseq(t, 6, 1); + + ups = array_get(&t->modseq_updates, &count); + test_assert(count == 4); + test_assert(ups[0].uid == 4 && + ups[0].modseq_high32 == 0x8234fefa && + ups[0].modseq_low32 == 0x02747429); + test_assert(ups[1].uid == 6 && + ups[1].modseq_high32 == 0x12345678 && + ups[1].modseq_low32 == 0x90abcdef); + test_assert(ups[2].uid == 2 && + ups[2].modseq_high32 == 0 && + ups[2].modseq_low32 == 0xfeed); + test_assert(ups[3].uid == 4 && + ups[3].modseq_high32 == 0 && + ups[3].modseq_low32 == 2); + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_expunge(void) +{ + static guid_128_t empty_guid = { 0, }; + struct mail_index_transaction *t; + const struct mail_transaction_expunge_guid *expunges; + guid_128_t guid2, guid3, guid4; + unsigned int i, count; + + test_begin("mail index expunge"); + + hdr.messages_count = 10; + t = mail_index_transaction_new(); + for (i = 0; i < sizeof(guid2); i++) { + guid2[i] = i + 1; + guid3[i] = i ^ 0xff; + guid4[i] = i + 0x80; + } + + mail_index_expunge_guid(t, 4, guid4); + test_assert(!t->expunges_nonsorted); + mail_index_expunge_guid(t, 2, guid2); + test_assert(t->expunges_nonsorted); + mail_index_expunge_guid(t, 3, guid3); + mail_index_expunge(t, 1); + mail_index_expunge(t, 5); + + expunges = array_get(&t->expunges, &count); + test_assert(count == 5); + test_assert(expunges[0].uid == 4); + test_assert(memcmp(expunges[0].guid_128, guid4, sizeof(guid4)) == 0); + test_assert(expunges[1].uid == 2); + test_assert(memcmp(expunges[1].guid_128, guid2, sizeof(guid2)) == 0); + test_assert(expunges[2].uid == 3); + test_assert(memcmp(expunges[2].guid_128, guid3, sizeof(guid3)) == 0); + test_assert(expunges[3].uid == 1); + test_assert(memcmp(expunges[3].guid_128, empty_guid, sizeof(empty_guid)) == 0); + test_assert(expunges[4].uid == 5); + test_assert(memcmp(expunges[4].guid_128, empty_guid, sizeof(empty_guid)) == 0); + + test_end(); + + mail_index_transaction_cleanup(t); +} + +static void test_mail_index_update_day_first_uid(void) +{ + struct { + uint32_t now; + uint32_t old_day_stamp; + uint32_t new_day_stamp; + uint32_t new_day_first_uid[8]; + } tests[] = { + /* 1487116800 = 2017-02-15 00:00:00 UTC */ + { 1487116800, 1487116800, 1487116800, { 8, 7, 6, 5, 4, 3, 2, 1 } }, + /* still same day */ + { 1487116800+3600*24-1, 1487116800, 1487116800, { 8, 7, 6, 5, 4, 3, 2, 1 } }, + /* one day earlier */ + { 1487116800-1, 1487116800, 1487116800, { 8, 7, 6, 5, 4, 3, 2, 1 } }, + /* next day */ + { 1487116800+3600*24, 1487116800, 1487116800+3600*24, { 9, 8, 7, 6, 5, 4, 3, 2 } }, + { 1487116800+3600*24*2-1, 1487116800, 1487116800+3600*24, { 9, 8, 7, 6, 5, 4, 3, 2 } }, + /* 2 days */ + { 1487116800+3600*24*2, 1487116800, 1487116800+3600*24*2, { 9, 8, 8, 7, 6, 5, 4, 3 } }, + /* 3 days */ + { 1487116800+3600*24*3, 1487116800, 1487116800+3600*24*3, { 9, 8, 8, 8, 7, 6, 5, 4 } }, + /* 4 days */ + { 1487116800+3600*24*4, 1487116800, 1487116800+3600*24*4, { 9, 8, 8, 8, 8, 7, 6, 5 } }, + /* 5 days */ + { 1487116800+3600*24*5, 1487116800, 1487116800+3600*24*5, { 9, 8, 8, 8, 8, 8, 7, 6 } }, + /* 6 days */ + { 1487116800+3600*24*6, 1487116800, 1487116800+3600*24*6, { 9, 8, 8, 8, 8, 8, 8, 7 } }, + /* 7 days */ + { 1487116800+3600*24*7, 1487116800, 1487116800+3600*24*7, { 9, 8, 8, 8, 8, 8, 8, 8 } }, + /* 8 days */ + { 1487116800+3600*24*8, 1487116800, 1487116800+3600*24*8, { 9, 8, 8, 8, 8, 8, 8, 8 } }, + /* 366 days */ + { 1487116800+3600*24*366, 1487116800, 1487116800+3600*24*366, { 9, 8, 8, 8, 8, 8, 8, 8 } }, + }; + struct mail_index_transaction *t; + struct mail_index_record *rec; + unsigned int i, j; + + test_begin("mail index update day first uid"); + + /* daylight savings times were confusing these tests, so we'll now + just assume that TZ=UTC */ + test_assert(timezone == 0); + + hdr.messages_count = 10; + t = mail_index_transaction_new(); + t->view = t_new(struct mail_index_view, 1); + t->view->map = t_new(struct mail_index_map, 1); + + t_array_init(&t->appends, 1); + rec = array_append_space(&t->appends); + rec->uid = 9; + + for (i = 0; i < N_ELEMENTS(tests); i++) { + i_zero(&hdr); + for (j = 0; j < N_ELEMENTS(hdr.day_first_uid); j++) + hdr.day_first_uid[j] = 8-j; + hdr.day_stamp = tests[i].old_day_stamp + timezone; + memcpy(t->post_hdr_change, &hdr, sizeof(hdr)); + mail_index_update_day_headers(t, tests[i].now + timezone); + + struct mail_index_header new_hdr; + memcpy(&new_hdr, t->post_hdr_change, sizeof(new_hdr)); + test_assert_idx(new_hdr.day_stamp == tests[i].new_day_stamp + timezone, i); + test_assert_idx(memcmp(new_hdr.day_first_uid, + tests[i].new_day_first_uid, + sizeof(uint32_t) * 8) == 0, i); + } + + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_index_append, + test_mail_index_flag_update_fastpath, + test_mail_index_flag_update_simple_merges, + test_mail_index_flag_update_complex_merges, + test_mail_index_flag_update_random, + test_mail_index_flag_update_appends, + test_mail_index_cancel_flag_updates, + test_mail_index_transaction_get_flag_update_pos, + test_mail_index_modseq_update, + test_mail_index_expunge, + test_mail_index_update_day_first_uid, + NULL + }; + /* daylight saving time confuses things */ + env_put("TZ", "UTC"); + tzset(); + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-index-write.c b/src/lib-index/test-mail-index-write.c new file mode 100644 index 0000000..88eaa4a --- /dev/null +++ b/src/lib-index/test-mail-index-write.c @@ -0,0 +1,151 @@ +/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "test-common.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" + +#define TEST_INDEX_FNAME ".test.index.write" +#define TEST_INDEXID 123456 +#define LOG_FILE1_HEAD_OFFSET 200 + +static bool expect_index_rewrite; +static bool rotate_fail; + +static struct mail_transaction_log_file log_file = { + .hdr = { + .indexid = TEST_INDEXID, + .file_seq = 1, + }, +}; +static struct mail_transaction_log_file log_file2 = { + .hdr = { + .indexid = TEST_INDEXID, + .file_seq = 2, + .prev_file_seq = 1, + .prev_file_offset = LOG_FILE1_HEAD_OFFSET, + }, +}; + +void mail_index_set_error(struct mail_index *index ATTR_UNUSED, + const char *fmt ATTR_UNUSED, ...) +{ +} + +void mail_index_set_syscall_error(struct mail_index *index ATTR_UNUSED, + const char *function) +{ + i_error("%s() failed: %m", function); +} + +void mail_index_file_set_syscall_error(struct mail_index *index ATTR_UNUSED, + const char *filepath, + const char *function) +{ + i_error("%s(%s) failed: %m", function, filepath); +} + +int mail_index_create_tmp_file(struct mail_index *index ATTR_UNUSED, + const char *path_prefix, const char **path_r) +{ + const char *path; + int fd; + + test_assert(expect_index_rewrite); + + path = *path_r = t_strconcat(path_prefix, ".tmp", NULL); + fd = open(path, O_RDWR|O_CREAT, 0600); + if (fd == -1) { + i_error("creat() failed: %m"); + return -1; + } + return fd; +} + +int mail_index_move_to_memory(struct mail_index *index ATTR_UNUSED) +{ + return -1; +} + +int mail_transaction_log_rotate(struct mail_transaction_log *log, bool reset) +{ + i_assert(!reset); + + if (rotate_fail) + return -1; + + log_file.next = &log_file2; + log->head = &log_file2; + return 0; +} + +static void test_mail_index_write(void) +{ + struct mail_transaction_log log = { + .head = &log_file, + .files = &log_file, + }; + struct mail_index_record_map rec_map = { + .records_count = 0, + }; + buffer_t hdr_copy; + struct mail_index_map map = { + .hdr = { + .indexid = TEST_INDEXID, + .log_file_seq = 1, + .log_file_tail_offset = 100, + .log_file_head_offset = LOG_FILE1_HEAD_OFFSET, + }, + .hdr_copy_buf = &hdr_copy, + .rec_map = &rec_map, + }; + buffer_create_from_const_data(&hdr_copy, &map.hdr, sizeof(map.hdr)); + struct mail_index index = { + .event = event_create(NULL), + .log = &log, + .map = &map, + .dir = ".", + .fd = -1, + .indexid = TEST_INDEXID, + .filepath = TEST_INDEX_FNAME, + .log_sync_locked = TRUE, + }; + + test_begin("test_mail_index_write()"); + + /* test failed rotation, no index rewrite */ + rotate_fail = TRUE; + expect_index_rewrite = FALSE; + test_assert(!index.reopen_main_index); + index.fd = 1; /* anything but -1 */ + mail_index_write(&index, TRUE, "testing"); + test_assert(log.head == log.files); + test_assert(index.reopen_main_index); + + /* test failed rotation, with index rewrite */ + expect_index_rewrite = TRUE; + index.reopen_main_index = FALSE; + index.fd = -1; + mail_index_write(&index, TRUE, "testing"); + test_assert(log.head == log.files); + test_assert(!index.reopen_main_index); + + /* test successful rotation, with index rewrite */ + rotate_fail = FALSE; + mail_index_write(&index, TRUE, "testing"); + test_assert(log.head != log.files && log.head == &log_file2); + test_assert(!index.reopen_main_index); + + event_unref(&index.event); + i_unlink(TEST_INDEX_FNAME); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_index_write, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-index.c b/src/lib-index/test-mail-index.c new file mode 100644 index 0000000..3e2fd02 --- /dev/null +++ b/src/lib-index/test-mail-index.c @@ -0,0 +1,169 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "test-common.h" +#include "test-mail-index.h" +#include "mail-transaction-log-private.h" + +static void test_mail_index_rotate(void) +{ + struct mail_index *index, *index2; + struct mail_index_view *view; + struct mail_index_transaction *trans; + struct mail_transaction_log_file *file; + const char *reason; + + test_begin("mail index rotate"); + index = test_mail_index_init(); + index2 = test_mail_index_open(); + view = mail_index_view_open(index); + + /* First rotation of the index. The view will point to the old index. */ + trans = mail_index_transaction_begin(view, 0); + mail_index_reset(trans); + test_assert(mail_index_transaction_commit(&trans) == 0); + + /* Second rotation of the index. The log head doesn't have any extra + references. */ + trans = mail_index_transaction_begin(view, 0); + mail_index_reset(trans); + test_assert(mail_index_transaction_commit(&trans) == 0); + + /* The 2nd index's log head also doesn't have any extra references. + Check that it doesn't crash. */ + test_assert(mail_transaction_log_find_file(index2->log, 3, FALSE, &file, &reason) == 0); + + mail_index_view_close(&view); + test_mail_index_deinit(&index); + test_mail_index_deinit(&index2); + test_end(); +} + +static void +test_mail_index_new_extension_rotate_write(struct mail_index *index2, + uint32_t uid) +{ + struct mail_index_view *view2; + struct mail_index_transaction *trans; + uint32_t hdr_ext_id, rec_ext_id, file_seq, seq, rec_ext = 0x12345678; + uoff_t file_offset; + + /* Rotate the index in the index */ + test_assert(mail_transaction_log_sync_lock(index2->log, "test", + &file_seq, &file_offset) == 0); + mail_index_write(index2, TRUE, "test"); + mail_transaction_log_sync_unlock(index2->log, "test"); + + /* Write a new extension header to the 2nd index. */ + hdr_ext_id = mail_index_ext_register(index2, "test", + sizeof(hdr_ext_id), 0, 0); + rec_ext_id = mail_index_ext_register(index2, "test-rec", 0, + sizeof(uint32_t), sizeof(uint32_t)); + view2 = mail_index_view_open(index2); + trans = mail_index_transaction_begin(view2, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_header_ext(trans, hdr_ext_id, 0, + &hdr_ext_id, sizeof(hdr_ext_id)); + mail_index_append(trans, uid, &seq); + mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&view2); +} + +static void test_mail_index_new_extension_sync(struct mail_index_view *view) +{ + struct mail_index_view_sync_ctx *sync_ctx; + struct mail_index_view_sync_rec sync_rec; + bool delayed_expunges; + + test_assert(mail_index_refresh(view->index) == 0); + sync_ctx = mail_index_view_sync_begin(view, + MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES); + test_assert(!mail_index_view_sync_next(sync_ctx, &sync_rec)); + test_assert(mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) == 0); +} + +static void test_mail_index_new_extension(void) +{ + struct mail_index *index, *index2; + struct mail_index_view *view, *view2; + struct mail_index_transaction *trans; + uint32_t seq, rec_ext_id, rec_ext = 0x12345678; + + test_begin("mail index new extension"); + index = test_mail_index_init(); + index2 = test_mail_index_open(); + view = mail_index_view_open(index); + + rec_ext_id = mail_index_ext_register(index, "test-rec", 0, + sizeof(uint32_t), sizeof(uint32_t)); + + /* Save two mails */ + uint32_t uid_validity = 123456; + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + mail_index_append(trans, 1, &seq); + mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL); + mail_index_append(trans, 2, &seq); + mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL); + test_assert(mail_index_transaction_commit(&trans) == 0); + + /* refresh indexes and view */ + test_assert(mail_index_refresh(index2) == 0); + mail_index_view_close(&view); + view = mail_index_view_open(index); + + /* Expunge the mail in the 2nd index */ + view2 = mail_index_view_open(index2); + trans = mail_index_transaction_begin(view2, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_expunge(trans, 1); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&view2); + + /* Sync the first view without expunges */ + test_mail_index_new_extension_sync(view); + + for (unsigned int i = 0; i < 3; i++) + test_mail_index_new_extension_rotate_write(index2, 3 + i); + + /* Sync the first view. It needs to generate the missing view. */ + test_expect_error_string("generating missing logs"); + test_mail_index_new_extension_sync(view); + test_expect_no_more_errors(); + test_assert(mail_index_get_header(view)->messages_count == 5); + + /* Make sure the extensions records are still there. + Note that this works, because the extensions are looked up from the + newly refreshed index, not the old index. */ + for (seq = 1; seq <= 5; seq++) { + const void *data; + bool expunged; + mail_index_lookup_ext(view, seq, rec_ext_id, &data, &expunged); + test_assert_idx(memcmp(data, &rec_ext, sizeof(rec_ext)) == 0, seq); + } + + /* Once more rotate and write using the new extension */ + test_mail_index_new_extension_rotate_write(index2, 6); + /* Make sure the first view understands the new extension by ID */ + test_mail_index_new_extension_sync(view); + + mail_index_view_close(&view); + test_mail_index_deinit(&index); + test_mail_index_deinit(&index2); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_index_rotate, + test_mail_index_new_extension, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-index.h b/src/lib-index/test-mail-index.h new file mode 100644 index 0000000..75b343f --- /dev/null +++ b/src/lib-index/test-mail-index.h @@ -0,0 +1,51 @@ +#ifndef TEST_MAIL_INDEX_H +#define TEST_MAIL_INDEX_H + +#include "ioloop.h" +#include "unlink-directory.h" +#include "mail-index-private.h" + +#define TESTDIR_NAME ".dovecot.test" + +static inline struct mail_index *test_mail_index_open(void) +{ + struct mail_index *index; + + index = mail_index_alloc(NULL, TESTDIR_NAME, "test.dovecot.index"); + test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0); + return index; +} + +static inline struct mail_index *test_mail_index_init(void) +{ + const char *error; + + (void)unlink_directory(TESTDIR_NAME, UNLINK_DIRECTORY_FLAG_RMDIR, &error); + if (mkdir(TESTDIR_NAME, 0700) < 0) + i_error("mkdir(%s) failed: %m", TESTDIR_NAME); + + ioloop_time = 1; + + return test_mail_index_open(); +} + +static inline void test_mail_index_close(struct mail_index **index) +{ + mail_index_close(*index); + mail_index_free(index); +} + +static inline void test_mail_index_delete(void) +{ + const char *error; + + (void)unlink_directory(TESTDIR_NAME, UNLINK_DIRECTORY_FLAG_RMDIR, &error); +} + +static inline void test_mail_index_deinit(struct mail_index **index) +{ + test_mail_index_close(index); + test_mail_index_delete(); +} + +#endif diff --git a/src/lib-index/test-mail-transaction-log-append.c b/src/lib-index/test-mail-transaction-log-append.c new file mode 100644 index 0000000..4029432 --- /dev/null +++ b/src/lib-index/test-mail-transaction-log-append.c @@ -0,0 +1,176 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "test-common.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" + +#include <sys/stat.h> + +static bool log_lock_failure = FALSE; + +void mail_index_file_set_syscall_error(struct mail_index *index ATTR_UNUSED, + const char *filepath ATTR_UNUSED, + const char *function ATTR_UNUSED) +{ +} + +int mail_transaction_log_lock_head(struct mail_transaction_log *log ATTR_UNUSED, + const char *lock_reason ATTR_UNUSED) +{ + return log_lock_failure ? -1 : 0; +} + +void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file ATTR_UNUSED, + const char *lock_reason ATTR_UNUSED) {} + +void mail_transaction_update_modseq(const struct mail_transaction_header *hdr, + const void *data ATTR_UNUSED, + uint64_t *cur_modseq, + unsigned int version ATTR_UNUSED) +{ + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) + *cur_modseq += 1; +} + +int mail_index_move_to_memory(struct mail_index *index ATTR_UNUSED) +{ + return -1; +} + +static void test_append_expunge(struct mail_transaction_log *log) +{ + static unsigned int buf[] = { 0x12345678, 0xabcdef09 }; + struct mail_transaction_log_file *file = log->head; + struct mail_transaction_log_append_ctx *ctx; + const struct mail_transaction_header *hdr; + const unsigned int *bufp; + const struct mail_transaction_boundary *bound; + + test_assert(mail_transaction_log_append_begin(log->index, MAIL_TRANSACTION_EXTERNAL, &ctx) == 0); + mail_transaction_log_append_add(ctx, MAIL_TRANSACTION_APPEND, + &buf[0], sizeof(buf[0])); + test_assert(ctx->new_highest_modseq == 0); + mail_transaction_log_append_add(ctx, MAIL_TRANSACTION_EXPUNGE, + &buf[1], sizeof(buf[1])); + test_assert(ctx->new_highest_modseq == 1); + + test_assert(mail_transaction_log_append_commit(&ctx) == 0); + test_assert(file->sync_highest_modseq == 1); + test_assert(file->sync_offset == file->buffer_offset + file->buffer->used); + + hdr = file->buffer->data; + test_assert(hdr->type == (MAIL_TRANSACTION_BOUNDARY | + MAIL_TRANSACTION_EXTERNAL)); + test_assert(mail_index_offset_to_uint32(hdr->size) == sizeof(*hdr) + sizeof(*bound)); + bound = (const void *)(hdr + 1); + test_assert(bound->size == file->buffer->used); + hdr = (const void *)(bound + 1); + + test_assert(hdr->type == (MAIL_TRANSACTION_APPEND | + MAIL_TRANSACTION_EXTERNAL)); + test_assert(mail_index_offset_to_uint32(hdr->size) == sizeof(*hdr) + sizeof(buf[0])); + bufp = (const void *)(hdr + 1); + test_assert(*bufp == buf[0]); + + hdr = (const void *)(bufp + 1); + test_assert(hdr->type == (MAIL_TRANSACTION_EXPUNGE | + MAIL_TRANSACTION_EXPUNGE_PROT | + MAIL_TRANSACTION_EXTERNAL)); + test_assert(mail_index_offset_to_uint32(hdr->size) == sizeof(*hdr) + sizeof(buf[0])); + bufp = (const void *)(hdr + 1); + test_assert(*bufp == buf[1]); + + test_assert(file->buffer->used == (size_t)((const char *)(bufp+1) - (const char *)file->buffer->data)); + + buffer_set_used_size(file->buffer, 0); + file->buffer_offset = 0; + test_end(); +} + +static void test_append_sync_offset(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file = log->head; + struct mail_transaction_log_append_ctx *ctx; + const struct mail_transaction_header *hdr; + const struct mail_transaction_header_update *u; + const uint32_t *offsetp; + + test_begin("transaction log append: append_sync_offset only"); + test_assert(mail_transaction_log_append_begin(log->index, 0, &ctx) == 0); + ctx->index_sync_transaction = TRUE; + file->max_tail_offset = 123; + test_assert(mail_transaction_log_append_commit(&ctx) == 0); + + test_assert(file->buffer->used == sizeof(*hdr) + sizeof(*u) + sizeof(*offsetp)); + hdr = file->buffer->data; + test_assert(hdr->type == MAIL_TRANSACTION_HEADER_UPDATE); + test_assert(mail_index_offset_to_uint32(hdr->size) == file->buffer->used); + u = (const void *)(hdr + 1); + test_assert(u->offset == offsetof(struct mail_index_header, log_file_tail_offset)); + test_assert(u->size == sizeof(*offsetp)); + offsetp = (const void *)(u+1); + test_assert(*offsetp == 123); + + test_end(); +} + +static void test_mail_transaction_log_append(void) +{ + struct mail_transaction_log *log; + struct mail_transaction_log_file *file; + struct mail_transaction_log_append_ctx *ctx; + char tmp_path[] = "/tmp/dovecot.test.XXXXXX"; + struct stat st; + int fd; + + fd = mkstemp(tmp_path); + if (fd == -1) + i_fatal("mkstemp(%s) failed: %m", tmp_path); + + test_begin("transaction log append"); + log = i_new(struct mail_transaction_log, 1); + log->index = i_new(struct mail_index, 1); + log->index->log = log; + log->head = file = i_new(struct mail_transaction_log_file, 1); + file->fd = -1; + + test_append_expunge(log); + + test_begin("transaction log append: lock failure"); + log_lock_failure = TRUE; + test_assert(mail_transaction_log_append_begin(log->index, 0, &ctx) < 0); + log_lock_failure = FALSE; + test_end(); + + test_append_sync_offset(log); + + /* do this after head->buffer has already been initialized */ + test_begin("transaction log append: garbage truncation"); + file->sync_offset = 1; + file->buffer_offset = 1; + file->last_size = 3; + file->fd = fd; + test_assert(mail_transaction_log_append_begin(log->index, 0, &ctx) == 0); + test_assert(mail_transaction_log_append_commit(&ctx) == 0); + if (fstat(fd, &st) < 0) i_fatal("fstat() failed: %m"); + test_assert(st.st_size == 1); + file->fd = -1; + test_end(); + + buffer_free(&log->head->buffer); + i_free(log->head); + i_free(log->index); + i_free(log); + i_unlink(tmp_path); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_transaction_log_append, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-transaction-log-file.c b/src/lib-index/test-mail-transaction-log-file.c new file mode 100644 index 0000000..6f591ce --- /dev/null +++ b/src/lib-index/test-mail-transaction-log-file.c @@ -0,0 +1,418 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "test-common.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" + +#define TEST_LOG_VERSION MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3) + +#define INITIAL_MODSEQ 100 + +struct update_modseq_test { + enum mail_transaction_type type; + unsigned int version; +#define NOUPDATE (INITIAL_MODSEQ) +#define UPDATE (INITIAL_MODSEQ+1) + uint64_t expected_modseq; + unsigned int count; + union { + const struct mail_transaction_flag_update *flag_update; + const struct mail_transaction_modseq_update *modseq_update; + } v; +} update_modseq_tests[] = { + /* expunges: increase modseq */ + { MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, UPDATE, 1, { } }, + { MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, UPDATE, 1, { } }, + /* expunges: don't increase modseq */ + { MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + + /* flag changes: don't increase modseq */ + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = 0 } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_BACKEND } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_INDEX_MAIL_FLAG_BACKEND } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_INDEX_MAIL_FLAG_DIRTY } + } + } }, + /* flag changes: increase modseq */ + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_SEEN } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_SEEN } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_BACKEND } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_DIRTY } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_BACKEND } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_DIRTY } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 2, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY }, + { .uid1 = 3, .uid2 = 4, .add_flags = MAIL_SEEN } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = 0, .modseq_inc_flag = 1 } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 2, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY }, + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY, .modseq_inc_flag = 1 } + } + } }, + /* flag changes: increase modseq with old version */ + { MAIL_TRANSACTION_FLAG_UPDATE, MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2), UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_BACKEND } + } + } }, + { MAIL_TRANSACTION_FLAG_UPDATE, MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2), UPDATE, 1, { + .flag_update = (const struct mail_transaction_flag_update[]) { + { .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY } + } + } }, + /* modseq updates: don't increase modseq */ + { MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { + .modseq_update = (const struct mail_transaction_modseq_update[]) { + { .uid = 1, .modseq_low32 = 50, .modseq_high32 = 0 } + } + } }, + { MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { + .modseq_update = (const struct mail_transaction_modseq_update[]) { + { .uid = 1, .modseq_low32 = 100, .modseq_high32 = 0 } + } + } }, + /* modseq updates: increase modseq */ + { MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, 500, 1, { + .modseq_update = (const struct mail_transaction_modseq_update[]) { + { .uid = 1, .modseq_low32 = 500, .modseq_high32 = 0 } + } + } }, + { MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, 500, 2, { + .modseq_update = (const struct mail_transaction_modseq_update[]) { + { .uid = 1, .modseq_low32 = 50, .modseq_high32 = 0 }, + { .uid = 1, .modseq_low32 = 500, .modseq_high32 = 0 } + } + } }, + { MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, 500, 1, { + .modseq_update = (const struct mail_transaction_modseq_update[]) { + { .uid = 1, .modseq_low32 = 500, .modseq_high32 = 0 }, + { .uid = 1, .modseq_low32 = 200, .modseq_high32 = 0 } + } + } }, + { MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, (uint64_t)4294967346, 1, { + .modseq_update = (const struct mail_transaction_modseq_update[]) { + { .uid = 1, .modseq_low32 = 50, .modseq_high32 = 1 } + } + } }, + + /* appends, keyword changes, attribute changes: increase modseq */ + { MAIL_TRANSACTION_APPEND, TEST_LOG_VERSION, UPDATE, 1, { } }, + { MAIL_TRANSACTION_KEYWORD_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { } }, + { MAIL_TRANSACTION_KEYWORD_RESET, TEST_LOG_VERSION, UPDATE, 1, { } }, + { MAIL_TRANSACTION_ATTRIBUTE_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { } }, + + /* others: don't increase modseq */ + { MAIL_TRANSACTION_HEADER_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_HEADER_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXT_INTRO, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXT_RESET, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXT_HDR_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXT_REC_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXT_ATOMIC_INC, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_EXT_HDR_UPDATE32, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_INDEX_DELETED, TEST_LOG_VERSION, NOUPDATE, 1, { } }, + { MAIL_TRANSACTION_INDEX_UNDELETED, TEST_LOG_VERSION, NOUPDATE, 1, { } }, +}; + +static size_t update_modseq_test_get_size(const struct update_modseq_test *test) +{ + enum mail_transaction_type type = + test->type & MAIL_TRANSACTION_TYPE_MASK; + + if (type == (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT)) + type = MAIL_TRANSACTION_EXPUNGE; + if (type == (MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT)) + type = MAIL_TRANSACTION_EXPUNGE_GUID; + + switch (type) { + case MAIL_TRANSACTION_EXPUNGE: + return sizeof(struct mail_transaction_expunge); + case MAIL_TRANSACTION_EXPUNGE_GUID: + return sizeof(struct mail_transaction_expunge_guid); + case MAIL_TRANSACTION_APPEND: + return sizeof(struct mail_index_record); + case MAIL_TRANSACTION_KEYWORD_UPDATE: + return sizeof(struct mail_transaction_keyword_update); + case MAIL_TRANSACTION_KEYWORD_RESET: + return sizeof(struct mail_transaction_keyword_reset); + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: + return 4; + case MAIL_TRANSACTION_FLAG_UPDATE: + return sizeof(struct mail_transaction_flag_update); + case MAIL_TRANSACTION_MODSEQ_UPDATE: + return sizeof(struct mail_transaction_modseq_update); + case MAIL_TRANSACTION_HEADER_UPDATE: + case MAIL_TRANSACTION_EXT_INTRO: + case MAIL_TRANSACTION_EXT_RESET: + case MAIL_TRANSACTION_EXT_HDR_UPDATE: + case MAIL_TRANSACTION_EXT_REC_UPDATE: + case MAIL_TRANSACTION_EXT_ATOMIC_INC: + case MAIL_TRANSACTION_EXT_HDR_UPDATE32: + case MAIL_TRANSACTION_INDEX_DELETED: + case MAIL_TRANSACTION_INDEX_UNDELETED: + return 4; + case MAIL_TRANSACTION_TYPE_MASK: + case MAIL_TRANSACTION_BOUNDARY: + case MAIL_TRANSACTION_EXPUNGE_PROT: + case MAIL_TRANSACTION_EXTERNAL: + case MAIL_TRANSACTION_SYNC: + break; + } + i_unreached(); +} + +static void test_mail_transaction_update_modseq(void) +{ + struct mail_transaction_header hdr; + unsigned char tempbuf[1024] = { 0 }; + + test_begin("mail_transaction_update_modseq()"); + for (unsigned int i = 0; i < N_ELEMENTS(update_modseq_tests); i++) { + const struct update_modseq_test *test = &update_modseq_tests[i]; + const void *data = test->v.flag_update; + uint64_t cur_modseq = INITIAL_MODSEQ; + + if (data == NULL) + data = tempbuf; + + hdr.type = test->type; + hdr.size = sizeof(hdr) + update_modseq_test_get_size(test) * test->count; + hdr.size = mail_index_uint32_to_offset(hdr.size); + mail_transaction_update_modseq(&hdr, data, &cur_modseq, test->version); + test_assert_idx(cur_modseq >= INITIAL_MODSEQ, i); + test_assert_idx(test->expected_modseq == cur_modseq, i); + } + test_end(); +} + +static struct mail_index *test_mail_index_open(void) +{ + struct mail_index *index = mail_index_alloc(NULL, NULL, "test.dovecot.index"); + test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0); + struct mail_index_view *view = mail_index_view_open(index); + + struct mail_index_transaction *trans = + mail_index_transaction_begin(view, 0); + uint32_t uid_validity = 1234; + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&view); + return index; +} + +static void test_mail_transaction_log_file_modseq_offsets(void) +{ + test_begin("mail_transaction_log_file_get_modseq_next_offset() and _get_highest_modseq_at()"); + + struct mail_index *index = test_mail_index_open(); + struct mail_transaction_log_file *file = index->log->head; + + const unsigned int max_modseq = LOG_FILE_MODSEQ_CACHE_SIZE+2; + uoff_t modseq_next_offset[max_modseq+1]; + uoff_t modseq_alt_next_offset[max_modseq+1]; + + /* start with modseq=2, because modseq=1 is the initial state */ + modseq_next_offset[1] = sizeof(struct mail_transaction_log_header); + modseq_alt_next_offset[1] = sizeof(struct mail_transaction_log_header); + for (uint64_t modseq = 2; modseq <= max_modseq; modseq++) { + uint32_t seq; + + struct mail_index_view *view = mail_index_view_open(index); + struct mail_index_transaction *trans = + mail_index_transaction_begin(view, 0); + mail_index_append(trans, modseq, &seq); + test_assert(mail_index_transaction_commit(&trans) == 0); + modseq_next_offset[modseq] = file->sync_offset; + mail_index_view_close(&view); + + /* add a non-modseq updating change */ + view = mail_index_view_open(index); + trans = mail_index_transaction_begin(view, 0); + mail_index_update_flags(trans, seq, MODIFY_ADD, + (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&view); + modseq_alt_next_offset[modseq] = file->sync_offset; + } + + /* mail_transaction_log_file_get_highest_modseq_at() is simultaneously + tested and it can also add offsets to cache. The difference is that + it adds the highest possible offset, while + mail_transaction_log_file_get_modseq_next_offset() adds the lowest + possible offset. So we'll need to allow both. */ +#define MODSEQ_MATCH(modseq, next_offset) \ + ((next_offset) == modseq_next_offset[modseq] || \ + (next_offset) == modseq_alt_next_offset[modseq]) + + /* 1) mail_transaction_log_file_get_modseq_next_offset() tests */ + uint64_t modseq; + uoff_t next_offset; + /* initial_modseq fast path */ + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 1, &next_offset) == 0); + test_assert(next_offset == modseq_next_offset[1]); + /* sync_highest_modseq fast path - it skips to sync_offset instead of + using exactly the same max_modseq */ + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, max_modseq, &next_offset) == 0); + test_assert(next_offset == file->sync_offset); + test_assert(next_offset != modseq_next_offset[max_modseq]); + /* update the offset for the random tests */ + modseq_next_offset[max_modseq] = file->sync_offset; + /* add to cache */ + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0); + test_assert(MODSEQ_MATCH(2, next_offset)); + /* get it from cache */ + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0); + test_assert(MODSEQ_MATCH(2, next_offset)); + /* get next value from cache */ + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 3, &next_offset) == 0); + test_assert(MODSEQ_MATCH(3, next_offset)); + /* get previous value from cache again */ + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0); + test_assert(MODSEQ_MATCH(2, next_offset)); + /* do some random testing with cache */ + for (unsigned int i = 0; i < LOG_FILE_MODSEQ_CACHE_SIZE*10; i++) { + modseq = i_rand_minmax(1, max_modseq); + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, modseq, &next_offset) == 0); + test_assert(MODSEQ_MATCH(modseq, next_offset)); + } + /* go through all modseqs - do this after randomness testing or + modseq_alt_next_offset[] matching isn't triggered */ + for (modseq = 1; modseq <= max_modseq; modseq++) { + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, modseq, &next_offset) == 0); + test_assert(MODSEQ_MATCH(modseq, next_offset)); + } + + /* 2) mail_transaction_log_file_get_highest_modseq_at() tests */ + uint64_t modseq_at; + const char *error; + /* initial_offset */ + test_assert(mail_transaction_log_file_get_highest_modseq_at( + file, modseq_next_offset[1], &modseq, &error) == 1); + test_assert(modseq == 1); + /* sync_offset fast path */ + test_assert(mail_transaction_log_file_get_highest_modseq_at( + file, file->sync_offset, &modseq, &error) == 1); + test_assert(modseq == max_modseq); + /* do some random testing with cache */ + for (unsigned int i = 0; i < LOG_FILE_MODSEQ_CACHE_SIZE*10; i++) { + modseq = i_rand_minmax(1, max_modseq); + test_assert(mail_transaction_log_file_get_highest_modseq_at( + file, modseq_next_offset[modseq], &modseq_at, &error) == 1); + test_assert(modseq_at == modseq); + test_assert(mail_transaction_log_file_get_highest_modseq_at( + file, modseq_alt_next_offset[modseq], &modseq_at, &error) == 1); + test_assert(modseq_at == modseq); + } + /* go through all modseqs - do this after randomness testing or + modseq_alt_next_offset[] matching isn't triggered */ + for (modseq = 1; modseq <= max_modseq; modseq++) { + test_assert(mail_transaction_log_file_get_highest_modseq_at( + file, modseq_next_offset[modseq], &modseq_at, &error) == 1); + test_assert(modseq_at == modseq); + } + + mail_index_close(index); + mail_index_free(&index); + test_end(); +} + +static void +test_mail_transaction_log_file_get_modseq_next_offset_inconsistency(void) +{ + test_begin("mail_transaction_log_file_get_modseq_next_offset() inconsistency"); + + struct mail_index *index = test_mail_index_open(); + struct mail_transaction_log_file *file = index->log->head; + uint32_t seq; + + /* add modseq=2 */ + struct mail_index_view *view = mail_index_view_open(index); + struct mail_index_transaction *trans = + mail_index_transaction_begin(view, 0); + mail_index_append(trans, 1, &seq); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&view); + + /* emulate a broken mail_index_modseq_header header */ + file->sync_highest_modseq = 3; + + uoff_t next_offset; + test_expect_error_string("Transaction log modseq tracking is corrupted"); + test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0); + test_expect_no_more_errors(); + test_assert(next_offset == file->sync_offset); + + mail_index_close(index); + mail_index_free(&index); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_transaction_update_modseq, + test_mail_transaction_log_file_modseq_offsets, + test_mail_transaction_log_file_get_modseq_next_offset_inconsistency, + NULL + }; + ioloop_time = 1; + return test_run(test_functions); +} diff --git a/src/lib-index/test-mail-transaction-log-view.c b/src/lib-index/test-mail-transaction-log-view.c new file mode 100644 index 0000000..17a7628 --- /dev/null +++ b/src/lib-index/test-mail-transaction-log-view.c @@ -0,0 +1,268 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "test-common.h" +#include "mail-index-private.h" +#include "mail-transaction-log-view-private.h" + +static struct mail_transaction_log *log; +static struct mail_transaction_log_view *view; +static bool clean_refcount0_files = FALSE; + +static void +test_transaction_log_file_add(uint32_t file_seq) +{ + struct mail_transaction_log_file **p, *file; + + file = i_new(struct mail_transaction_log_file, 1); + file->hdr.file_seq = file_seq; + file->hdr.hdr_size = file->sync_offset = sizeof(file->hdr); + file->hdr.prev_file_seq = file_seq - 1; + file->hdr.prev_file_offset = (uint32_t)-1; + file->log = log; + file->fd = -1; + file->buffer = buffer_create_dynamic(default_pool, 256); + file->buffer_offset = file->hdr.hdr_size; + + /* files must be sorted by file_seq */ + for (p = &log->files; *p != NULL; p = &(*p)->next) { + if ((*p)->hdr.file_seq > file->hdr.file_seq) { + file->next = *p; + break; + } + } + *p = file; + log->head = file; +} + +void mail_index_set_error(struct mail_index *index ATTR_UNUSED, + const char *fmt ATTR_UNUSED, ...) +{ +} + +void mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file ATTR_UNUSED, + const char *fmt ATTR_UNUSED, ...) +{ +} + +void mail_transaction_logs_clean(struct mail_transaction_log *log ATTR_UNUSED) +{ +} + +int mail_transaction_log_find_file(struct mail_transaction_log *log, + uint32_t file_seq, bool nfs_flush ATTR_UNUSED, + struct mail_transaction_log_file **file_r, + const char **reason_r) +{ + struct mail_transaction_log_file *file, *next; + + for (file = log->files; file != NULL; file = next) { + next = file->next; + if (file->hdr.file_seq == file_seq) { + *file_r = file; + return 1; + } + /* refcount=0 files at the beginning of the list may be freed */ + if (file->refcount == 0 && file == log->files && + clean_refcount0_files) + log->files = next; + } + if (clean_refcount0_files && file_seq == 4) { + /* "clean refcount=0 files" test autocreates this file */ + test_transaction_log_file_add(4); + *file_r = log->head; + return 1; + } + *reason_r = "not found"; + return 0; +} + +int mail_transaction_log_file_map(struct mail_transaction_log_file *file ATTR_UNUSED, + uoff_t start_offset ATTR_UNUSED, uoff_t end_offset ATTR_UNUSED, + const char **reason_r ATTR_UNUSED) +{ + return 1; +} + +int mail_transaction_log_file_get_highest_modseq_at( + struct mail_transaction_log_file *file ATTR_UNUSED, + uoff_t offset ATTR_UNUSED, uint64_t *highest_modseq_r, + const char **error_r ATTR_UNUSED) +{ + *highest_modseq_r = 0; + return 1; +} + +void mail_transaction_update_modseq(const struct mail_transaction_header *hdr ATTR_UNUSED, + const void *data ATTR_UNUSED, + uint64_t *cur_modseq, + unsigned int version ATTR_UNUSED) +{ + *cur_modseq += 1; +} + +static bool view_is_file_refed(uint32_t file_seq) +{ + struct mail_transaction_log_file *const *files; + unsigned int i, count; + bool ret = FALSE; + + files = array_get(&view->file_refs, &count); + for (i = 0; i < count; i++) { + if (files[i]->hdr.file_seq == file_seq) { + i_assert(!ret); /* could be a test too.. */ + ret = TRUE; + } + } + return ret; +} + +static size_t +add_append_record(struct mail_transaction_log_file *file, + const struct mail_index_record *rec) +{ + struct mail_transaction_header hdr; + size_t size; + + i_zero(&hdr); + hdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL; + hdr.size = mail_index_uint32_to_offset(sizeof(hdr) + sizeof(*rec)); + + buffer_append(file->buffer, &hdr, sizeof(hdr)); + buffer_append(file->buffer, rec, sizeof(*rec)); + + size = sizeof(hdr) + sizeof(*rec); + file->sync_offset += size; + return size; +} + +static void test_mail_transaction_log_view(void) +{ + const struct mail_transaction_header *hdr; + const struct mail_index_record *rec; + struct mail_index_record append_rec; + const void *data; + void *oldfile; + uint32_t seq; + uoff_t offset, last_log_size; + const char *reason; + bool reset; + + test_begin("init"); + log = i_new(struct mail_transaction_log, 1); + log->index = i_new(struct mail_index, 1); + log->index->log = log; + log->index->log_sync_locked = TRUE; + test_transaction_log_file_add(1); + test_transaction_log_file_add(2); + test_transaction_log_file_add(3); + + /* add an append record to the 3rd log file */ + i_zero(&append_rec); + append_rec.uid = 1; + + last_log_size = sizeof(struct mail_transaction_log_header) + + add_append_record(log->head, &append_rec); + + view = mail_transaction_log_view_open(log); + i_assert(view != NULL); + test_assert(log->views == view && + !view_is_file_refed(1) && !view_is_file_refed(2) && + view_is_file_refed(3)); + test_end(); + + /* we have files 1-3 opened */ + test_begin("set all"); + test_assert(mail_transaction_log_view_set(view, 0, 0, (uint32_t)-1, UOFF_T_MAX, &reset, &reason) == 1 && + reset && view_is_file_refed(1) && view_is_file_refed(2) && + view_is_file_refed(3) && + !mail_transaction_log_view_is_corrupted(view)); + mail_transaction_log_view_get_prev_pos(view, &seq, &offset); + test_assert(seq == 1 && offset == sizeof(struct mail_transaction_log_header)); + test_assert(mail_transaction_log_view_next(view, &hdr, &data) == 1); + test_assert(hdr->type == (MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL)); + rec = data; + test_assert(memcmp(rec, &append_rec, sizeof(*rec)) == 0); + test_assert(mail_transaction_log_view_next(view, &hdr, &data) == 0); + test_assert(mail_transaction_log_view_is_last(view)); + mail_transaction_log_view_get_prev_pos(view, &seq, &offset); + test_assert(seq == 3 && offset == last_log_size); + test_end(); + + test_begin("set first"); + test_assert(mail_transaction_log_view_set(view, 0, 0, 0, 0, &reset, &reason) == 1); + mail_transaction_log_view_get_prev_pos(view, &seq, &offset); + test_assert(seq == 1 && offset == sizeof(struct mail_transaction_log_header)); + test_assert(mail_transaction_log_view_next(view, &hdr, &data) == 0); + mail_transaction_log_view_get_prev_pos(view, &seq, &offset); + test_assert(seq == 1 && offset == sizeof(struct mail_transaction_log_header)); + test_end(); + + test_begin("set end"); + test_assert(mail_transaction_log_view_set(view, 3, last_log_size, (uint32_t)-1, UOFF_T_MAX, &reset, &reason) == 1); + mail_transaction_log_view_get_prev_pos(view, &seq, &offset); + test_assert(seq == 3 && offset == last_log_size); + test_assert(mail_transaction_log_view_next(view, &hdr, &data) == 0); + mail_transaction_log_view_get_prev_pos(view, &seq, &offset); + test_assert(seq == 3 && offset == last_log_size); + test_end(); + + test_begin("log clear"); + mail_transaction_log_view_clear(view, 2); + test_assert(!view_is_file_refed(1) && view_is_file_refed(2) && + view_is_file_refed(3)); + oldfile = log->files; + buffer_free(&log->files->buffer); + log->files = log->files->next; + i_free(oldfile); + test_assert(log->files->hdr.file_seq == 2); + test_end(); + + /* --- first file has been removed --- */ + + test_begin("set 2-3"); + test_assert(mail_transaction_log_view_set(view, 2, 0, (uint32_t)-1, UOFF_T_MAX, &reset, &reason) == 1); + test_end(); + + test_begin("missing log handing"); + test_assert(mail_transaction_log_view_set(view, 0, 0, (uint32_t)-1, UOFF_T_MAX, &reset, &reason) == 0); + test_end(); + + test_begin("closed log handling"); + view->log = NULL; + test_assert(mail_transaction_log_view_set(view, 0, 0, (uint32_t)-1, UOFF_T_MAX, &reset, &reason) == 0); + view->log = log; + test_end(); + + test_begin("clean refcount=0 files"); + oldfile = log->files; + /* clear all references */ + mail_transaction_log_view_clear(view, 0); + clean_refcount0_files = TRUE; + /* create a new file during mail_transaction_log_view_set(), which + triggers freeing any unreferenced files. */ + test_assert(mail_transaction_log_view_set(view, 2, 0, 4, UOFF_T_MAX, &reset, &reason) == 1); + clean_refcount0_files = FALSE; + log->files = oldfile; + test_end(); + + mail_transaction_log_view_close(&view); + i_free(log->index); + while (log->files != NULL) { + oldfile = log->files; + buffer_free(&log->files->buffer); + log->files = log->files->next; + i_free(oldfile); + } + i_free(log); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_mail_transaction_log_view, + NULL + }; + return test_run(test_functions); +} |