Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

168 files changed, 64231 insertions, 0 deletions

diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am
new file mode 100644
index 0000000..23461e2
--- /dev/null
+++ b/src/lib-storage/index/Makefile.am
@@ -0,0 +1,58 @@
+SUBDIRS = maildir mbox dbox-common dbox-multi dbox-single imapc pop3c raw shared
+
+noinst_LTLIBRARIES = libstorage_index.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test \
+	-I$(top_srcdir)/src/lib-dict \
+	-I$(top_srcdir)/src/lib-fs \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage
+
+libstorage_index_la_SOURCES = \
+	istream-mail.c \
+	index-attachment.c \
+	index-attribute.c \
+	index-mail.c \
+	index-mail-binary.c \
+	index-mail-headers.c \
+	index-mailbox-size.c \
+	index-pop3-uidl.c \
+	index-rebuild.c \
+	index-search.c \
+	index-search-mime.c \
+	index-search-result.c \
+	index-sort.c \
+	index-sort-string.c \
+	index-status.c \
+	index-storage.c \
+	index-sync.c \
+	index-sync-changes.c \
+	index-sync-pvt.c \
+	index-sync-search.c \
+	index-thread.c \
+	index-thread-finish.c \
+	index-thread-links.c \
+	index-transaction.c
+
+headers = \
+	istream-mail.h \
+	index-attachment.h \
+	index-mail.h \
+	index-mailbox-size.h \
+	index-pop3-uidl.h \
+	index-rebuild.h \
+	index-search-private.h \
+	index-search-result.h \
+	index-sort.h \
+	index-sort-private.h \
+	index-storage.h \
+	index-sync-changes.h \
+	index-sync-private.h \
+	index-thread-private.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/Makefile.in b/src/lib-storage/index/Makefile.in
new file mode 100644
index 0000000..3069140
--- /dev/null
+++ b/src/lib-storage/index/Makefile.in
@@ -0,0 +1,1061 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_index_la_LIBADD =
+am_libstorage_index_la_OBJECTS = istream-mail.lo index-attachment.lo \
+	index-attribute.lo index-mail.lo index-mail-binary.lo \
+	index-mail-headers.lo index-mailbox-size.lo index-pop3-uidl.lo \
+	index-rebuild.lo index-search.lo index-search-mime.lo \
+	index-search-result.lo index-sort.lo index-sort-string.lo \
+	index-status.lo index-storage.lo index-sync.lo \
+	index-sync-changes.lo index-sync-pvt.lo index-sync-search.lo \
+	index-thread.lo index-thread-finish.lo index-thread-links.lo \
+	index-transaction.lo
+libstorage_index_la_OBJECTS = $(am_libstorage_index_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/index-attachment.Plo \
+	./$(DEPDIR)/index-attribute.Plo \
+	./$(DEPDIR)/index-mail-binary.Plo \
+	./$(DEPDIR)/index-mail-headers.Plo ./$(DEPDIR)/index-mail.Plo \
+	./$(DEPDIR)/index-mailbox-size.Plo \
+	./$(DEPDIR)/index-pop3-uidl.Plo ./$(DEPDIR)/index-rebuild.Plo \
+	./$(DEPDIR)/index-search-mime.Plo \
+	./$(DEPDIR)/index-search-result.Plo \
+	./$(DEPDIR)/index-search.Plo ./$(DEPDIR)/index-sort-string.Plo \
+	./$(DEPDIR)/index-sort.Plo ./$(DEPDIR)/index-status.Plo \
+	./$(DEPDIR)/index-storage.Plo \
+	./$(DEPDIR)/index-sync-changes.Plo \
+	./$(DEPDIR)/index-sync-pvt.Plo \
+	./$(DEPDIR)/index-sync-search.Plo ./$(DEPDIR)/index-sync.Plo \
+	./$(DEPDIR)/index-thread-finish.Plo \
+	./$(DEPDIR)/index-thread-links.Plo \
+	./$(DEPDIR)/index-thread.Plo ./$(DEPDIR)/index-transaction.Plo \
+	./$(DEPDIR)/istream-mail.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_index_la_SOURCES)
+DIST_SOURCES = $(libstorage_index_la_SOURCES)
+RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \
+	ctags-recursive dvi-recursive html-recursive info-recursive \
+	install-data-recursive install-dvi-recursive \
+	install-exec-recursive install-html-recursive \
+	install-info-recursive install-pdf-recursive \
+	install-ps-recursive install-recursive installcheck-recursive \
+	installdirs-recursive pdf-recursive ps-recursive \
+	tags-recursive uninstall-recursive
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive	\
+  distclean-recursive maintainer-clean-recursive
+am__recursive_targets = \
+  $(RECURSIVE_TARGETS) \
+  $(RECURSIVE_CLEAN_TARGETS) \
+  $(am__extra_recursive_targets)
+AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \
+	distdir distdir-am
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+DIST_SUBDIRS = $(SUBDIRS)
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+am__relativize = \
+  dir0=`pwd`; \
+  sed_first='s,^\([^/]*\)/.*$$,\1,'; \
+  sed_rest='s,^[^/]*/*,,'; \
+  sed_last='s,^.*/\([^/]*\)$$,\1,'; \
+  sed_butlast='s,/*[^/]*$$,,'; \
+  while test -n "$$dir1"; do \
+    first=`echo "$$dir1" | sed -e "$$sed_first"`; \
+    if test "$$first" != "."; then \
+      if test "$$first" = ".."; then \
+        dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \
+        dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \
+      else \
+        first2=`echo "$$dir2" | sed -e "$$sed_first"`; \
+        if test "$$first2" = "$$first"; then \
+          dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \
+        else \
+          dir2="../$$dir2"; \
+        fi; \
+        dir0="$$dir0"/"$$first"; \
+      fi; \
+    fi; \
+    dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \
+  done; \
+  reldir="$$dir2"
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+SUBDIRS = maildir mbox dbox-common dbox-multi dbox-single imapc pop3c raw shared
+noinst_LTLIBRARIES = libstorage_index.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test \
+	-I$(top_srcdir)/src/lib-dict \
+	-I$(top_srcdir)/src/lib-fs \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage
+
+libstorage_index_la_SOURCES = \
+	istream-mail.c \
+	index-attachment.c \
+	index-attribute.c \
+	index-mail.c \
+	index-mail-binary.c \
+	index-mail-headers.c \
+	index-mailbox-size.c \
+	index-pop3-uidl.c \
+	index-rebuild.c \
+	index-search.c \
+	index-search-mime.c \
+	index-search-result.c \
+	index-sort.c \
+	index-sort-string.c \
+	index-status.c \
+	index-storage.c \
+	index-sync.c \
+	index-sync-changes.c \
+	index-sync-pvt.c \
+	index-sync-search.c \
+	index-thread.c \
+	index-thread-finish.c \
+	index-thread-links.c \
+	index-transaction.c
+
+headers = \
+	istream-mail.h \
+	index-attachment.h \
+	index-mail.h \
+	index-mailbox-size.h \
+	index-pop3-uidl.h \
+	index-rebuild.h \
+	index-search-private.h \
+	index-search-result.h \
+	index-sort.h \
+	index-sort-private.h \
+	index-storage.h \
+	index-sync-changes.h \
+	index-sync-private.h \
+	index-thread-private.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-recursive
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_index.la: $(libstorage_index_la_OBJECTS) $(libstorage_index_la_DEPENDENCIES) $(EXTRA_libstorage_index_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_index_la_OBJECTS) $(libstorage_index_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-attachment.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-attribute.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mail-binary.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mail-headers.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-mailbox-size.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-pop3-uidl.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-rebuild.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-search-mime.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-search-result.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-search.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sort-string.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sort.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-status.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync-changes.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync-pvt.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync-search.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-sync.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-thread-finish.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-thread-links.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-thread.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/index-transaction.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-mail.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+# This directory's subdirectories are mostly independent; you can cd
+# into them and run 'make' without going through this Makefile.
+# To change the values of 'make' variables: instead of editing Makefiles,
+# (1) if the variable is set in 'config.status', edit 'config.status'
+#     (which will cause the Makefiles to be regenerated when you run 'make');
+# (2) otherwise, pass the desired values on the 'make' command line.
+$(am__recursive_targets):
+	@fail=; \
+	if $(am__make_keepgoing); then \
+	  failcom='fail=yes'; \
+	else \
+	  failcom='exit 1'; \
+	fi; \
+	dot_seen=no; \
+	target=`echo $@ | sed s/-recursive//`; \
+	case "$@" in \
+	  distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \
+	  *) list='$(SUBDIRS)' ;; \
+	esac; \
+	for subdir in $$list; do \
+	  echo "Making $$target in $$subdir"; \
+	  if test "$$subdir" = "."; then \
+	    dot_seen=yes; \
+	    local_target="$$target-am"; \
+	  else \
+	    local_target="$$target"; \
+	  fi; \
+	  ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \
+	  || eval $$failcom; \
+	done; \
+	if test "$$dot_seen" = "no"; then \
+	  $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \
+	fi; test -z "$$fail"
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-recursive
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \
+	  include_option=--etags-include; \
+	  empty_fix=.; \
+	else \
+	  include_option=--include; \
+	  empty_fix=; \
+	fi; \
+	list='$(SUBDIRS)'; for subdir in $$list; do \
+	  if test "$$subdir" = .; then :; else \
+	    test ! -f $$subdir/TAGS || \
+	      set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \
+	  fi; \
+	done; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-recursive
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-recursive
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+	@list='$(DIST_SUBDIRS)'; for subdir in $$list; do \
+	  if test "$$subdir" = .; then :; else \
+	    $(am__make_dryrun) \
+	      || test -d "$(distdir)/$$subdir" \
+	      || $(MKDIR_P) "$(distdir)/$$subdir" \
+	      || exit 1; \
+	    dir1=$$subdir; dir2="$(distdir)/$$subdir"; \
+	    $(am__relativize); \
+	    new_distdir=$$reldir; \
+	    dir1=$$subdir; dir2="$(top_distdir)"; \
+	    $(am__relativize); \
+	    new_top_distdir=$$reldir; \
+	    echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \
+	    echo "     am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \
+	    ($(am__cd) $$subdir && \
+	      $(MAKE) $(AM_MAKEFLAGS) \
+	        top_distdir="$$new_top_distdir" \
+	        distdir="$$new_distdir" \
+		am__remove_distdir=: \
+		am__skip_length_check=: \
+		am__skip_mode_fix=: \
+	        distdir) \
+	      || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-recursive
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs: installdirs-recursive
+installdirs-am:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-recursive
+install-exec: install-exec-recursive
+install-data: install-data-recursive
+uninstall: uninstall-recursive
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-recursive
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-recursive
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-recursive
+		-rm -f ./$(DEPDIR)/index-attachment.Plo
+	-rm -f ./$(DEPDIR)/index-attribute.Plo
+	-rm -f ./$(DEPDIR)/index-mail-binary.Plo
+	-rm -f ./$(DEPDIR)/index-mail-headers.Plo
+	-rm -f ./$(DEPDIR)/index-mail.Plo
+	-rm -f ./$(DEPDIR)/index-mailbox-size.Plo
+	-rm -f ./$(DEPDIR)/index-pop3-uidl.Plo
+	-rm -f ./$(DEPDIR)/index-rebuild.Plo
+	-rm -f ./$(DEPDIR)/index-search-mime.Plo
+	-rm -f ./$(DEPDIR)/index-search-result.Plo
+	-rm -f ./$(DEPDIR)/index-search.Plo
+	-rm -f ./$(DEPDIR)/index-sort-string.Plo
+	-rm -f ./$(DEPDIR)/index-sort.Plo
+	-rm -f ./$(DEPDIR)/index-status.Plo
+	-rm -f ./$(DEPDIR)/index-storage.Plo
+	-rm -f ./$(DEPDIR)/index-sync-changes.Plo
+	-rm -f ./$(DEPDIR)/index-sync-pvt.Plo
+	-rm -f ./$(DEPDIR)/index-sync-search.Plo
+	-rm -f ./$(DEPDIR)/index-sync.Plo
+	-rm -f ./$(DEPDIR)/index-thread-finish.Plo
+	-rm -f ./$(DEPDIR)/index-thread-links.Plo
+	-rm -f ./$(DEPDIR)/index-thread.Plo
+	-rm -f ./$(DEPDIR)/index-transaction.Plo
+	-rm -f ./$(DEPDIR)/istream-mail.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-recursive
+
+dvi-am:
+
+html: html-recursive
+
+html-am:
+
+info: info-recursive
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-recursive
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-recursive
+
+install-html-am:
+
+install-info: install-info-recursive
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-recursive
+
+install-pdf-am:
+
+install-ps: install-ps-recursive
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-recursive
+		-rm -f ./$(DEPDIR)/index-attachment.Plo
+	-rm -f ./$(DEPDIR)/index-attribute.Plo
+	-rm -f ./$(DEPDIR)/index-mail-binary.Plo
+	-rm -f ./$(DEPDIR)/index-mail-headers.Plo
+	-rm -f ./$(DEPDIR)/index-mail.Plo
+	-rm -f ./$(DEPDIR)/index-mailbox-size.Plo
+	-rm -f ./$(DEPDIR)/index-pop3-uidl.Plo
+	-rm -f ./$(DEPDIR)/index-rebuild.Plo
+	-rm -f ./$(DEPDIR)/index-search-mime.Plo
+	-rm -f ./$(DEPDIR)/index-search-result.Plo
+	-rm -f ./$(DEPDIR)/index-search.Plo
+	-rm -f ./$(DEPDIR)/index-sort-string.Plo
+	-rm -f ./$(DEPDIR)/index-sort.Plo
+	-rm -f ./$(DEPDIR)/index-status.Plo
+	-rm -f ./$(DEPDIR)/index-storage.Plo
+	-rm -f ./$(DEPDIR)/index-sync-changes.Plo
+	-rm -f ./$(DEPDIR)/index-sync-pvt.Plo
+	-rm -f ./$(DEPDIR)/index-sync-search.Plo
+	-rm -f ./$(DEPDIR)/index-sync.Plo
+	-rm -f ./$(DEPDIR)/index-thread-finish.Plo
+	-rm -f ./$(DEPDIR)/index-thread-links.Plo
+	-rm -f ./$(DEPDIR)/index-thread.Plo
+	-rm -f ./$(DEPDIR)/index-transaction.Plo
+	-rm -f ./$(DEPDIR)/istream-mail.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-recursive
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-recursive
+
+pdf-am:
+
+ps: ps-recursive
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: $(am__recursive_targets) install-am install-strip
+
+.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am \
+	am--depfiles check check-am clean clean-generic clean-libtool \
+	clean-noinstLTLIBRARIES cscopelist-am ctags ctags-am distclean \
+	distclean-compile distclean-generic distclean-libtool \
+	distclean-tags distdir dvi dvi-am html html-am info info-am \
+	install install-am install-data install-data-am install-dvi \
+	install-dvi-am install-exec install-exec-am install-html \
+	install-html-am install-info install-info-am install-man \
+	install-pdf install-pdf-am install-pkginc_libHEADERS \
+	install-ps install-ps-am install-strip installcheck \
+	installcheck-am installdirs installdirs-am maintainer-clean \
+	maintainer-clean-generic mostlyclean mostlyclean-compile \
+	mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+	tags tags-am uninstall uninstall-am \
+	uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/dbox-common/Makefile.am b/src/lib-storage/index/dbox-common/Makefile.am
new file mode 100644
index 0000000..ec3d511
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/Makefile.am
@@ -0,0 +1,29 @@
+noinst_LTLIBRARIES = libstorage_dbox_common.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-fs \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_dbox_common_la_SOURCES = \
+	dbox-attachment.c \
+	dbox-file.c \
+	dbox-file-fix.c \
+	dbox-mail.c \
+	dbox-save.c \
+	dbox-storage.c
+
+headers = \
+	dbox-attachment.h \
+	dbox-file.h \
+	dbox-mail.h \
+	dbox-save.h \
+	dbox-storage.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/dbox-common/Makefile.in b/src/lib-storage/index/dbox-common/Makefile.in
new file mode 100644
index 0000000..60aaf66
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/Makefile.in
@@ -0,0 +1,843 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/dbox-common
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_dbox_common_la_LIBADD =
+am_libstorage_dbox_common_la_OBJECTS = dbox-attachment.lo dbox-file.lo \
+	dbox-file-fix.lo dbox-mail.lo dbox-save.lo dbox-storage.lo
+libstorage_dbox_common_la_OBJECTS =  \
+	$(am_libstorage_dbox_common_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/dbox-attachment.Plo \
+	./$(DEPDIR)/dbox-file-fix.Plo ./$(DEPDIR)/dbox-file.Plo \
+	./$(DEPDIR)/dbox-mail.Plo ./$(DEPDIR)/dbox-save.Plo \
+	./$(DEPDIR)/dbox-storage.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_dbox_common_la_SOURCES)
+DIST_SOURCES = $(libstorage_dbox_common_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_dbox_common.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-fs \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_dbox_common_la_SOURCES = \
+	dbox-attachment.c \
+	dbox-file.c \
+	dbox-file-fix.c \
+	dbox-mail.c \
+	dbox-save.c \
+	dbox-storage.c
+
+headers = \
+	dbox-attachment.h \
+	dbox-file.h \
+	dbox-mail.h \
+	dbox-save.h \
+	dbox-storage.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/dbox-common/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/dbox-common/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_dbox_common.la: $(libstorage_dbox_common_la_OBJECTS) $(libstorage_dbox_common_la_DEPENDENCIES) $(EXTRA_libstorage_dbox_common_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_dbox_common_la_OBJECTS) $(libstorage_dbox_common_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-attachment.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-file-fix.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-file.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-save.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbox-storage.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/dbox-attachment.Plo
+	-rm -f ./$(DEPDIR)/dbox-file-fix.Plo
+	-rm -f ./$(DEPDIR)/dbox-file.Plo
+	-rm -f ./$(DEPDIR)/dbox-mail.Plo
+	-rm -f ./$(DEPDIR)/dbox-save.Plo
+	-rm -f ./$(DEPDIR)/dbox-storage.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/dbox-attachment.Plo
+	-rm -f ./$(DEPDIR)/dbox-file-fix.Plo
+	-rm -f ./$(DEPDIR)/dbox-file.Plo
+	-rm -f ./$(DEPDIR)/dbox-mail.Plo
+	-rm -f ./$(DEPDIR)/dbox-save.Plo
+	-rm -f ./$(DEPDIR)/dbox-storage.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/dbox-common/dbox-attachment.c b/src/lib-storage/index/dbox-common/dbox-attachment.c
new file mode 100644
index 0000000..cfc3f62
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-attachment.c
@@ -0,0 +1,77 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "str.h"
+#include "dbox-file.h"
+#include "dbox-save.h"
+#include "dbox-attachment.h"
+
+void dbox_attachment_save_write_metadata(struct mail_save_context *ctx,
+					 string_t *str)
+{
+	const ARRAY_TYPE(mail_attachment_extref) *extrefs;
+
+	extrefs = index_attachment_save_get_extrefs(ctx);
+	if (extrefs == NULL || array_count(extrefs) == 0)
+		return;
+
+	str_append_c(str, DBOX_METADATA_EXT_REF);
+	index_attachment_append_extrefs(str, extrefs);
+	str_append_c(str, '\n');
+}
+
+static int
+dbox_attachment_file_get_stream_from(struct dbox_file *file,
+				     const char *ext_refs,
+				     struct istream **stream,
+				     const char **error_r)
+{
+	const char *path_suffix;
+	uoff_t msg_size;
+
+	if (file->storage->attachment_dir == NULL) {
+		mail_storage_set_critical(&file->storage->storage,
+			"%s contains references to external attachments, "
+			"but mail_attachment_dir is unset", file->cur_path);
+		return -1;
+	}
+
+	msg_size = dbox_file_get_plaintext_size(file);
+	path_suffix = file->storage->v.get_attachment_path_suffix(file);
+	if (index_attachment_stream_get(file->storage->attachment_fs,
+					file->storage->attachment_dir,
+					path_suffix, stream, msg_size,
+					ext_refs, error_r) < 0)
+		return 0;
+	return 1;
+}
+
+int dbox_attachment_file_get_stream(struct dbox_file *file,
+				    struct istream **stream)
+{
+	const char *ext_refs, *error;
+	int ret;
+
+	/* need to read metadata in case there are external references */
+	if ((ret = dbox_file_metadata_read(file)) <= 0)
+		return ret;
+
+	i_stream_seek(file->input, file->cur_offset + file->msg_header_size);
+
+	ext_refs = dbox_file_metadata_get(file, DBOX_METADATA_EXT_REF);
+	if (ext_refs == NULL)
+		return 1;
+
+	/* we have external references. */
+	T_BEGIN {
+		ret = dbox_attachment_file_get_stream_from(file, ext_refs,
+							   stream, &error);
+		if (ret == 0) {
+			dbox_file_set_corrupted(file,
+				"Corrupted ext-refs metadata %s: %s",
+				ext_refs, error);
+		}
+	} T_END;
+	return ret;
+}
diff --git a/src/lib-storage/index/dbox-common/dbox-attachment.h b/src/lib-storage/index/dbox-common/dbox-attachment.h
new file mode 100644
index 0000000..a90ba54
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-attachment.h
@@ -0,0 +1,16 @@
+#ifndef DBOX_ATTACHMENT_H
+#define DBOX_ATTACHMENT_H
+
+#include "index-attachment.h"
+
+struct dbox_file;
+
+void dbox_attachment_save_write_metadata(struct mail_save_context *ctx,
+					 string_t *str);
+
+/* Build a single message body stream out of the current message and all of its
+   attachments. */
+int dbox_attachment_file_get_stream(struct dbox_file *file,
+				    struct istream **stream);
+
+#endif
diff --git a/src/lib-storage/index/dbox-common/dbox-file-fix.c b/src/lib-storage/index/dbox-common/dbox-file-fix.c
new file mode 100644
index 0000000..1c44ca4
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-file-fix.c
@@ -0,0 +1,519 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "hex-dec.h"
+#include "istream.h"
+#include "ostream.h"
+#include "message-size.h"
+#include "dbox-storage.h"
+#include "dbox-file.h"
+
+#include <stdio.h>
+
+#define DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX ".broken"
+
+static int
+dbox_file_match_pre_magic(struct istream *input,
+			  uoff_t *pre_offset, size_t *need_bytes)
+{
+	const struct dbox_message_header *hdr;
+	const unsigned char *data;
+	size_t size;
+	uoff_t offset = input->v_offset;
+	bool have_lf = FALSE;
+
+	data = i_stream_get_data(input, &size);
+	if (data[0] == '\n') {
+		data++; size--; offset++;
+		have_lf = TRUE;
+	}
+	i_assert(data[0] == DBOX_MAGIC_PRE[0]);
+	if (size < sizeof(*hdr)) {
+		*need_bytes = sizeof(*hdr) + (have_lf ? 1 : 0);
+		return -1;
+	}
+	hdr = (const void *)data;
+	if (memcmp(hdr->magic_pre, DBOX_MAGIC_PRE, strlen(DBOX_MAGIC_PRE)) != 0)
+		return 0;
+	if (hdr->type != DBOX_MESSAGE_TYPE_NORMAL)
+		return 0;
+	if (hdr->space1 != ' ' || hdr->space2 != ' ')
+		return 0;
+	if (hex2dec(hdr->message_size_hex, sizeof(hdr->message_size_hex)) == 0 &&
+	    memcmp(hdr->message_size_hex, "0000000000000000", sizeof(hdr->message_size_hex)) != 0)
+		return 0;
+
+	*pre_offset = offset;
+	return 1;
+}
+
+static bool memchr_nocontrol(const unsigned char *data, char chr,
+			     unsigned int len, const unsigned char **pos_r)
+{
+	unsigned int i;
+
+	for (i = 0; i < len; i++) {
+		if (data[i] == chr) {
+			*pos_r = data+i;
+			return TRUE;
+		}
+		if (data[i] < ' ')
+			return FALSE;
+	}
+	*pos_r = NULL;
+	return TRUE;
+}
+
+static int
+dbox_file_match_post_magic(struct istream *input, bool input_full,
+			   size_t *need_bytes)
+{
+	const unsigned char *data, *p;
+	size_t i, size;
+	bool allow_control;
+
+	data = i_stream_get_data(input, &size);
+	if (size < strlen(DBOX_MAGIC_POST)) {
+		*need_bytes = strlen(DBOX_MAGIC_POST);
+		return -1;
+	}
+	if (memcmp(data, DBOX_MAGIC_POST, strlen(DBOX_MAGIC_POST)) != 0)
+		return 0;
+
+	/* see if the metadata block looks valid */
+	for (i = strlen(DBOX_MAGIC_POST); i < size; ) {
+		switch (data[i]) {
+		case '\n':
+			return 1;
+		case DBOX_METADATA_GUID:
+		case DBOX_METADATA_POP3_UIDL:
+		case DBOX_METADATA_ORIG_MAILBOX:
+		case DBOX_METADATA_OLDV1_KEYWORDS:
+			/* these could contain anything */
+			allow_control = TRUE;
+			break;
+		case DBOX_METADATA_POP3_ORDER:
+		case DBOX_METADATA_RECEIVED_TIME:
+		case DBOX_METADATA_PHYSICAL_SIZE:
+		case DBOX_METADATA_VIRTUAL_SIZE:
+		case DBOX_METADATA_EXT_REF:
+		case DBOX_METADATA_OLDV1_EXPUNGED:
+		case DBOX_METADATA_OLDV1_FLAGS:
+		case DBOX_METADATA_OLDV1_SAVE_TIME:
+		case DBOX_METADATA_OLDV1_SPACE:
+			/* no control chars */
+			allow_control = FALSE;
+			break;
+		default:
+			if (data[i] < 'A' || data[i] > 'Z')
+				return 0;
+			/* unknown */
+			allow_control = TRUE;
+			break;
+		}
+		if (allow_control) {
+			p = memchr(data+i, '\n', size-i);
+		} else {
+			if (!memchr_nocontrol(data+i, '\n', size-i, &p))
+				return 0;
+		}
+		if (p == NULL) {
+			/* LF not found - try to find the end-of-metadata LF */
+			if (input_full) {
+				/* can't look any further - assume it's ok */
+				return 1;
+			}
+			*need_bytes = size+1;
+			return -1;
+		}
+		i = p - data+1;
+	}
+	*need_bytes = size+1;
+	return -1;
+}
+
+static int
+dbox_file_find_next_magic(struct dbox_file *file, uoff_t *offset_r, bool *pre_r)
+{
+	/* We're scanning message bodies here, trying to find the beginning of
+	   the next message. Although our magic strings are very unlikely to
+	   be found in regular emails, they are much more likely when emails
+	   are stored compressed.. So try to be sure we find the correct
+	   magic markers. */
+
+	struct istream *input = file->input;
+	uoff_t orig_offset, pre_offset, post_offset, prev_offset;
+	const unsigned char *data, *magic;
+	size_t size, need_bytes, prev_need_bytes;
+	int ret, match;
+
+	*pre_r = FALSE;
+
+	orig_offset = prev_offset = input->v_offset;
+	need_bytes = strlen(DBOX_MAGIC_POST); prev_need_bytes = 0;
+	while ((ret = i_stream_read_bytes(input, &data, &size, need_bytes)) > 0 ||
+	       ret == -2) {
+		/* search for the beginning of a potential pre/post magic */
+		i_assert(size > 1);
+		i_assert(prev_offset != input->v_offset ||
+			 need_bytes > prev_need_bytes);
+		prev_offset = input->v_offset;
+		prev_need_bytes = need_bytes;
+
+		magic = memchr(data, DBOX_MAGIC_PRE[0], size);
+		if (magic == NULL) {
+			i_stream_skip(input, size-1);
+			need_bytes = strlen(DBOX_MAGIC_POST);
+			continue;
+		}
+		if (magic == data && input->v_offset == orig_offset) {
+			/* beginning of the file */
+		} else if (magic != data && magic[-1] == '\n') {
+			/* PRE/POST block? leave \n */
+			i_stream_skip(input, magic-data-1);
+		} else {
+			i_stream_skip(input, magic-data+1);
+			need_bytes = strlen(DBOX_MAGIC_POST);
+			continue;
+		}
+
+		pre_offset = UOFF_T_MAX;
+		match = dbox_file_match_pre_magic(input, &pre_offset, &need_bytes);
+		if (match < 0) {
+			/* more data needed */
+			if (ret == -2) {
+				i_stream_skip(input, 2);
+				need_bytes = strlen(DBOX_MAGIC_POST);
+			}
+			continue;
+		}
+		if (match > 0)
+			*pre_r = TRUE;
+
+		match = dbox_file_match_post_magic(input, ret == -2, &need_bytes);
+		if (match < 0) {
+			/* more data needed */
+			if (ret == -2) {
+				i_stream_skip(input, 2);
+				need_bytes = strlen(DBOX_MAGIC_POST);
+			}
+			continue;
+		}
+		if (match > 0) {
+			post_offset = input->v_offset;
+			if (pre_offset == UOFF_T_MAX ||
+			    post_offset < pre_offset) {
+				pre_offset = post_offset;
+				*pre_r = FALSE;
+			}
+		}
+
+		if (pre_offset != UOFF_T_MAX) {
+			*offset_r = pre_offset;
+			ret = 1;
+			break;
+		}
+		i_stream_skip(input, size-1);
+	}
+	if (ret <= 0) {
+		i_assert(ret == -1);
+		if (input->stream_errno != 0)
+			dbox_file_set_syscall_error(file, "read()");
+		else {
+			ret = 0;
+			*offset_r = input->v_offset;
+		} 
+	}
+	i_stream_seek(input, orig_offset);
+	return ret <= 0 ? ret : 1;
+}
+
+static int
+stream_copy(struct dbox_file *file, struct ostream *output,
+	    const char *out_path, uoff_t count)
+{
+	struct istream *input;
+	int ret = 0;
+
+	input = i_stream_create_limit(file->input, count);
+	o_stream_nsend_istream(output, input);
+
+	if (input->stream_errno != 0) {
+		mail_storage_set_critical(&file->storage->storage,
+			"read(%s) failed: %s", file->cur_path,
+			i_stream_get_error(input));
+		ret = -1;
+	} else if (o_stream_flush(output) < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+			"write(%s) failed: %s", out_path,
+			o_stream_get_error(output));
+		ret = -1;
+	} else if (input->v_offset != count) {
+		mail_storage_set_critical(&file->storage->storage,
+			"o_stream_send_istream(%s) copied only %"
+			PRIuUOFF_T" of %"PRIuUOFF_T" bytes",
+			out_path, input->v_offset, count);
+		ret = -1;
+	}
+	i_stream_unref(&input);
+	return ret;
+}
+
+static void dbox_file_skip_broken_header(struct dbox_file *file)
+{
+	const size_t magic_len = strlen(DBOX_MAGIC_PRE);
+	const unsigned char *data;
+	size_t i, size;
+
+	/* if there's LF close to our position, assume that the header ends
+	   there. */
+	data = i_stream_get_data(file->input, &size);
+	if (size > file->msg_header_size + 16)
+		size = file->msg_header_size + 16;
+	for (i = 0; i < size; i++) {
+		if (data[i] == '\n') {
+			i_stream_skip(file->input, i);
+			return;
+		}
+	}
+
+	/* skip at least the magic bytes if possible */
+	if (size > magic_len && memcmp(data, DBOX_MAGIC_PRE, magic_len) == 0)
+		i_stream_skip(file->input, magic_len);
+}
+
+static void
+dbox_file_copy_metadata(struct dbox_file *file, struct ostream *output,
+			bool *have_guid_r)
+{
+	const char *line;
+	uoff_t prev_offset = file->input->v_offset;
+
+	*have_guid_r = FALSE;
+	while ((line = i_stream_read_next_line(file->input)) != NULL) {
+		if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
+			/* end of metadata */
+			return;
+		}
+		if (*line < 32) {
+			/* broken - possibly a new pre-magic block */
+			i_stream_seek(file->input, prev_offset);
+			return;
+		}
+		if (*line == DBOX_METADATA_VIRTUAL_SIZE) {
+			/* it may be wrong - recreate it */
+			continue;
+		}
+		if (*line == DBOX_METADATA_GUID)
+			*have_guid_r = TRUE;
+		o_stream_nsend_str(output, line);
+		o_stream_nsend_str(output, "\n");
+	}
+}
+
+static int
+dbox_file_fix_write_stream(struct dbox_file *file, uoff_t start_offset,
+			   const char *temp_path, struct ostream *output)
+{
+	struct dbox_message_header msg_hdr;
+	uoff_t offset, msg_size, hdr_offset, body_offset;
+	bool pre, write_header, have_guid;
+	struct message_size body;
+	bool has_nuls;
+	struct istream *body_input;
+	guid_128_t guid_128;
+	int ret;
+
+	i_stream_seek(file->input, 0);
+	if (start_offset > 0) {
+		/* copy the valid data */
+		if (stream_copy(file, output, temp_path, start_offset) < 0)
+			return -1;
+	} else {
+		/* the file header is broken. recreate it */
+		if (dbox_file_header_write(file, output) < 0) {
+			dbox_file_set_syscall_error(file, "write()");
+			return -1;
+		}
+	}
+
+	while ((ret = dbox_file_find_next_magic(file, &offset, &pre)) > 0) {
+		msg_size = offset - file->input->v_offset;
+		if (msg_size < 256 && pre) {
+			/* probably some garbage or some broken headers.
+			   we most likely don't miss anything by skipping
+			   over this data. */
+			i_stream_skip(file->input, msg_size);
+			hdr_offset = file->input->v_offset;
+			ret = dbox_file_read_mail_header(file, &msg_size);
+			if (ret <= 0) {
+				if (ret < 0)
+					return -1;
+				dbox_file_skip_broken_header(file);
+				body_offset = file->input->v_offset;
+				msg_size = UOFF_T_MAX;
+			} else {
+				i_stream_skip(file->input,
+					      file->msg_header_size);
+				body_offset = file->input->v_offset;
+				i_stream_skip(file->input, msg_size);
+			}
+
+			ret = dbox_file_find_next_magic(file, &offset, &pre);
+			if (ret <= 0)
+				break;
+
+			if (!pre && msg_size == offset - body_offset) {
+				/* msg header ok, copy it */
+				i_stream_seek(file->input, hdr_offset);
+				if (stream_copy(file, output, temp_path,
+						file->msg_header_size) < 0)
+					return -1;
+				write_header = FALSE;
+			} else {
+				/* msg header is broken. write our own. */
+				i_stream_seek(file->input, body_offset);
+				if (msg_size != UOFF_T_MAX) {
+					/* previous magic find might have
+					   skipped too much. seek back and
+					   make sure */
+					ret = dbox_file_find_next_magic(file, &offset, &pre);
+					if (ret <= 0)
+						break;
+				}
+
+				write_header = TRUE;
+				msg_size = offset - body_offset;
+			}
+		} else {
+			/* treat this data as a separate message. */
+			write_header = TRUE;
+			body_offset = file->input->v_offset;
+		}
+		/* write msg header */
+		if (write_header) {
+			dbox_msg_header_fill(&msg_hdr, msg_size);
+			o_stream_nsend(output, &msg_hdr, sizeof(msg_hdr));
+		}
+		/* write msg body */
+		i_assert(file->input->v_offset == body_offset);
+		if (stream_copy(file, output, temp_path, msg_size) < 0)
+			return -1;
+		i_assert(file->input->v_offset == offset);
+
+		/* get message body size */
+		i_stream_seek(file->input, body_offset);
+		body_input = i_stream_create_limit(file->input, msg_size);
+		ret = message_get_body_size(body_input, &body, &has_nuls);
+		i_stream_unref(&body_input);
+		if (ret < 0) {
+			mail_storage_set_critical(&file->storage->storage,
+				"read(%s) failed: %s", file->cur_path,
+				i_stream_get_error(body_input));
+			return -1;
+		}
+
+		/* write msg metadata. */
+		i_assert(file->input->v_offset == offset);
+		ret = dbox_file_metadata_skip_header(file);
+		if (ret < 0)
+			return -1;
+		o_stream_nsend_str(output, DBOX_MAGIC_POST);
+		if (ret == 0)
+			have_guid = FALSE;
+		else
+			dbox_file_copy_metadata(file, output, &have_guid);
+		if (!have_guid) {
+			guid_128_generate(guid_128);
+			o_stream_nsend_str(output,
+				t_strdup_printf("%c%s\n", DBOX_METADATA_GUID,
+				guid_128_to_string(guid_128)));
+		}
+		o_stream_nsend_str(output,
+			t_strdup_printf("%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE,
+					(unsigned long long)body.virtual_size));
+		o_stream_nsend_str(output, "\n");
+		if (output->stream_errno != 0)
+			break;
+	}
+	if (o_stream_flush(output) < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+			"write(%s) failed: %s", temp_path, o_stream_get_error(output));
+		ret = -1;
+	}
+	return ret;
+}
+
+int dbox_file_fix(struct dbox_file *file, uoff_t start_offset)
+{
+	struct ostream *output;
+	const char *dir, *p, *temp_path, *broken_path;
+	bool deleted, have_messages;
+	int fd, ret;
+
+	i_assert(dbox_file_is_open(file));
+
+	p = strrchr(file->cur_path, '/');
+	i_assert(p != NULL);
+	dir = t_strdup_until(file->cur_path, p);
+
+	temp_path = t_strdup_printf("%s/%s", dir, dbox_generate_tmp_filename());
+	fd = file->storage->v.file_create_fd(file, temp_path, FALSE);
+	if (fd == -1)
+		return -1;
+
+	output = o_stream_create_fd_file(fd, 0, FALSE);
+	o_stream_cork(output);
+	ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output);
+	if (ret < 0)
+		o_stream_abort(output);
+	have_messages = output->offset > file->file_header_size;
+	o_stream_unref(&output);
+	if (close(fd) < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+					  "close(%s) failed: %m", temp_path);
+		ret = -1;
+	}
+	if (ret < 0) {
+		if (unlink(temp_path) < 0) {
+			mail_storage_set_critical(&file->storage->storage,
+				"unlink(%s) failed: %m", temp_path);
+		}
+		return -1;
+	}
+	/* keep a copy of the original file in case someone wants to look
+	   at it */
+	broken_path = t_strconcat(file->cur_path,
+				  DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX, NULL);
+	if (link(file->cur_path, broken_path) < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+					  "link(%s, %s) failed: %m",
+					  file->cur_path, broken_path);
+	} else {
+		i_warning("dbox: Copy of the broken file saved to %s",
+			  broken_path);
+	}
+	if (!have_messages) {
+		/* the resulting file has no messages. just delete the file. */
+		dbox_file_close(file);
+		i_unlink(temp_path);
+		i_unlink(file->cur_path);
+		return 0;
+	}
+	if (rename(temp_path, file->cur_path) < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+					  "rename(%s, %s) failed: %m",
+					  temp_path, file->cur_path);
+		return -1;
+	}
+
+	/* file was successfully recreated - reopen it */
+	dbox_file_close(file);
+	if (dbox_file_open(file, &deleted) <= 0) {
+		mail_storage_set_critical(&file->storage->storage,
+			"dbox_file_fix(%s): reopening file failed",
+			file->cur_path);
+		return -1;
+	}
+	return 1;
+}
diff --git a/src/lib-storage/index/dbox-common/dbox-file.c b/src/lib-storage/index/dbox-common/dbox-file.c
new file mode 100644
index 0000000..16810b0
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-file.c
@@ -0,0 +1,796 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "hex-dec.h"
+#include "hex-binary.h"
+#include "hostpid.h"
+#include "istream.h"
+#include "ostream.h"
+#include "file-lock.h"
+#include "file-dotlock.h"
+#include "mkdir-parents.h"
+#include "eacces-error.h"
+#include "str.h"
+#include "dbox-storage.h"
+#include "dbox-file.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <fcntl.h>
+
+#define DBOX_READ_BLOCK_SIZE IO_BLOCK_SIZE
+
+#ifndef DBOX_FILE_LOCK_METHOD_FLOCK
+static const struct dotlock_settings dotlock_set = {
+	.stale_timeout = 60*10,
+	.use_excl_lock = TRUE
+};
+#endif
+
+const char *dbox_generate_tmp_filename(void)
+{
+	static unsigned int create_count = 0;
+
+	return t_strdup_printf(DBOX_TEMP_FILE_PREFIX"%"PRIdTIME_T".P%sQ%uM%u.%s",
+			       ioloop_timeval.tv_sec, my_pid,
+			       create_count++,
+			       (unsigned int)ioloop_timeval.tv_usec,
+			       my_hostname);
+}
+
+void dbox_file_set_syscall_error(struct dbox_file *file, const char *function)
+{
+	mail_storage_set_critical(&file->storage->storage,
+				  "%s failed for file %s: %m",
+				  function, file->cur_path);
+}
+
+void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...)
+{
+	va_list args;
+
+	va_start(args, reason);
+	mail_storage_set_critical(&file->storage->storage,
+		"Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s",
+		file->cur_path, file->input == NULL ? 0 : file->input->v_offset,
+		t_strdup_vprintf(reason, args));
+	va_end(args);
+
+	file->storage->v.set_file_corrupted(file);
+}
+
+void dbox_file_init(struct dbox_file *file)
+{
+	file->refcount = 1;
+	file->fd = -1;
+	file->cur_offset = UOFF_T_MAX;
+	file->cur_path = file->primary_path;
+}
+
+void dbox_file_free(struct dbox_file *file)
+{
+	i_assert(file->refcount == 0);
+
+	pool_unref(&file->metadata_pool);
+	dbox_file_close(file);
+	i_free(file->primary_path);
+	i_free(file->alt_path);
+	i_free(file);
+}
+
+void dbox_file_unref(struct dbox_file **_file)
+{
+	struct dbox_file *file = *_file;
+
+	*_file = NULL;
+
+	i_assert(file->refcount > 0);
+	if (--file->refcount == 0)
+		file->storage->v.file_unrefed(file);
+}
+
+static int dbox_file_parse_header(struct dbox_file *file, const char *line)
+{
+	const char *const *tmp, *value;
+	enum dbox_header_key key;
+
+	file->file_version = *line - '0';
+	if (!i_isdigit(line[0]) || line[1] != ' ' ||
+	    (file->file_version != 1 && file->file_version != DBOX_VERSION)) {
+		dbox_file_set_corrupted(file, "Invalid dbox version");
+		return -1;
+	}
+	line += 2;
+
+	file->msg_header_size = 0;
+
+	for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) {
+		uintmax_t time;
+		key = **tmp;
+		value = *tmp + 1;
+
+		switch (key) {
+		case DBOX_HEADER_OLDV1_APPEND_OFFSET:
+			break;
+		case DBOX_HEADER_MSG_HEADER_SIZE:
+			if (str_to_uint_hex(value, &file->msg_header_size) < 0) {
+				dbox_file_set_corrupted(file, "Invalid message header size");
+				return -1;
+			}
+			break;
+		case DBOX_HEADER_CREATE_STAMP:
+			if (str_to_uintmax_hex(value, &time) < 0) {
+				dbox_file_set_corrupted(file, "Invalid create time stamp");
+				return -1;
+			}
+			file->create_time = (time_t)time;
+			break;
+		}
+	}
+
+	if (file->msg_header_size == 0) {
+		dbox_file_set_corrupted(file, "Missing message header size");
+		return -1;
+	}
+	return 0;
+}
+
+static int dbox_file_read_header(struct dbox_file *file)
+{
+	const char *line;
+	unsigned int hdr_size;
+	int ret;
+
+	i_stream_seek(file->input, 0);
+	line = i_stream_read_next_line(file->input);
+	if (line == NULL) {
+		if (file->input->stream_errno == 0) {
+			dbox_file_set_corrupted(file,
+				"EOF while reading file header");
+			return 0;
+		}
+
+		dbox_file_set_syscall_error(file, "read()");
+		return -1;
+	}
+	hdr_size = file->input->v_offset;
+	T_BEGIN {
+		ret = dbox_file_parse_header(file, line) < 0 ? 0 : 1;
+	} T_END;
+	if (ret > 0)
+		file->file_header_size = hdr_size;
+	return ret;
+}
+
+static int dbox_file_open_fd(struct dbox_file *file, bool try_altpath)
+{
+	const char *path;
+	int flags = O_RDWR;
+	bool alt = FALSE;
+
+	/* try the primary path first */
+	path = file->primary_path;
+	while ((file->fd = open(path, flags)) == -1) {
+		if (errno == EACCES && flags == O_RDWR) {
+			flags = O_RDONLY;
+			continue;
+		}
+		if (errno != ENOENT) {
+			mail_storage_set_critical(&file->storage->storage,
+						  "open(%s) failed: %m", path);
+			return -1;
+		}
+
+		if (file->alt_path == NULL || alt || !try_altpath) {
+			/* not found */
+			return 0;
+		}
+
+		/* try the alternative path */
+		path = file->alt_path;
+		alt = TRUE;
+	}
+	file->cur_path = path;
+	return 1;
+}
+
+static int dbox_file_open_full(struct dbox_file *file, bool try_altpath,
+			       bool *notfound_r)
+{
+	int ret, fd;
+
+	*notfound_r = FALSE;
+	if (file->input != NULL)
+		return 1;
+
+	if (file->fd == -1) {
+		T_BEGIN {
+			ret = dbox_file_open_fd(file, try_altpath);
+		} T_END;
+		if (ret <= 0) {
+			if (ret < 0)
+				return -1;
+			*notfound_r = TRUE;
+			return 1;
+		}
+	}
+
+	/* we're manually checking at dbox_file_close() if we need to close the
+	   fd or not. */
+	fd = file->fd;
+	file->input = i_stream_create_fd_autoclose(&fd, DBOX_READ_BLOCK_SIZE);
+	i_stream_set_name(file->input, file->cur_path);
+	i_stream_set_init_buffer_size(file->input, DBOX_READ_BLOCK_SIZE);
+	return dbox_file_read_header(file);
+}
+
+int dbox_file_open(struct dbox_file *file, bool *deleted_r)
+{
+	return dbox_file_open_full(file, TRUE, deleted_r);
+}
+
+int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r)
+{
+	return dbox_file_open_full(file, FALSE, notfound_r);
+}
+
+int dbox_file_stat(struct dbox_file *file, struct stat *st_r)
+{
+	const char *path;
+	bool alt = FALSE;
+
+	if (dbox_file_is_open(file)) {
+		if (fstat(file->fd, st_r) < 0) {
+			mail_storage_set_critical(&file->storage->storage,
+				"fstat(%s) failed: %m", file->cur_path);
+			return -1;
+		}
+		return 0;
+	}
+
+	/* try the primary path first */
+	path = file->primary_path;
+	while (stat(path, st_r) < 0) {
+		if (errno != ENOENT) {
+			mail_storage_set_critical(&file->storage->storage,
+						  "stat(%s) failed: %m", path);
+			return -1;
+		}
+
+		if (file->alt_path == NULL || alt) {
+			/* not found */
+			return -1;
+		}
+
+		/* try the alternative path */
+		path = file->alt_path;
+		alt = TRUE;
+	}
+	file->cur_path = path;
+	return 0;
+}
+
+int dbox_file_header_write(struct dbox_file *file, struct ostream *output)
+{
+	string_t *hdr;
+
+	hdr = t_str_new(128);
+	str_printfa(hdr, "%u %c%x %c%x\n", DBOX_VERSION,
+		    DBOX_HEADER_MSG_HEADER_SIZE,
+		    (unsigned int)sizeof(struct dbox_message_header),
+		    DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time);
+
+	file->file_version = DBOX_VERSION;
+	file->file_header_size = str_len(hdr);
+	file->msg_header_size = sizeof(struct dbox_message_header);
+	return o_stream_send(output, str_data(hdr), str_len(hdr));
+}
+
+void dbox_file_close(struct dbox_file *file)
+{
+	dbox_file_unlock(file);
+	if (file->input != NULL) {
+		/* stream autocloses the fd when it gets destroyed. note that
+		   the stream may outlive the struct dbox_file. */
+		i_stream_unref(&file->input);
+		file->fd = -1;
+	} else if (file->fd != -1) {
+		if (close(file->fd) < 0)
+			dbox_file_set_syscall_error(file, "close()");
+		file->fd = -1;
+	}
+	file->cur_offset = UOFF_T_MAX;
+}
+
+int dbox_file_try_lock(struct dbox_file *file)
+{
+	const char *error;
+	int ret;
+
+	i_assert(file->fd != -1);
+
+#ifdef DBOX_FILE_LOCK_METHOD_FLOCK
+	struct file_lock_settings lock_set = {
+		.lock_method = FILE_LOCK_METHOD_FLOCK,
+	};
+	ret = file_try_lock(file->fd, file->cur_path, F_WRLCK,
+			    &lock_set, &file->lock, &error);
+	if (ret < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+			"file_try_lock(%s) failed: %s", file->cur_path, error);
+	}
+#else
+	ret = file_dotlock_create(&dotlock_set, file->cur_path,
+				  DOTLOCK_CREATE_FLAG_NONBLOCK, &file->lock);
+	if (ret < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+			"file_dotlock_create(%s) failed: %m", file->cur_path);
+	}
+#endif
+	return ret;
+}
+
+void dbox_file_unlock(struct dbox_file *file)
+{
+	i_assert(!file->appending || file->lock == NULL);
+
+	if (file->lock != NULL) {
+#ifdef DBOX_FILE_LOCK_METHOD_FLOCK
+		file_unlock(&file->lock);
+#else
+		file_dotlock_delete(&file->lock);
+#endif
+	}
+	if (file->input != NULL)
+		i_stream_sync(file->input);
+}
+
+int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r)
+{
+	struct dbox_message_header hdr;
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	ret = i_stream_read_bytes(file->input, &data, &size,
+				  file->msg_header_size);
+	if (ret <= 0) {
+		if (file->input->stream_errno == 0) {
+			/* EOF, broken offset or file truncated */
+			dbox_file_set_corrupted(file, "EOF reading msg header "
+						"(got %zu/%u bytes)",
+						size, file->msg_header_size);
+			return 0;
+		}
+		dbox_file_set_syscall_error(file, "read()");
+		return -1;
+	}
+	memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size));
+	if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) {
+		/* probably broken offset */
+		dbox_file_set_corrupted(file, "msg header has bad magic value");
+		return 0;
+	}
+
+	if (data[file->msg_header_size-1] != '\n') {
+		dbox_file_set_corrupted(file, "msg header doesn't end with LF");
+		return 0;
+	}
+
+	*physical_size_r = hex2dec(hdr.message_size_hex,
+				   sizeof(hdr.message_size_hex));
+	return 1;
+}
+
+int dbox_file_seek(struct dbox_file *file, uoff_t offset)
+{
+	uoff_t size;
+	int ret;
+
+	i_assert(file->input != NULL);
+
+	if (offset == 0)
+		offset = file->file_header_size;
+
+	if (offset != file->cur_offset) {
+		i_stream_seek(file->input, offset);
+		ret = dbox_file_read_mail_header(file, &size);
+		if (ret <= 0)
+			return ret;
+		file->cur_offset = offset;
+		file->cur_physical_size = size;
+	}
+	i_stream_seek(file->input, offset + file->msg_header_size);
+	return 1;
+}
+
+static int
+dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset)
+{
+	const char *line;
+	size_t buf_size;
+	int ret;
+
+	i_stream_seek(file->input, *offset);
+	if ((ret = dbox_file_metadata_skip_header(file)) <= 0)
+		return ret;
+
+	/* skip over the actual metadata */
+	buf_size = i_stream_get_max_buffer_size(file->input);
+	i_stream_set_max_buffer_size(file->input, SIZE_MAX);
+	while ((line = i_stream_read_next_line(file->input)) != NULL) {
+		if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
+			/* end of metadata */
+			break;
+		}
+	}
+	i_stream_set_max_buffer_size(file->input, buf_size);
+	*offset = file->input->v_offset;
+	return 1;
+}
+
+void dbox_file_seek_rewind(struct dbox_file *file)
+{
+	file->cur_offset = UOFF_T_MAX;
+}
+
+int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r)
+{
+	uoff_t offset;
+	int ret;
+
+	i_assert(file->input != NULL);
+
+	if (file->cur_offset == UOFF_T_MAX) {
+		/* first mail. we may not have read the file at all yet,
+		   so set the offset afterwards. */
+		offset = 0;
+	} else {
+		offset = file->cur_offset + file->msg_header_size +
+			file->cur_physical_size;
+		if ((ret = dbox_file_seek_next_at_metadata(file, &offset)) <= 0) {
+			*offset_r = file->cur_offset;
+			return ret;
+		}
+		if (i_stream_read_eof(file->input)) {
+			*last_r = TRUE;
+			return 0;
+		}
+	}
+	*offset_r = offset;
+
+	*last_r = FALSE;
+
+	ret = dbox_file_seek(file, offset);
+	if (*offset_r == 0)
+		*offset_r = file->file_header_size;
+	return ret;
+}
+
+struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file)
+{
+	struct dbox_file_append_context *ctx;
+
+	i_assert(!file->appending);
+
+	file->appending = TRUE;
+
+	ctx = i_new(struct dbox_file_append_context, 1);
+	ctx->file = file;
+	if (file->fd != -1) {
+		ctx->output = o_stream_create_fd_file(file->fd, 0, FALSE);
+		o_stream_set_name(ctx->output, file->cur_path);
+		o_stream_set_finish_via_child(ctx->output, FALSE);
+		o_stream_cork(ctx->output);
+	}
+	return ctx;
+}
+
+int dbox_file_append_commit(struct dbox_file_append_context **_ctx)
+{
+	struct dbox_file_append_context *ctx = *_ctx;
+	int ret;
+
+	i_assert(ctx->file->appending);
+
+	*_ctx = NULL;
+
+	ret = dbox_file_append_flush(ctx);
+	if (ctx->last_checkpoint_offset != ctx->output->offset) {
+		o_stream_close(ctx->output);
+		if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) {
+			dbox_file_set_syscall_error(ctx->file, "ftruncate()");
+			return -1;
+		}
+	}
+	o_stream_unref(&ctx->output);
+	ctx->file->appending = FALSE;
+	i_free(ctx);
+	return ret;
+}
+
+void dbox_file_append_rollback(struct dbox_file_append_context **_ctx)
+{
+	struct dbox_file_append_context *ctx = *_ctx;
+	struct dbox_file *file = ctx->file;
+	bool close_file = FALSE;
+
+	i_assert(ctx->file->appending);
+
+	*_ctx = NULL;
+	if (ctx->first_append_offset == 0) {
+		/* nothing changed */
+	} else if (ctx->first_append_offset == file->file_header_size) {
+		/* rolling back everything */
+		if (unlink(file->cur_path) < 0)
+			dbox_file_set_syscall_error(file, "unlink()");
+		close_file = TRUE;
+	} else {
+		/* truncating only some mails */
+		o_stream_close(ctx->output);
+		if (ftruncate(file->fd, ctx->first_append_offset) < 0)
+			dbox_file_set_syscall_error(file, "ftruncate()");
+	}
+	if (ctx->output != NULL) {
+		o_stream_abort(ctx->output);
+		o_stream_unref(&ctx->output);
+	}
+	i_free(ctx);
+
+	if (close_file)
+		dbox_file_close(file);
+	file->appending = FALSE;
+}
+
+int dbox_file_append_flush(struct dbox_file_append_context *ctx)
+{
+	struct mail_storage *storage = &ctx->file->storage->storage;
+
+	if (ctx->last_flush_offset == ctx->output->offset &&
+	    ctx->last_checkpoint_offset == ctx->output->offset)
+		return 0;
+
+	if (o_stream_flush(ctx->output) < 0) {
+		dbox_file_set_syscall_error(ctx->file, "write()");
+		return -1;
+	}
+
+	if (ctx->last_checkpoint_offset != ctx->output->offset) {
+		if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) {
+			dbox_file_set_syscall_error(ctx->file, "ftruncate()");
+			return -1;
+		}
+		if (o_stream_seek(ctx->output, ctx->last_checkpoint_offset) < 0) {
+			dbox_file_set_syscall_error(ctx->file, "lseek()");
+			return -1;
+		}
+	}
+
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
+		if (fdatasync(ctx->file->fd) < 0) {
+			dbox_file_set_syscall_error(ctx->file, "fdatasync()");
+			return -1;
+		}
+	}
+	ctx->last_flush_offset = ctx->output->offset;
+	return 0;
+}
+
+void dbox_file_append_checkpoint(struct dbox_file_append_context *ctx)
+{
+	ctx->last_checkpoint_offset = ctx->output->offset;
+}
+
+int dbox_file_get_append_stream(struct dbox_file_append_context *ctx,
+				struct ostream **output_r)
+{
+	struct dbox_file *file = ctx->file;
+	struct stat st;
+
+	if (ctx->output == NULL) {
+		/* file creation had failed */
+		return -1;
+	}
+	if (ctx->last_checkpoint_offset != ctx->output->offset) {
+		/* a message was aborted. don't try appending to this
+		   file anymore. */
+		return -1;
+	}
+
+	if (file->file_version == 0) {
+		/* newly created file, write the file header */
+		if (dbox_file_header_write(file, ctx->output) < 0) {
+			dbox_file_set_syscall_error(file, "write()");
+			return -1;
+		}
+		*output_r = ctx->output;
+		return 1;
+	}
+
+	/* file has existing mails */
+	if (file->file_version != DBOX_VERSION ||
+	    file->msg_header_size != sizeof(struct dbox_message_header)) {
+		/* created by an incompatible version, can't append */
+		return 0;
+	}
+
+	if (ctx->output->offset == 0) {
+		/* first append to existing file. seek to eof first. */
+		if (fstat(file->fd, &st) < 0) {
+			dbox_file_set_syscall_error(file, "fstat()");
+			return -1;
+		}
+		if (st.st_size < file->msg_header_size) {
+			dbox_file_set_corrupted(file,
+				"dbox file size too small");
+			return 0;
+		}
+		if (o_stream_seek(ctx->output, st.st_size) < 0) {
+			dbox_file_set_syscall_error(file, "lseek()");
+			return -1;
+		}
+	}
+	*output_r = ctx->output;
+	return 1;
+}
+
+int dbox_file_metadata_skip_header(struct dbox_file *file)
+{
+	struct dbox_metadata_header metadata_hdr;
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	ret = i_stream_read_bytes(file->input, &data, &size,
+				  sizeof(metadata_hdr));
+	if (ret <= 0) {
+		if (file->input->stream_errno == 0) {
+			/* EOF, broken offset */
+			dbox_file_set_corrupted(file,
+				"Unexpected EOF while reading metadata header");
+			return 0;
+		}
+		dbox_file_set_syscall_error(file, "read()");
+		return -1;
+	}
+	memcpy(&metadata_hdr, data, sizeof(metadata_hdr));
+	if (memcmp(metadata_hdr.magic_post, DBOX_MAGIC_POST,
+		   sizeof(metadata_hdr.magic_post)) != 0) {
+		/* probably broken offset */
+		dbox_file_set_corrupted(file,
+			"metadata header has bad magic value");
+		return 0;
+	}
+	i_stream_skip(file->input, sizeof(metadata_hdr));
+	return 1;
+}
+
+static int
+dbox_file_metadata_read_at(struct dbox_file *file, uoff_t metadata_offset)
+{
+	const char *line;
+	size_t buf_size;
+	int ret;
+
+	if (file->metadata_pool != NULL)
+		p_clear(file->metadata_pool);
+	else {
+		file->metadata_pool =
+			pool_alloconly_create("dbox metadata", 1024);
+	}
+	p_array_init(&file->metadata, file->metadata_pool, 16);
+
+	i_stream_seek(file->input, metadata_offset);
+	if ((ret = dbox_file_metadata_skip_header(file)) <= 0)
+		return ret;
+
+	ret = 0;
+	buf_size = i_stream_get_max_buffer_size(file->input);
+	/* use unlimited line length for metadata */
+	i_stream_set_max_buffer_size(file->input, SIZE_MAX);
+	while ((line = i_stream_read_next_line(file->input)) != NULL) {
+		if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
+			/* end of metadata */
+			ret = 1;
+			break;
+		}
+		line = p_strdup(file->metadata_pool, line);
+		array_push_back(&file->metadata, &line);
+	}
+	i_stream_set_max_buffer_size(file->input, buf_size);
+	if (ret == 0)
+		dbox_file_set_corrupted(file, "missing end-of-metadata line");
+	return ret;
+}
+
+int dbox_file_metadata_read(struct dbox_file *file)
+{
+	uoff_t metadata_offset;
+	int ret;
+
+	i_assert(file->cur_offset != UOFF_T_MAX);
+
+	if (file->metadata_read_offset == file->cur_offset)
+		return 1;
+
+	metadata_offset = file->cur_offset + file->msg_header_size +
+		file->cur_physical_size;
+	ret = dbox_file_metadata_read_at(file, metadata_offset);
+	if (ret <= 0)
+		return ret;
+
+	file->metadata_read_offset = file->cur_offset;
+	return 1;
+}
+
+const char *dbox_file_metadata_get(struct dbox_file *file,
+				   enum dbox_metadata_key key)
+{
+	const char *const *metadata;
+	unsigned int i, count;
+
+	metadata = array_get(&file->metadata, &count);
+	for (i = 0; i < count; i++) {
+		if (*metadata[i] == (char)key)
+			return metadata[i] + 1;
+	}
+	return NULL;
+}
+
+uoff_t dbox_file_get_plaintext_size(struct dbox_file *file)
+{
+	const char *value;
+	uintmax_t size;
+
+	i_assert(file->metadata_read_offset == file->cur_offset);
+
+	/* see if we have it in metadata */
+	value = dbox_file_metadata_get(file, DBOX_METADATA_PHYSICAL_SIZE);
+	if (value == NULL ||
+	    str_to_uintmax_hex(value, &size) < 0 ||
+	    size > UOFF_T_MAX) {
+		/* no. that means we can use the size in the header */
+		return file->cur_physical_size;
+	}
+
+	return (uoff_t)size;
+}
+
+void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr,
+			  uoff_t message_size)
+{
+	memset(dbox_msg_hdr, ' ', sizeof(*dbox_msg_hdr));
+	memcpy(dbox_msg_hdr->magic_pre, DBOX_MAGIC_PRE,
+	       sizeof(dbox_msg_hdr->magic_pre));
+	dbox_msg_hdr->type = DBOX_MESSAGE_TYPE_NORMAL;
+	dec2hex(dbox_msg_hdr->message_size_hex, message_size,
+		sizeof(dbox_msg_hdr->message_size_hex));
+	dbox_msg_hdr->save_lf = '\n';
+}
+
+int dbox_file_unlink(struct dbox_file *file)
+{
+	const char *path;
+	bool alt = FALSE;
+
+	path = file->primary_path;
+	while (unlink(path) < 0) {
+		if (errno != ENOENT) {
+			mail_storage_set_critical(&file->storage->storage,
+				"unlink(%s) failed: %m", path);
+			return -1;
+		}
+		if (file->alt_path == NULL || alt) {
+			/* not found */
+			return 0;
+		}
+
+		/* try the alternative path */
+		path = file->alt_path;
+		alt = TRUE;
+	}
+	return 1;
+}
diff --git a/src/lib-storage/index/dbox-common/dbox-file.h b/src/lib-storage/index/dbox-common/dbox-file.h
new file mode 100644
index 0000000..309c705
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-file.h
@@ -0,0 +1,218 @@
+#ifndef DBOX_FILE_H
+#define DBOX_FILE_H
+
+/* The file begins with a header followed by zero or more messages:
+
+   <dbox message header>
+   <LF>
+   <message body>
+   <metadata>
+
+   Metadata block begins with DBOX_MAGIC_POST, followed by zero or more lines
+   in format <key character><value><LF>. The block ends with an empty line.
+   Unknown metadata should be ignored, but preserved when copying.
+
+   There should be no duplicates for the current metadata, but future
+   extensions may need them so they should be preserved.
+*/
+#define DBOX_VERSION 2
+#define DBOX_MAGIC_PRE "\001\002"
+#define DBOX_MAGIC_POST "\n\001\003\n"
+
+/* prefer flock(). fcntl() locking currently breaks if trying to access the
+   same file from multiple mail_storages within same process. that's why we
+   fallback to dotlocks. */
+#ifdef HAVE_FLOCK
+#  define DBOX_FILE_LOCK_METHOD_FLOCK
+#endif
+
+struct dbox_file;
+struct stat;
+
+enum dbox_header_key {
+	/* Must be sizeof(struct dbox_message_header) when appending (hex) */
+	DBOX_HEADER_MSG_HEADER_SIZE	= 'M',
+	/* Creation UNIX timestamp (hex) */
+	DBOX_HEADER_CREATE_STAMP	= 'C',
+
+	/* metadata used by old Dovecot versions */
+	DBOX_HEADER_OLDV1_APPEND_OFFSET	= 'A'
+};
+
+/* NOTE: all valid keys are uppercase characters. if this changes, change
+   dbox-file-fix.c:dbox_file_match_post_magic() to recognize them */
+enum dbox_metadata_key {
+	/* Globally unique identifier for the message. Preserved when
+	   copying. */
+	DBOX_METADATA_GUID		= 'G',
+	/* POP3 UIDL overriding the default format */
+	DBOX_METADATA_POP3_UIDL		= 'P',
+	/* POP3 message ordering (for migrated mails) */
+	DBOX_METADATA_POP3_ORDER	= 'O',
+	/* Received UNIX timestamp in hex */
+	DBOX_METADATA_RECEIVED_TIME	= 'R',
+	/* Physical message size in hex. Necessary only if it differs from
+	   the dbox_message_header.message_size_hex, for example because the
+	   message is compressed. */
+	DBOX_METADATA_PHYSICAL_SIZE	= 'Z',
+	/* Virtual message size in hex (line feeds counted as CRLF) */
+	DBOX_METADATA_VIRTUAL_SIZE	= 'V',
+	/* Pointer to external message data. Format is:
+	   1*(<start offset> <byte count> <options> <ref>) */
+	DBOX_METADATA_EXT_REF		= 'X',
+	/* Mailbox name where this message was originally saved to.
+	   When rebuild finds a message whose mailbox is unknown, it's
+	   placed to this mailbox. */
+	DBOX_METADATA_ORIG_MAILBOX	= 'B',
+
+	/* metadata used by old Dovecot versions */
+	DBOX_METADATA_OLDV1_EXPUNGED	= 'E',
+	DBOX_METADATA_OLDV1_FLAGS	= 'F',
+	DBOX_METADATA_OLDV1_KEYWORDS	= 'K',
+	DBOX_METADATA_OLDV1_SAVE_TIME	= 'S',
+	DBOX_METADATA_OLDV1_SPACE	= ' '
+};
+
+enum dbox_message_type {
+	/* Normal message */
+	DBOX_MESSAGE_TYPE_NORMAL	= 'N'
+};
+
+struct dbox_message_header {
+	unsigned char magic_pre[2];
+	unsigned char type;
+	unsigned char space1;
+	unsigned char oldv1_uid_hex[8];
+	unsigned char space2;
+	unsigned char message_size_hex[16];
+	/* <space reserved for future extensions, LF is always last> */
+	unsigned char save_lf;
+};
+
+struct dbox_metadata_header {
+	unsigned char magic_post[sizeof(DBOX_MAGIC_POST)-1];
+};
+
+struct dbox_file {
+	struct dbox_storage *storage;
+	int refcount;
+
+	time_t create_time;
+	unsigned int file_version;
+	unsigned int file_header_size;
+	unsigned int msg_header_size;
+
+	const char *cur_path;
+	char *primary_path, *alt_path;
+	int fd;
+	struct istream *input;
+#ifdef DBOX_FILE_LOCK_METHOD_FLOCK
+	struct file_lock *lock;
+#else
+	struct dotlock *lock;
+#endif
+
+	uoff_t cur_offset;
+	uoff_t cur_physical_size;
+
+	/* Metadata for the currently seeked metadata block. */
+	pool_t metadata_pool;
+	ARRAY(const char *) metadata;
+	uoff_t metadata_read_offset;
+
+	bool appending:1;
+	bool corrupted:1;
+};
+
+struct dbox_file_append_context {
+	struct dbox_file *file;
+
+	uoff_t first_append_offset, last_checkpoint_offset, last_flush_offset;
+	struct ostream *output;
+};
+
+#define dbox_file_is_open(file) ((file)->fd != -1)
+#define dbox_file_is_in_alt(file) ((file)->cur_path == (file)->alt_path)
+
+void dbox_file_init(struct dbox_file *file);
+void dbox_file_unref(struct dbox_file **file);
+
+/* Open the file. Returns 1 if ok, 0 if file header is corrupted, -1 if error.
+   If file is deleted, deleted_r=TRUE and 1 is returned. */
+int dbox_file_open(struct dbox_file *file, bool *deleted_r);
+/* Try to open file only from primary path. */
+int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r);
+/* Close the file handle from the file, but don't free it. */
+void dbox_file_close(struct dbox_file *file);
+
+/* fstat() or stat() the file. If file is already deleted, fails with
+   errno=ENOENT. */
+int dbox_file_stat(struct dbox_file *file, struct stat *st_r);
+
+/* Try to lock the dbox file. Returns 1 if ok, 0 if already locked by someone
+   else, -1 if error. */
+int dbox_file_try_lock(struct dbox_file *file);
+void dbox_file_unlock(struct dbox_file *file);
+
+/* Seek to given offset in file. Returns 1 if ok/expunged, 0 if file/offset is
+   corrupted, -1 if I/O error. */
+int dbox_file_seek(struct dbox_file *file, uoff_t offset);
+/* Start seeking at the beginning of the file. */
+void dbox_file_seek_rewind(struct dbox_file *file);
+/* Seek to next message after current one. If there are no more messages,
+   returns 0 and last_r is set to TRUE. Returns 1 if ok, 0 if file is
+   corrupted, -1 if I/O error. */
+int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r);
+
+/* Start appending to dbox file */
+struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file);
+/* Finish writing appended mails. */
+int dbox_file_append_commit(struct dbox_file_append_context **ctx);
+/* Truncate appended mails. */
+void dbox_file_append_rollback(struct dbox_file_append_context **ctx);
+/* Get output stream for appending a new message. Returns 1 if ok, 0 if file
+   can't be appended to (old file version or corruption) or -1 if error. */
+int dbox_file_get_append_stream(struct dbox_file_append_context *ctx,
+				struct ostream **output_r);
+/* Call after message has been fully saved. If this isn't done, the writes
+   since the last checkpoint are truncated. */
+void dbox_file_append_checkpoint(struct dbox_file_append_context *ctx);
+/* Flush output buffer. */
+int dbox_file_append_flush(struct dbox_file_append_context *ctx);
+
+/* Read current message's metadata. Returns 1 if ok, 0 if metadata is
+   corrupted, -1 if I/O error. */
+int dbox_file_metadata_read(struct dbox_file *file);
+/* Return wanted metadata value, or NULL if not found. */
+const char *dbox_file_metadata_get(struct dbox_file *file,
+				   enum dbox_metadata_key key);
+
+/* Returns DBOX_METADATA_PHYSICAL_SIZE if set, otherwise physical size from
+   header. They differ only for e.g. compressed mails. */
+uoff_t dbox_file_get_plaintext_size(struct dbox_file *file);
+
+/* Fix a broken dbox file by rename()ing over it with a fixed file. Everything
+   before start_offset is assumed to be valid and is simply copied. The file
+   is reopened afterwards. Returns 1 if ok, 0 if the resulting file has no
+   mails and was deleted, -1 if I/O error. */
+int dbox_file_fix(struct dbox_file *file, uoff_t start_offset);
+/* Delete the given dbox file. Returns 1 if deleted, 0 if file wasn't found
+   or -1 if error. */
+int dbox_file_unlink(struct dbox_file *file);
+
+/* Fill dbox_message_header with given size. */
+void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr,
+			  uoff_t message_size);
+
+void dbox_file_set_syscall_error(struct dbox_file *file, const char *function);
+void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...)
+	ATTR_FORMAT(2, 3);
+
+/* private: */
+const char *dbox_generate_tmp_filename(void);
+void dbox_file_free(struct dbox_file *file);
+int dbox_file_header_write(struct dbox_file *file, struct ostream *output);
+int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r);
+int dbox_file_metadata_skip_header(struct dbox_file *file);
+
+#endif
diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c
new file mode 100644
index 0000000..49279f9
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-mail.c
@@ -0,0 +1,318 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "str.h"
+#include "index-storage.h"
+#include "index-mail.h"
+#include "index-pop3-uidl.h"
+#include "dbox-attachment.h"
+#include "dbox-storage.h"
+#include "dbox-file.h"
+#include "dbox-mail.h"
+
+
+struct mail *
+dbox_mail_alloc(struct mailbox_transaction_context *t,
+		enum mail_fetch_field wanted_fields,
+		struct mailbox_header_lookup_ctx *wanted_headers)
+{
+	struct dbox_mail *mail;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mail", 2048);
+	mail = p_new(pool, struct dbox_mail, 1);
+
+	index_mail_init(&mail->imail, t, wanted_fields, wanted_headers, pool, NULL);
+	return &mail->imail.mail.mail;
+}
+
+void dbox_mail_close(struct mail *_mail)
+{
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+
+	index_mail_close(_mail);
+	/* close the dbox file only after index is closed, since it may still
+	   try to read from it. */
+	if (mail->open_file != NULL)
+		dbox_file_unref(&mail->open_file);
+}
+
+int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r)
+{
+	struct dbox_storage *storage =
+		DBOX_STORAGE(mail->imail.mail.mail.box->storage);
+	uoff_t offset;
+
+	if (storage->v.mail_open(mail, &offset, file_r) < 0)
+		return -1;
+
+	if (dbox_file_seek(*file_r, offset) <= 0)
+		return -1;
+	if (dbox_file_metadata_read(*file_r) <= 0)
+		return -1;
+
+	if (mail->imail.data.stream != NULL) {
+		/* we just messed up mail's input stream by reading metadata */
+		i_stream_seek((*file_r)->input, offset);
+		i_stream_sync(mail->imail.data.stream);
+	}
+	return 0;
+}
+
+static int
+dbox_mail_metadata_get(struct dbox_mail *mail, enum dbox_metadata_key key,
+		       const char **value_r)
+{
+	struct dbox_file *file;
+
+	if (dbox_mail_metadata_read(mail, &file) < 0)
+		return -1;
+
+	*value_r = dbox_file_metadata_get(file, key);
+	return 0;
+}
+
+int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	struct index_mail_data *data = &mail->imail.data;
+	struct dbox_file *file;
+
+	if (index_mail_get_physical_size(_mail, size_r) == 0)
+		return 0;
+
+	if (dbox_mail_metadata_read(mail, &file) < 0)
+		return -1;
+
+	data->physical_size = dbox_file_get_plaintext_size(file);
+	*size_r = data->physical_size;
+	return 0;
+}
+
+int dbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	struct index_mail_data *data = &mail->imail.data;
+	const char *value;
+	uintmax_t size;
+
+	if (index_mail_get_cached_virtual_size(&mail->imail, size_r))
+		return 0;
+
+	if (dbox_mail_metadata_get(mail, DBOX_METADATA_VIRTUAL_SIZE,
+				   &value) < 0)
+		return -1;
+	if (value == NULL)
+		return index_mail_get_virtual_size(_mail, size_r);
+
+	if (str_to_uintmax_hex(value, &size) < 0 || size > UOFF_T_MAX)
+		return -1;
+	data->virtual_size = (uoff_t)size;
+	*size_r = data->virtual_size;
+	return 0;
+}
+
+int dbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
+{
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	struct index_mail_data *data = &mail->imail.data;
+	const char *value;
+	uintmax_t time;
+
+	if (index_mail_get_received_date(_mail, date_r) == 0)
+		return 0;
+
+	if (dbox_mail_metadata_get(mail, DBOX_METADATA_RECEIVED_TIME,
+				   &value) < 0)
+		return -1;
+
+	time = 0;
+	if (value != NULL && str_to_uintmax_hex(value, &time) < 0)
+		return -1;
+
+	data->received_date = (time_t)time;
+	*date_r = data->received_date;
+	return 0;
+}
+
+int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r)
+{
+	struct dbox_storage *storage = DBOX_STORAGE(_mail->box->storage);
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	struct index_mail_data *data = &mail->imail.data;
+	struct dbox_file *file;
+	struct stat st;
+	uoff_t offset;
+
+	if (index_mail_get_save_date(_mail, date_r) > 0)
+		return 1;
+
+	if (storage->v.mail_open(mail, &offset, &file) < 0)
+		return -1;
+
+	_mail->transaction->stats.fstat_lookup_count++;
+	if (dbox_file_stat(file, &st) < 0) {
+		if (errno == ENOENT)
+			mail_set_expunged(_mail);
+		return -1;
+	}
+	*date_r = data->save_date = st.st_ctime;
+	return 1;
+}
+
+static int
+dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key,
+			 enum index_cache_field cache_field,
+			 const char **value_r)
+{
+	struct index_mail *imail = &mail->imail;
+	struct index_mailbox_context *ibox =
+		INDEX_STORAGE_CONTEXT(imail->mail.mail.box);
+	const char *value;
+	string_t *str;
+	uint32_t order;
+
+	str = str_new(imail->mail.data_pool, 64);
+	if (mail_cache_lookup_field(imail->mail.mail.transaction->cache_view,
+				    str, imail->mail.mail.seq,
+				    ibox->cache_fields[cache_field].idx) > 0) {
+		if (cache_field == MAIL_CACHE_POP3_ORDER) {
+			i_assert(str_len(str) == sizeof(order));
+			memcpy(&order, str_data(str), sizeof(order));
+			str_truncate(str, 0);
+			if (order != 0)
+				str_printfa(str, "%u", order);
+			else {
+				/* order=0 means it doesn't exist. we don't
+				   want to return "0" though, because then the
+				   mails get ordered to beginning, while
+				   nonexistent are supposed to be ordered at
+				   the end. */
+			}
+		}
+		*value_r = str_c(str);
+		return 0;
+	}
+
+	if (dbox_mail_metadata_get(mail, key, &value) < 0)
+		return -1;
+
+	if (value == NULL)
+		value = "";
+	if (cache_field != MAIL_CACHE_POP3_ORDER) {
+		index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx,
+					 value, strlen(value));
+	} else {
+		if (str_to_uint(value, &order) < 0)
+			order = 0;
+		index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx,
+					 &order, sizeof(order));
+	}
+
+	/* don't return pointer to dbox metadata directly, since it may
+	   change unexpectedly */
+	str_truncate(str, 0);
+	str_append(str, value);
+	*value_r = str_c(str);
+	return 0;
+}
+
+int dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+			  const char **value_r)
+{
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	int ret;
+
+	/* keep the UIDL in cache file, otherwise POP3 would open all
+	   mail files and read the metadata. same for GUIDs if they're
+	   used. */
+	switch (field) {
+	case MAIL_FETCH_UIDL_BACKEND:
+		if (!index_pop3_uidl_can_exist(_mail)) {
+			*value_r = "";
+			return 0;
+		}
+		ret = dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL,
+					       MAIL_CACHE_POP3_UIDL, value_r);
+		if (ret == 0) {
+			index_pop3_uidl_update_exists(&mail->imail.mail.mail,
+						      (*value_r)[0] != '\0');
+		}
+		return ret;
+	case MAIL_FETCH_POP3_ORDER:
+		if (!index_pop3_uidl_can_exist(_mail)) {
+			/* we're assuming that if there's a POP3 order, there's
+			   also a UIDL */
+			*value_r = "";
+			return 0;
+		}
+		return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_ORDER,
+						MAIL_CACHE_POP3_ORDER, value_r);
+	case MAIL_FETCH_GUID:
+		return dbox_get_cached_metadata(mail, DBOX_METADATA_GUID,
+						MAIL_CACHE_GUID, value_r);
+	default:
+		break;
+	}
+
+	return index_mail_get_special(_mail, field, value_r);
+}
+
+static int
+get_mail_stream(struct dbox_mail *mail, uoff_t offset,
+		struct istream **stream_r)
+{
+	struct mail_private *pmail = &mail->imail.mail;
+	struct dbox_file *file = mail->open_file;
+	int ret;
+
+	if ((ret = dbox_file_seek(file, offset)) <= 0) {
+		*stream_r = NULL;
+		return ret;
+	}
+
+	*stream_r = i_stream_create_limit(file->input, file->cur_physical_size);
+	if (pmail->v.istream_opened != NULL) {
+		if (pmail->v.istream_opened(&pmail->mail, stream_r) < 0)
+			return -1;
+	}
+	if (file->storage->attachment_dir == NULL)
+		return 1;
+	else
+		return dbox_attachment_file_get_stream(file, stream_r);
+}
+
+int dbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
+			 struct message_size *hdr_size,
+			 struct message_size *body_size,
+			 struct istream **stream_r)
+{
+	struct dbox_storage *storage = DBOX_STORAGE(_mail->box->storage);
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	struct index_mail_data *data = &mail->imail.data;
+	struct istream *input;
+	uoff_t offset;
+	int ret;
+
+	if (data->stream == NULL) {
+		if (storage->v.mail_open(mail, &offset, &mail->open_file) < 0)
+			return -1;
+
+		ret = get_mail_stream(mail, offset, &input);
+		if (ret <= 0) {
+			if (ret < 0)
+				return -1;
+			dbox_file_set_corrupted(mail->open_file,
+				"uid=%u points to broken data at offset="
+				"%"PRIuUOFF_T, _mail->uid, offset);
+			i_stream_unref(&input);
+			return -1;
+		}
+		data->stream = input;
+		index_mail_set_read_buffer_size(_mail, input);
+	}
+
+	return index_mail_init_stream(&mail->imail, hdr_size, body_size,
+				      stream_r);
+}
diff --git a/src/lib-storage/index/dbox-common/dbox-mail.h b/src/lib-storage/index/dbox-common/dbox-mail.h
new file mode 100644
index 0000000..c03652c
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-mail.h
@@ -0,0 +1,34 @@
+#ifndef DBOX_MAIL_H
+#define DBOX_MAIL_H
+
+#include "index-mail.h"
+
+struct dbox_mail {
+	struct index_mail imail;
+
+	struct dbox_file *open_file;
+	uoff_t offset;
+};
+
+#define DBOX_MAIL(s)	container_of(s, struct dbox_mail, imail.mail.mail)
+
+struct mail *
+dbox_mail_alloc(struct mailbox_transaction_context *t,
+		enum mail_fetch_field wanted_fields,
+		struct mailbox_header_lookup_ctx *wanted_headers);
+void dbox_mail_close(struct mail *mail);
+
+int dbox_mail_get_physical_size(struct mail *mail, uoff_t *size_r);
+int dbox_mail_get_virtual_size(struct mail *mail, uoff_t *size_r);
+int dbox_mail_get_received_date(struct mail *mail, time_t *date_r);
+int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r);
+int dbox_mail_get_special(struct mail *mail, enum mail_fetch_field field,
+			  const char **value_r);
+int dbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
+			 struct message_size *hdr_size,
+			 struct message_size *body_size,
+			 struct istream **stream_r);
+
+int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r);
+
+#endif
diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c
new file mode 100644
index 0000000..c5af8cc
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-save.c
@@ -0,0 +1,226 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "ostream.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "index-mail.h"
+#include "index-storage.h"
+#include "dbox-attachment.h"
+#include "dbox-file.h"
+#include "dbox-save.h"
+
+void dbox_save_add_to_index(struct dbox_save_context *ctx)
+{
+	struct mail_save_data *mdata = &ctx->ctx.data;
+	enum mail_flags save_flags;
+
+	save_flags = mdata->flags & ENUM_NEGATE(MAIL_RECENT);
+	mail_index_append(ctx->trans, mdata->uid, &ctx->seq);
+	mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
+				save_flags);
+	if (mdata->keywords != NULL) {
+		mail_index_update_keywords(ctx->trans, ctx->seq,
+					   MODIFY_REPLACE, mdata->keywords);
+	}
+	if (mdata->min_modseq != 0) {
+		mail_index_update_modseq(ctx->trans, ctx->seq,
+					 mdata->min_modseq);
+	}
+}
+
+void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input)
+{
+	struct mail_save_context *_ctx = &ctx->ctx;
+	struct mail_storage *_storage = _ctx->transaction->box->storage;
+	struct dbox_storage *storage = DBOX_STORAGE(_storage);
+	struct dbox_message_header dbox_msg_hdr;
+	struct istream *crlf_input;
+
+	dbox_save_add_to_index(ctx);
+
+	mail_set_seq_saving(_ctx->dest_mail, ctx->seq);
+
+	crlf_input = i_stream_create_lf(input);
+	ctx->input = index_mail_cache_parse_init(_ctx->dest_mail, crlf_input);
+	i_stream_unref(&crlf_input);
+
+	/* write a dummy header. it'll get rewritten when we're finished */
+	i_zero(&dbox_msg_hdr);
+	o_stream_cork(ctx->dbox_output);
+	if (o_stream_send(ctx->dbox_output, &dbox_msg_hdr,
+			  sizeof(dbox_msg_hdr)) < 0) {
+		mail_set_critical(_ctx->dest_mail, "write(%s) failed: %s",
+				  o_stream_get_name(ctx->dbox_output),
+				  o_stream_get_error(ctx->dbox_output));
+		ctx->failed = TRUE;
+	}
+	_ctx->data.output = ctx->dbox_output;
+
+	if (_ctx->data.received_date == (time_t)-1)
+		_ctx->data.received_date = ioloop_time;
+	index_attachment_save_begin(_ctx, storage->attachment_fs, ctx->input);
+}
+
+int dbox_save_continue(struct mail_save_context *_ctx)
+{
+	struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx);
+
+	if (ctx->failed)
+		return -1;
+
+	if (_ctx->data.attach != NULL)
+		return index_attachment_save_continue(_ctx);
+
+	if (index_storage_save_continue(_ctx, ctx->input,
+					_ctx->dest_mail) < 0) {
+		ctx->failed = TRUE;
+		return -1;
+	}
+	return 0;
+}
+
+void dbox_save_end(struct dbox_save_context *ctx)
+{
+	struct mail_save_data *mdata = &ctx->ctx.data;
+	struct ostream *dbox_output = ctx->dbox_output;
+	int ret;
+
+	i_assert(mdata->output != NULL);
+
+	if (mdata->attach != NULL && !ctx->failed) {
+		if (index_attachment_save_finish(&ctx->ctx) < 0)
+			ctx->failed = TRUE;
+	}
+	if (mdata->output != dbox_output) {
+		/* e.g. zlib plugin had changed this. make sure we
+		   successfully write the trailer. */
+		ret = o_stream_finish(mdata->output);
+	} else {
+		/* no plugins - flush the output so far */
+		ret = o_stream_flush(mdata->output);
+	}
+	if (ret < 0) {
+		mail_set_critical(ctx->ctx.dest_mail,
+				  "write(%s) failed: %s",
+				  o_stream_get_name(mdata->output),
+				  o_stream_get_error(mdata->output));
+		ctx->failed = TRUE;
+	}
+	if (mdata->output != dbox_output) {
+		o_stream_ref(dbox_output);
+		o_stream_destroy(&mdata->output);
+		mdata->output = dbox_output;
+	}
+	index_mail_cache_parse_deinit(ctx->ctx.dest_mail,
+				      ctx->ctx.data.received_date,
+				      !ctx->failed);
+	if (!ctx->failed)
+		index_mail_cache_pop3_data(ctx->ctx.dest_mail,
+					   mdata->pop3_uidl,
+					   mdata->pop3_order);
+}
+
+void dbox_save_write_metadata(struct mail_save_context *_ctx,
+			      struct ostream *output, uoff_t output_msg_size,
+			      const char *orig_mailbox_name,
+			      guid_128_t guid_128)
+{
+	struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx);
+	struct mail_save_data *mdata = &ctx->ctx.data;
+	struct dbox_metadata_header metadata_hdr;
+	const char *guid;
+	string_t *str;
+	uoff_t vsize;
+
+	i_zero(&metadata_hdr);
+	memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST,
+	       sizeof(metadata_hdr.magic_post));
+	o_stream_nsend(output, &metadata_hdr, sizeof(metadata_hdr));
+
+	str = t_str_new(256);
+	if (output_msg_size != ctx->input->v_offset) {
+		/* a plugin changed the data written to disk, so the
+		   "message size" dbox header doesn't contain the actual
+		   "physical" message size. we need to save it as a
+		   separate metadata header. */
+		str_printfa(str, "%c%llx\n", DBOX_METADATA_PHYSICAL_SIZE,
+			    (unsigned long long)ctx->input->v_offset);
+	}
+	str_printfa(str, "%c%"PRIxTIME_T"\n", DBOX_METADATA_RECEIVED_TIME,
+		    mdata->received_date);
+	if (mail_get_virtual_size(_ctx->dest_mail, &vsize) < 0)
+		i_unreached();
+	str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE,
+		    (unsigned long long)vsize);
+	if (mdata->pop3_uidl != NULL) {
+		i_assert(strchr(mdata->pop3_uidl, '\n') == NULL);
+		str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL,
+			    mdata->pop3_uidl);
+		ctx->have_pop3_uidls = TRUE;
+		ctx->highest_pop3_uidl_seq =
+			I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq);
+	}
+	if (mdata->pop3_order != 0) {
+		str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER,
+			    mdata->pop3_order);
+		ctx->have_pop3_orders = TRUE;
+		ctx->highest_pop3_uidl_seq =
+			I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq);
+	}
+
+	guid = mdata->guid;
+	if (guid != NULL)
+		mail_generate_guid_128_hash(guid, guid_128);
+	else {
+		guid_128_generate(guid_128);
+		guid = guid_128_to_string(guid_128);
+	}
+	str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid);
+
+	if (orig_mailbox_name != NULL &&
+	    strchr(orig_mailbox_name, '\r') == NULL &&
+	    strchr(orig_mailbox_name, '\n') == NULL) {
+		/* save the original mailbox name so if mailbox indexes get
+		   corrupted we can place at least some (hopefully most) of
+		   the messages to correct mailboxes. */
+		str_printfa(str, "%c%s\n", DBOX_METADATA_ORIG_MAILBOX,
+			    orig_mailbox_name);
+	}
+
+	dbox_attachment_save_write_metadata(_ctx, str);
+
+	str_append_c(str, '\n');
+	o_stream_nsend(output, str_data(str), str_len(str));
+}
+
+void dbox_save_update_header_flags(struct dbox_save_context *ctx,
+				   struct mail_index_view *sync_view,
+				   uint32_t ext_id,
+				   unsigned int flags_offset)
+{
+	const void *data;
+	size_t data_size;
+	uint8_t old_flags = 0, flags;
+
+	mail_index_get_header_ext(sync_view, ext_id, &data, &data_size);
+	if (flags_offset < data_size)
+		old_flags = *((const uint8_t *)data + flags_offset);
+	else {
+		/* grow old dbox header */
+		mail_index_ext_resize_hdr(ctx->trans, ext_id, flags_offset+1);
+	}
+
+	flags = old_flags;
+	if (ctx->have_pop3_uidls)
+		flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS;
+	if (ctx->have_pop3_orders)
+		flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS;
+	if (flags != old_flags) {
+		/* flags changed, update them */
+		mail_index_update_header_ext(ctx->trans, ext_id,
+					     flags_offset, &flags, 1);
+	}
+}
diff --git a/src/lib-storage/index/dbox-common/dbox-save.h b/src/lib-storage/index/dbox-common/dbox-save.h
new file mode 100644
index 0000000..a17c923
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-save.h
@@ -0,0 +1,41 @@
+#ifndef DBOX_SAVE_H
+#define DBOX_SAVE_H
+
+#include "dbox-storage.h"
+
+struct dbox_save_context {
+	struct mail_save_context ctx;
+	struct mail_index_transaction *trans;
+
+	/* updated for each appended mail: */
+	uint32_t seq;
+	struct istream *input;
+
+	struct ostream *dbox_output;
+
+	uint32_t highest_pop3_uidl_seq;
+	bool failed:1;
+	bool finished:1;
+	bool have_pop3_uidls:1;
+	bool have_pop3_orders:1;
+};
+
+#define DBOX_SAVECTX(s)		container_of(s, struct dbox_save_context, ctx)
+
+void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input);
+int dbox_save_continue(struct mail_save_context *_ctx);
+void dbox_save_end(struct dbox_save_context *ctx);
+
+void dbox_save_write_metadata(struct mail_save_context *ctx,
+			      struct ostream *output, uoff_t output_msg_size,
+			      const char *orig_mailbox_name,
+			      guid_128_t guid_128_r) ATTR_NULL(4);
+
+void dbox_save_add_to_index(struct dbox_save_context *ctx);
+
+void dbox_save_update_header_flags(struct dbox_save_context *ctx,
+				   struct mail_index_view *sync_view,
+				   uint32_t ext_id,
+				   unsigned int flags_offset);
+
+#endif
diff --git a/src/lib-storage/index/dbox-common/dbox-storage.c b/src/lib-storage/index/dbox-common/dbox-storage.c
new file mode 100644
index 0000000..5cac0d6
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-storage.c
@@ -0,0 +1,416 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "path-util.h"
+#include "ioloop.h"
+#include "fs-api.h"
+#include "mkdir-parents.h"
+#include "unlink-old-files.h"
+#include "mailbox-uidvalidity.h"
+#include "mailbox-list-private.h"
+#include "index-storage.h"
+#include "dbox-storage.h"
+
+#include <stdio.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <utime.h>
+
+void dbox_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED,
+				    struct mailbox_list_settings *set)
+{
+	if (set->layout == NULL)
+		set->layout = MAILBOX_LIST_NAME_FS;
+	if (set->subscription_fname == NULL)
+		set->subscription_fname = DBOX_SUBSCRIPTION_FILE_NAME;
+	if (*set->maildir_name == '\0')
+		set->maildir_name = DBOX_MAILDIR_NAME;
+	if (*set->mailbox_dir_name == '\0')
+		set->mailbox_dir_name = DBOX_MAILBOX_DIR_NAME;
+}
+
+static bool
+dbox_alt_path_has_changed(const char *root_dir, const char *alt_path,
+			  const char *alt_path2, const char *alt_symlink_path)
+{
+	const char *linkpath, *error;
+
+	if (t_readlink(alt_symlink_path, &linkpath, &error) < 0) {
+		if (errno == ENOENT)
+			return alt_path != NULL;
+		i_error("t_readlink(%s) failed: %s", alt_symlink_path, error);
+		return FALSE;
+	}
+
+	if (alt_path == NULL) {
+		i_warning("dbox %s: Original ALT=%s, "
+			  "but currently no ALT path set", root_dir, linkpath);
+		return TRUE;
+	} else if (strcmp(linkpath, alt_path) != 0) {
+		if (strcmp(linkpath, alt_path2) == 0) {
+			/* FIXME: for backwards compatibility. old versions
+			   created the symlink to mailboxes/ directory, which
+			   was fine with sdbox, but didn't even exist with
+			   mdbox. we'll silently replace the symlink. */
+			return TRUE;
+		}
+		i_warning("dbox %s: Original ALT=%s, "
+			  "but currently ALT=%s", root_dir, linkpath, alt_path);
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static void dbox_verify_alt_path(struct mailbox_list *list)
+{
+	const char *root_dir, *alt_symlink_path, *alt_path, *alt_path2;
+
+	root_dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_DIR);
+	alt_symlink_path =
+		t_strconcat(root_dir, "/"DBOX_ALT_SYMLINK_NAME, NULL);
+	(void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_DIR,
+					 &alt_path);
+	(void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX,
+					 &alt_path2);
+	if (!dbox_alt_path_has_changed(root_dir, alt_path, alt_path2,
+				       alt_symlink_path))
+		return;
+
+	/* unlink/create the current alt path symlink */
+	i_unlink_if_exists(alt_symlink_path);
+	if (alt_path != NULL) {
+		int ret = symlink(alt_path, alt_symlink_path);
+		if (ret < 0 && errno == ENOENT) {
+			/* root_dir doesn't exist yet - create it */
+			if (mailbox_list_mkdir_root(list, root_dir,
+					MAILBOX_LIST_PATH_TYPE_DIR) < 0)
+				return;
+			ret = symlink(alt_path, alt_symlink_path);
+		}
+		if (ret < 0 && errno != EEXIST) {
+			i_error("symlink(%s, %s) failed: %m",
+				alt_path, alt_symlink_path);
+		}
+	}
+}
+
+int dbox_storage_create(struct mail_storage *_storage,
+			struct mail_namespace *ns,
+			const char **error_r)
+{
+	struct dbox_storage *storage = DBOX_STORAGE(_storage);
+	const struct mail_storage_settings *set = _storage->set;
+	const char *error;
+
+	if (*set->mail_attachment_fs != '\0' &&
+	    *set->mail_attachment_dir != '\0') {
+		const char *name, *args, *dir;
+
+		args = strpbrk(set->mail_attachment_fs, ": ");
+		if (args == NULL) {
+			name = set->mail_attachment_fs;
+			args = "";
+		} else {
+			name = t_strdup_until(set->mail_attachment_fs, args++);
+		}
+		if (strcmp(name, "sis-queue") == 0 &&
+		    (_storage->class_flags & MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG) != 0) {
+			/* FIXME: the deduplication part doesn't work, because
+			   sdbox renames the files.. */
+			*error_r = "mail_attachment_fs: "
+				"sis-queue not currently supported by sdbox";
+			return -1;
+		}
+		dir = mail_user_home_expand(_storage->user,
+					    set->mail_attachment_dir);
+		storage->attachment_dir = p_strdup(_storage->pool, dir);
+
+		if (mailbox_list_init_fs(ns->list, _storage->event, name, args,
+					 storage->attachment_dir,
+					 &storage->attachment_fs, &error) < 0) {
+			*error_r = t_strdup_printf("mail_attachment_fs: %s",
+						   error);
+			return -1;
+		}
+	}
+
+	if (!ns->list->set.alt_dir_nocheck)
+		dbox_verify_alt_path(ns->list);
+	return 0;
+}
+
+void dbox_storage_destroy(struct mail_storage *_storage)
+{
+	struct dbox_storage *storage = DBOX_STORAGE(_storage);
+
+	fs_deinit(&storage->attachment_fs);
+	index_storage_destroy(_storage);
+}
+
+uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list)
+{
+	const char *path;
+
+	path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL);
+	path = t_strconcat(path, "/"DBOX_UIDVALIDITY_FILE_NAME, NULL);
+	return mailbox_uidvalidity_next(list, path);
+}
+
+void dbox_notify_changes(struct mailbox *box)
+{
+	const char *dir, *path;
+
+	if (box->notify_callback == NULL)
+		mailbox_watch_remove_all(box);
+	else {
+		if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX,
+					&dir) <= 0)
+			return;
+		path = t_strdup_printf("%s/"MAIL_INDEX_PREFIX".log", dir);
+		mailbox_watch_add(box, path);
+	}
+}
+
+static bool
+dbox_cleanup_temp_files(struct mailbox_list *list, const char *path,
+			time_t last_scan_time, time_t last_change_time)
+{
+	unsigned int interval = list->mail_set->mail_temp_scan_interval;
+
+	/* check once in a while if there are temp files to clean up */
+	if (interval == 0) {
+		/* disabled */
+		return FALSE;
+	} else if (last_scan_time >= ioloop_time - (time_t)interval) {
+		/* not the time to scan it yet */
+		return FALSE;
+	} else {
+		bool stated = FALSE;
+		if (last_change_time == (time_t)-1) {
+			/* Don't know the ctime yet - look it up. */
+			struct stat st;
+
+			if (stat(path, &st) < 0) {
+				if (errno == ENOENT)
+					i_error("stat(%s) failed: %m", path);
+				return FALSE;
+			}
+			last_change_time = st.st_ctime;
+			stated = TRUE;
+		}
+		if (last_scan_time > last_change_time + DBOX_TMP_DELETE_SECS) {
+			/* there haven't been any changes to this directory
+			   since we last checked it. If we did an extra stat(),
+			   we need to update the last_scan_time to avoid
+			   stat()ing the next time. */
+			return stated;
+		}
+		const char *prefix =
+			mailbox_list_get_global_temp_prefix(list);
+		(void)unlink_old_files(path, prefix,
+				       ioloop_time - DBOX_TMP_DELETE_SECS);
+		return TRUE;
+	}
+	return FALSE;
+}
+
+int dbox_mailbox_check_existence(struct mailbox *box, time_t *path_ctime_r)
+{
+	const char *index_path, *box_path = mailbox_get_path(box);
+	struct stat st;
+	int ret = -1;
+
+	*path_ctime_r = (time_t)-1;
+
+	if (box->list->set.iter_from_index_dir) {
+		/* Just because the index directory exists, it doesn't mean
+		   that the mailbox is selectable. Check that by seeing if
+		   dovecot.index.log exists. If it doesn't, fallback to
+		   checking for the dbox-Mails in the mail root directory.
+		   So this also means that if a mailbox is \NoSelect, listing
+		   it will always do a stat() for dbox-Mails in the mail root
+		   directory. That's not ideal, but this makes the behavior
+		   safer and \NoSelect mailboxes are somewhat rare. */
+		if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX,
+					&index_path) < 0)
+			return -1;
+		i_assert(index_path != NULL);
+		index_path = t_strconcat(index_path, "/", box->index_prefix,
+					 ".log", NULL);
+		ret = stat(index_path, &st);
+	}
+	if (ret < 0) {
+		ret = stat(box_path, &st);
+		if (ret == 0)
+			*path_ctime_r = st.st_ctime;
+	}
+
+	if (ret == 0) {
+		return 0;
+	} else if (errno == ENOENT || errno == ENAMETOOLONG) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+			T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname));
+		return -1;
+	} else if (errno == EACCES) {
+		mailbox_set_critical(box, "%s",
+			mail_error_eacces_msg("stat", box_path));
+		return -1;
+	} else {
+		mailbox_set_critical(box, "stat(%s) failed: %m", box_path);
+		return -1;
+	}
+}
+
+int dbox_mailbox_open(struct mailbox *box, time_t path_ctime)
+{
+	const char *box_path = mailbox_get_path(box);
+
+	if (index_storage_mailbox_open(box, FALSE) < 0)
+		return -1;
+	mail_index_set_fsync_mode(box->index,
+				  box->storage->set->parsed_fsync_mode,
+				  MAIL_INDEX_FSYNC_MASK_APPENDS |
+				  MAIL_INDEX_FSYNC_MASK_EXPUNGES);
+
+	const struct mail_index_header *hdr = mail_index_get_header(box->view);
+	if (dbox_cleanup_temp_files(box->list, box_path,
+				    hdr->last_temp_file_scan, path_ctime)) {
+		/* temp files were scanned. update the last scan timestamp. */
+		index_mailbox_update_last_temp_file_scan(box);
+	}
+	return 0;
+}
+
+static int dir_is_empty(struct mail_storage *storage, const char *path)
+{
+	DIR *dir;
+	struct dirent *d;
+	int ret = 1;
+
+	dir = opendir(path);
+	if (dir == NULL) {
+		if (errno == ENOENT) {
+			/* race condition with DELETE/RENAME? */
+			return 1;
+		}
+		mail_storage_set_critical(storage, "opendir(%s) failed: %m",
+					  path);
+		return -1;
+	}
+	while ((d = readdir(dir)) != NULL) {
+		if (*d->d_name == '.')
+			continue;
+
+		ret = 0;
+		break;
+	}
+	if (closedir(dir) < 0) {
+		mail_storage_set_critical(storage, "closedir(%s) failed: %m",
+					  path);
+		ret = -1;
+	}
+	return ret;
+}
+
+int dbox_mailbox_create(struct mailbox *box,
+			const struct mailbox_update *update, bool directory)
+{
+	struct dbox_storage *storage = DBOX_STORAGE(box->storage);
+	const char *alt_path;
+	struct stat st;
+	int ret;
+
+	if ((ret = index_storage_mailbox_create(box, directory)) <= 0)
+		return ret;
+	if (mailbox_open(box) < 0)
+		return -1;
+	if (mail_index_get_header(box->view)->uid_validity != 0 &&
+	    !box->storage->rebuilding_list_index) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS,
+				       "Mailbox already exists");
+		return -1;
+	}
+
+	/* if alt path already exists and contains files, rebuild storage so
+	   that we don't start overwriting files. */
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX, &alt_path);
+	if (ret > 0 && stat(alt_path, &st) == 0) {
+		ret = dir_is_empty(box->storage, alt_path);
+		if (ret < 0)
+			return -1;
+		if (ret == 0) {
+			mailbox_set_critical(box,
+				"Existing files in alt path, "
+				"rebuilding storage to avoid losing messages");
+			storage->v.set_mailbox_corrupted(box);
+			return -1;
+		}
+		/* dir is empty, ignore it */
+	}
+	return dbox_mailbox_create_indexes(box, update);
+}
+
+int dbox_mailbox_create_indexes(struct mailbox *box,
+				const struct mailbox_update *update)
+{
+	struct dbox_storage *storage = DBOX_STORAGE(box->storage);
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	int ret;
+
+	/* use syncing as a lock */
+	ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans, 0);
+	if (ret <= 0) {
+		i_assert(ret != 0);
+		mailbox_set_index_error(box);
+		return -1;
+	}
+
+	if (mail_index_get_header(view)->uid_validity == 0) {
+		if (storage->v.mailbox_create_indexes(box, update, trans) < 0) {
+			mail_index_sync_rollback(&sync_ctx);
+			return -1;
+		}
+	}
+
+	return mail_index_sync_commit(&sync_ctx);
+}
+
+int dbox_verify_alt_storage(struct mailbox_list *list)
+{
+	const char *alt_path;
+	struct stat st;
+
+	if (!mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_DIR,
+					&alt_path))
+		return 0;
+
+	/* make sure alt storage is mounted. if it's not, abort the rebuild. */
+	if (stat(alt_path, &st) == 0)
+		return 0;
+	if (errno != ENOENT) {
+		i_error("stat(%s) failed: %m", alt_path);
+		return -1;
+	}
+
+	/* try to create the alt directory. if it fails, it means alt
+	   storage isn't mounted. */
+	if (mailbox_list_mkdir_root(list, alt_path,
+				    MAILBOX_LIST_PATH_TYPE_ALT_DIR) < 0)
+		return -1;
+	return 0;
+}
+
+bool dbox_header_have_flag(struct mailbox *box, uint32_t ext_id,
+			   unsigned int flags_offset, uint8_t flag)
+{
+	const void *data;
+	size_t data_size;
+	uint8_t flags = 0;
+
+	mail_index_get_header_ext(box->view, ext_id, &data, &data_size);
+	if (flags_offset < data_size)
+		flags = *((const uint8_t *)data + flags_offset);
+	return (flags & flag) != 0;
+}
diff --git a/src/lib-storage/index/dbox-common/dbox-storage.h b/src/lib-storage/index/dbox-common/dbox-storage.h
new file mode 100644
index 0000000..af085df
--- /dev/null
+++ b/src/lib-storage/index/dbox-common/dbox-storage.h
@@ -0,0 +1,85 @@
+#ifndef DBOX_STORAGE_H
+#define DBOX_STORAGE_H
+
+#include "mail-storage-private.h"
+
+struct dbox_file;
+struct dbox_mail;
+struct dbox_storage;
+struct dbox_save_context;
+
+#define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions"
+#define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity"
+#define DBOX_TEMP_FILE_PREFIX ".temp."
+#define DBOX_ALT_SYMLINK_NAME "dbox-alt-root"
+
+#define DBOX_MAILBOX_DIR_NAME "mailboxes"
+#define DBOX_TRASH_DIR_NAME "trash"
+#define DBOX_MAILDIR_NAME "dbox-Mails"
+
+/* Delete temp files having ctime older than this. */
+#define DBOX_TMP_DELETE_SECS (36*60*60)
+
+/* Flag specifies if the message should be in primary or alternative storage */
+#define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND
+
+enum dbox_index_header_flags {
+	/* messages' metadata contain POP3 UIDLs */
+	DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS	= 0x01,
+	/* messages' metadata contain POP3 orders */
+	DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS	= 0x02
+};
+
+struct dbox_storage_vfuncs {
+	/* dbox file has zero references now. it should be either freed or
+	   left open in case it's accessed again soon */
+	void (*file_unrefed)(struct dbox_file *file);
+	/* create a new file using the same permissions as file.
+	   if parents=TRUE, create the directory if necessary */
+	int (*file_create_fd)(struct dbox_file *file, const char *path,
+			      bool parents);
+	/* open the mail and return its file/offset */
+	int (*mail_open)(struct dbox_mail *mail, uoff_t *offset_r,
+			 struct dbox_file **file_r);
+	/* create/update mailbox indexes */
+	int (*mailbox_create_indexes)(struct mailbox *box,
+				      const struct mailbox_update *update,
+				      struct mail_index_transaction *trans);
+	/* returns attachment path suffix. mdbox returns "", sdbox returns
+	   "-<mailbox_guid>-<uid>" */
+	const char *(*get_attachment_path_suffix)(struct dbox_file *file);
+	/* mark the mailbox corrupted */
+	void (*set_mailbox_corrupted)(struct mailbox *box);
+	/* mark the file corrupted */
+	void (*set_file_corrupted)(struct dbox_file *file);
+};
+
+struct dbox_storage {
+	struct mail_storage storage;
+	struct dbox_storage_vfuncs v;
+
+	struct fs *attachment_fs;
+	const char *attachment_dir;
+};
+
+#define DBOX_STORAGE(s)		container_of(s, struct dbox_storage, storage)
+
+void dbox_storage_get_list_settings(const struct mail_namespace *ns,
+				    struct mailbox_list_settings *set);
+int dbox_storage_create(struct mail_storage *storage,
+			struct mail_namespace *ns,
+			const char **error_r);
+void dbox_storage_destroy(struct mail_storage *storage);
+uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list);
+void dbox_notify_changes(struct mailbox *box);
+int dbox_mailbox_check_existence(struct mailbox *box, time_t *path_ctime_r);
+int dbox_mailbox_open(struct mailbox *box, time_t path_ctime);
+int dbox_mailbox_create(struct mailbox *box,
+			const struct mailbox_update *update, bool directory);
+int dbox_mailbox_create_indexes(struct mailbox *box,
+				const struct mailbox_update *update);
+int dbox_verify_alt_storage(struct mailbox_list *list);
+bool dbox_header_have_flag(struct mailbox *box, uint32_t ext_id,
+			   unsigned int flags_offset, uint8_t flag);
+
+#endif
diff --git a/src/lib-storage/index/dbox-multi/Makefile.am b/src/lib-storage/index/dbox-multi/Makefile.am
new file mode 100644
index 0000000..810edd2
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/Makefile.am
@@ -0,0 +1,36 @@
+noinst_LTLIBRARIES = libstorage_dbox_multi.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/lib-storage/index/dbox-common
+
+libstorage_dbox_multi_la_SOURCES = \
+	mdbox-deleted-storage.c \
+	mdbox-file.c \
+	mdbox-mail.c \
+	mdbox-map.c \
+	mdbox-purge.c \
+	mdbox-save.c \
+	mdbox-settings.c \
+	mdbox-sync.c \
+	mdbox-storage.c \
+	mdbox-storage-rebuild.c
+
+headers = \
+	mdbox-file.h \
+	mdbox-map.h \
+	mdbox-map-private.h \
+	mdbox-settings.h \
+	mdbox-storage.h \
+	mdbox-storage-rebuild.h \
+	mdbox-sync.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/dbox-multi/Makefile.in b/src/lib-storage/index/dbox-multi/Makefile.in
new file mode 100644
index 0000000..7e87b89
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/Makefile.in
@@ -0,0 +1,866 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/dbox-multi
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_dbox_multi_la_LIBADD =
+am_libstorage_dbox_multi_la_OBJECTS = mdbox-deleted-storage.lo \
+	mdbox-file.lo mdbox-mail.lo mdbox-map.lo mdbox-purge.lo \
+	mdbox-save.lo mdbox-settings.lo mdbox-sync.lo mdbox-storage.lo \
+	mdbox-storage-rebuild.lo
+libstorage_dbox_multi_la_OBJECTS =  \
+	$(am_libstorage_dbox_multi_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/mdbox-deleted-storage.Plo \
+	./$(DEPDIR)/mdbox-file.Plo ./$(DEPDIR)/mdbox-mail.Plo \
+	./$(DEPDIR)/mdbox-map.Plo ./$(DEPDIR)/mdbox-purge.Plo \
+	./$(DEPDIR)/mdbox-save.Plo ./$(DEPDIR)/mdbox-settings.Plo \
+	./$(DEPDIR)/mdbox-storage-rebuild.Plo \
+	./$(DEPDIR)/mdbox-storage.Plo ./$(DEPDIR)/mdbox-sync.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_dbox_multi_la_SOURCES)
+DIST_SOURCES = $(libstorage_dbox_multi_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_dbox_multi.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/lib-storage/index/dbox-common
+
+libstorage_dbox_multi_la_SOURCES = \
+	mdbox-deleted-storage.c \
+	mdbox-file.c \
+	mdbox-mail.c \
+	mdbox-map.c \
+	mdbox-purge.c \
+	mdbox-save.c \
+	mdbox-settings.c \
+	mdbox-sync.c \
+	mdbox-storage.c \
+	mdbox-storage-rebuild.c
+
+headers = \
+	mdbox-file.h \
+	mdbox-map.h \
+	mdbox-map-private.h \
+	mdbox-settings.h \
+	mdbox-storage.h \
+	mdbox-storage-rebuild.h \
+	mdbox-sync.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/dbox-multi/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/dbox-multi/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_dbox_multi.la: $(libstorage_dbox_multi_la_OBJECTS) $(libstorage_dbox_multi_la_DEPENDENCIES) $(EXTRA_libstorage_dbox_multi_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_dbox_multi_la_OBJECTS) $(libstorage_dbox_multi_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-deleted-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-file.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-map.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-purge.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-save.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-settings.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-storage-rebuild.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mdbox-sync.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/mdbox-deleted-storage.Plo
+	-rm -f ./$(DEPDIR)/mdbox-file.Plo
+	-rm -f ./$(DEPDIR)/mdbox-mail.Plo
+	-rm -f ./$(DEPDIR)/mdbox-map.Plo
+	-rm -f ./$(DEPDIR)/mdbox-purge.Plo
+	-rm -f ./$(DEPDIR)/mdbox-save.Plo
+	-rm -f ./$(DEPDIR)/mdbox-settings.Plo
+	-rm -f ./$(DEPDIR)/mdbox-storage-rebuild.Plo
+	-rm -f ./$(DEPDIR)/mdbox-storage.Plo
+	-rm -f ./$(DEPDIR)/mdbox-sync.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/mdbox-deleted-storage.Plo
+	-rm -f ./$(DEPDIR)/mdbox-file.Plo
+	-rm -f ./$(DEPDIR)/mdbox-mail.Plo
+	-rm -f ./$(DEPDIR)/mdbox-map.Plo
+	-rm -f ./$(DEPDIR)/mdbox-purge.Plo
+	-rm -f ./$(DEPDIR)/mdbox-save.Plo
+	-rm -f ./$(DEPDIR)/mdbox-settings.Plo
+	-rm -f ./$(DEPDIR)/mdbox-storage-rebuild.Plo
+	-rm -f ./$(DEPDIR)/mdbox-storage.Plo
+	-rm -f ./$(DEPDIR)/mdbox-sync.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c
new file mode 100644
index 0000000..4fdf1ab
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c
@@ -0,0 +1,319 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "mkdir-parents.h"
+#include "master-service.h"
+#include "mail-index-modseq.h"
+#include "mail-index-alloc-cache.h"
+#include "mailbox-log.h"
+#include "mailbox-list-private.h"
+#include "mail-copy.h"
+#include "dbox-mail.h"
+#include "dbox-save.h"
+#include "mdbox-map.h"
+#include "mdbox-file.h"
+#include "mdbox-sync.h"
+#include "mdbox-storage-rebuild.h"
+#include "mdbox-storage.h"
+
+extern struct mail_storage mdbox_deleted_storage;
+extern struct mailbox mdbox_deleted_mailbox;
+extern struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs;
+
+static struct mail_storage *mdbox_deleted_storage_alloc(void)
+{
+	struct mdbox_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mdbox deleted storage", 2048);
+	storage = p_new(pool, struct mdbox_storage, 1);
+	storage->storage.v = mdbox_dbox_storage_vfuncs;
+	storage->storage.storage = mdbox_deleted_storage;
+	storage->storage.storage.pool = pool;
+	return &storage->storage.storage;
+}
+
+static struct mailbox *
+mdbox_deleted_mailbox_alloc(struct mail_storage *storage,
+			    struct mailbox_list *list,
+			    const char *vname, enum mailbox_flags flags)
+{
+	struct mdbox_mailbox *mbox;
+	pool_t pool;
+
+	flags |= MAILBOX_FLAG_READONLY | MAILBOX_FLAG_NO_INDEX_FILES;
+
+	pool = pool_alloconly_create("mdbox deleted mailbox", 1024*3);
+	mbox = p_new(pool, struct mdbox_mailbox, 1);
+	mbox->box = mdbox_deleted_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.mail_vfuncs = &mdbox_mail_vfuncs;
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
+
+	mbox->storage = MDBOX_STORAGE(storage);
+	return &mbox->box;
+}
+
+static int
+mdbox_deleted_mailbox_create_indexes(struct mailbox *box,
+				     const struct mailbox_update *update,
+				     struct mail_index_transaction *trans)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box);
+	struct mail_index_transaction *new_trans = NULL;
+	uint32_t uid_validity = ioloop_time;
+	uint32_t uid_next = 1;
+
+	if (update != NULL && update->uid_validity != 0)
+		uid_validity = update->uid_validity;
+
+	if (trans == NULL) {
+		new_trans = mail_index_transaction_begin(box->view, 0);
+		trans = new_trans;
+	}
+
+	mail_index_update_header(trans,
+		offsetof(struct mail_index_header, uid_validity),
+		&uid_validity, sizeof(uid_validity), TRUE);
+	mail_index_update_header(trans,
+		offsetof(struct mail_index_header, next_uid),
+		&uid_next, sizeof(uid_next), TRUE);
+	mbox->creating = TRUE;
+	mdbox_update_header(mbox, trans, update);
+	mbox->creating = FALSE;
+
+	if (new_trans != NULL) {
+		if (mail_index_transaction_commit(&new_trans) < 0) {
+			mailbox_set_index_error(box);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static const char *
+mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED)
+{
+	return "";
+}
+
+static int
+mdbox_deleted_mailbox_get_metadata(struct mailbox *box,
+				   enum mailbox_metadata_items items,
+				   struct mailbox_metadata *metadata_r)
+{
+	if (index_mailbox_get_metadata(box, items, metadata_r) < 0)
+		return -1;
+
+	if ((items & MAILBOX_METADATA_GUID) != 0)
+		guid_128_generate(metadata_r->guid);
+	return 0;
+}
+
+static struct mail_save_context *
+mdbox_deleted_save_alloc(struct mailbox_transaction_context *t)
+{
+	struct mail_save_context *ctx;
+
+	ctx = i_new(struct mail_save_context, 1);
+	ctx->transaction = t;
+	return ctx;
+}
+
+static int
+mdbox_deleted_save_begin(struct mail_save_context *ctx,
+			 struct istream *input ATTR_UNUSED)
+{
+	mail_storage_set_error(ctx->transaction->box->storage,
+		MAIL_ERROR_NOTPOSSIBLE, "mdbox_deleted doesn't support saving mails");
+	return -1;
+}
+
+static int
+mdbox_deleted_save_continue(struct mail_save_context *ctx ATTR_UNUSED)
+{
+	return -1;
+}
+
+static int mdbox_deleted_save_finish(struct mail_save_context *ctx)
+{
+	index_save_context_free(ctx);
+	return -1;
+}
+
+static void
+mdbox_deleted_save_cancel(struct mail_save_context *ctx)
+{
+	index_save_context_free(ctx);
+}
+
+static int mdbox_deleted_sync(struct mdbox_mailbox *mbox,
+			      enum mdbox_sync_flags flags ATTR_UNUSED)
+{
+        struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *trans;
+	struct mdbox_mail_index_record rec;
+	struct mdbox_map_mail_index_record map_rec;
+	enum mail_index_sync_flags sync_flags;
+	uint16_t refcount;
+	uint32_t map_seq, map_count, seq, uid = 0;
+	int ret = 0;
+
+	if (mbox->mdbox_deleted_synced) {
+		/* don't bother supporting incremental syncs */
+		return 0;
+	}
+	if (!mbox->box.inbox_user && mbox->box.name[0] != '\0') {
+		/* since mailbox list currently shows all the existing
+		   mailboxes, we don't want all of them to list the deleted
+		   messages. only show messages in user's INBOX or the
+		   namespace prefix. */
+		return 0;
+	}
+
+	if (mdbox_map_open(mbox->storage->map) < 0)
+		return -1;
+
+	if (mdbox_deleted_mailbox_create_indexes(&mbox->box, NULL, NULL) < 0)
+		return -1;
+
+	i_zero(&rec);
+	rec.save_date = ioloop_time;
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box);
+	if (mail_index_sync_begin(mbox->box.index, &index_sync_ctx,
+				  &sync_view, &trans, sync_flags) < 0) {
+		mailbox_set_index_error(&mbox->box);
+		return -1;
+	}
+
+	map_count = mdbox_map_get_messages_count(mbox->storage->map);
+	for (map_seq = 1; map_seq <= map_count; map_seq++) {
+		if (mdbox_map_lookup_seq_full(mbox->storage->map, map_seq,
+					      &map_rec, &refcount) < 0) {
+			ret = -1;
+			break;
+		}
+		if (refcount == 0) {
+			rec.map_uid = mdbox_map_lookup_uid(mbox->storage->map,
+							   map_seq);
+			mail_index_append(trans, ++uid, &seq);
+			mail_index_update_ext(trans, seq,
+					      mbox->ext_id, &rec, NULL);
+		}
+	}
+
+	if (ret < 0)
+		mail_index_sync_rollback(&index_sync_ctx);
+	else {
+		if (mail_index_sync_commit(&index_sync_ctx) < 0) {
+			mailbox_set_index_error(&mbox->box);
+			ret = -1;
+		} else {
+			mbox->mdbox_deleted_synced = TRUE;
+		}
+	}
+	return ret;
+}
+
+static struct mailbox_sync_context *
+mdbox_deleted_storage_sync_init(struct mailbox *box,
+				enum mailbox_sync_flags flags)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box);
+	enum mdbox_sync_flags mdbox_sync_flags = 0;
+	int ret = 0;
+
+	if (index_mailbox_want_full_sync(&mbox->box, flags) ||
+	    mbox->storage->corrupted)
+		ret = mdbox_deleted_sync(mbox, mdbox_sync_flags);
+
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
+
+struct mail_storage mdbox_deleted_storage = {
+	.name = MDBOX_DELETED_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_BINARY_DATA,
+
+	.v = {
+                mdbox_get_setting_parser_info,
+		mdbox_deleted_storage_alloc,
+		mdbox_storage_create,
+		mdbox_storage_destroy,
+		NULL,
+		dbox_storage_get_list_settings,
+		NULL,
+		mdbox_deleted_mailbox_alloc,
+		NULL,
+		NULL,
+	}
+};
+
+struct mailbox mdbox_deleted_mailbox = {
+	.v = {
+		index_storage_is_readonly,
+		index_storage_mailbox_enable,
+		index_storage_mailbox_exists,
+		mdbox_mailbox_open,
+		index_storage_mailbox_close,
+		index_storage_mailbox_free,
+		dbox_mailbox_create,
+		index_storage_mailbox_update,
+		index_storage_mailbox_delete,
+		index_storage_mailbox_rename,
+		index_storage_get_status,
+		mdbox_deleted_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		index_storage_list_index_has_changed,
+		index_storage_list_index_update_sync,
+		mdbox_deleted_storage_sync_init,
+		index_mailbox_sync_next,
+		index_mailbox_sync_deinit,
+		NULL,
+		dbox_notify_changes,
+		index_transaction_begin,
+		index_transaction_commit,
+		index_transaction_rollback,
+		NULL,
+		dbox_mail_alloc,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		mdbox_deleted_save_alloc,
+		mdbox_deleted_save_begin,
+		mdbox_deleted_save_continue,
+		mdbox_deleted_save_finish,
+		mdbox_deleted_save_cancel,
+		mail_storage_copy,
+		NULL,
+		NULL,
+		NULL,
+		index_storage_is_inconsistent
+	}
+};
+
+struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs = {
+	mdbox_file_unrefed,
+	mdbox_file_create_fd,
+	mdbox_mail_open,
+	mdbox_deleted_mailbox_create_indexes,
+	mdbox_get_attachment_path_suffix,
+	mdbox_set_mailbox_corrupted,
+	mdbox_set_file_corrupted
+};
diff --git a/src/lib-storage/index/dbox-multi/mdbox-file.c b/src/lib-storage/index/dbox-multi/mdbox-file.c
new file mode 100644
index 0000000..65138bd
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-file.c
@@ -0,0 +1,349 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "hex-dec.h"
+#include "hex-binary.h"
+#include "hostpid.h"
+#include "istream.h"
+#include "ostream.h"
+#include "file-lock.h"
+#include "file-set-size.h"
+#include "mkdir-parents.h"
+#include "fdatasync-path.h"
+#include "eacces-error.h"
+#include "str.h"
+#include "mailbox-list-private.h"
+#include "mdbox-storage.h"
+#include "mdbox-map-private.h"
+#include "mdbox-file.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <fcntl.h>
+
+static struct mdbox_file *
+mdbox_find_and_move_open_file(struct mdbox_storage *storage, uint32_t file_id)
+{
+	struct mdbox_file *const *files;
+	unsigned int i, count;
+
+	files = array_get(&storage->open_files, &count);
+	for (i = 0; i < count; i++) {
+		if (files[i]->file_id == file_id)
+			return files[i];
+	}
+	return NULL;
+}
+
+void mdbox_files_free(struct mdbox_storage *storage)
+{
+	struct mdbox_file *const *files;
+	unsigned int i, count;
+
+	files = array_get(&storage->open_files, &count);
+	for (i = 0; i < count; i++)
+		dbox_file_free(&files[i]->file);
+	array_clear(&storage->open_files);
+}
+
+void mdbox_files_sync_input(struct mdbox_storage *storage)
+{
+	struct mdbox_file *const *files;
+	unsigned int i, count;
+
+	files = array_get(&storage->open_files, &count);
+	for (i = 0; i < count; i++) {
+		if (files[i]->file.input != NULL)
+			i_stream_sync(files[i]->file.input);
+	}
+}
+
+static void
+mdbox_close_open_files(struct mdbox_storage *storage, unsigned int close_count)
+{
+	struct mdbox_file *const *files;
+	unsigned int i, count;
+
+	files = array_get(&storage->open_files, &count);
+	for (i = 0; i < count;) {
+		if (files[i]->file.refcount == 0) {
+			dbox_file_free(&files[i]->file);
+			array_delete(&storage->open_files, i, 1);
+
+			if (--close_count == 0)
+				break;
+
+			files = array_get(&storage->open_files, &count);
+		} else {
+			i++;
+		}
+	}
+}
+
+static void
+mdbox_file_init_paths(struct mdbox_file *file, const char *fname, bool alt)
+{
+	i_free(file->file.primary_path);
+	i_free(file->file.alt_path);
+	file->file.primary_path =
+		i_strdup_printf("%s/%s", file->storage->storage_dir, fname);
+	if (file->storage->alt_storage_dir != NULL) {
+		file->file.alt_path =
+			i_strdup_printf("%s/%s", file->storage->alt_storage_dir,
+					fname);
+	}
+	file->file.cur_path = !alt ? file->file.primary_path :
+		file->file.alt_path;
+}
+
+static int mdbox_file_create(struct mdbox_file *file)
+{
+	struct dbox_file *_file = &file->file;
+	bool create_parents;
+	int ret;
+
+	create_parents = dbox_file_is_in_alt(_file);
+	_file->fd = _file->storage->v.
+		file_create_fd(_file, _file->cur_path, create_parents);
+	if (_file->fd == -1)
+		return -1;
+
+	if (file->storage->preallocate_space) {
+		ret = file_preallocate(_file->fd,
+				       file->storage->set->mdbox_rotate_size);
+		if (ret < 0) {
+			switch (errno) {
+			case ENOSPC:
+			case EDQUOT:
+				/* ignore */
+				break;
+			default:
+				i_error("file_preallocate(%s) failed: %m",
+					_file->cur_path);
+				break;
+			}
+		} else if (ret == 0) {
+			/* not supported by filesystem, disable. */
+			file->storage->preallocate_space = FALSE;
+		}
+	}
+	return 0;
+}
+
+static struct dbox_file *
+mdbox_file_init_full(struct mdbox_storage *storage,
+		     uint32_t file_id, bool alt_dir)
+{
+	struct mdbox_file *file;
+	const char *fname;
+	unsigned int count;
+
+	file = file_id == 0 ? NULL :
+		mdbox_find_and_move_open_file(storage, file_id);
+	if (file != NULL) {
+		file->file.refcount++;
+		return &file->file;
+	}
+
+	count = array_count(&storage->open_files);
+	if (count > MDBOX_MAX_OPEN_UNUSED_FILES) {
+		mdbox_close_open_files(storage,
+				       count - MDBOX_MAX_OPEN_UNUSED_FILES);
+	}
+
+	file = i_new(struct mdbox_file, 1);
+	file->storage = storage;
+	file->file.storage = &storage->storage;
+	file->file_id = file_id;
+	fname = file_id == 0 ? dbox_generate_tmp_filename() :
+		t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id);
+	mdbox_file_init_paths(file, fname, FALSE);
+	dbox_file_init(&file->file);
+	if (alt_dir)
+		file->file.cur_path = file->file.alt_path;
+
+	if (file_id != 0)
+		array_push_back(&storage->open_files, &file);
+	else
+		(void)mdbox_file_create(file);
+	return &file->file;
+}
+
+struct dbox_file *
+mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id)
+{
+	return mdbox_file_init_full(storage, file_id, FALSE);
+}
+
+struct dbox_file *
+mdbox_file_init_new_alt(struct mdbox_storage *storage)
+{
+	return mdbox_file_init_full(storage, 0, TRUE);
+}
+
+int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id)
+{
+	struct stat st;
+	const char *old_path;
+	const char *new_dir, *new_fname, *new_path;
+
+	i_assert(file->file_id == 0);
+	i_assert(file_id != 0);
+
+	old_path = file->file.cur_path;
+	new_fname = t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id);
+	new_dir = !dbox_file_is_in_alt(&file->file) ?
+		file->storage->storage_dir : file->storage->alt_storage_dir;
+	new_path = t_strdup_printf("%s/%s", new_dir, new_fname);
+
+	if (stat(new_path, &st) == 0) {
+		mail_storage_set_critical(&file->file.storage->storage,
+			"mdbox: %s already exists, rebuilding index", new_path);
+		mdbox_storage_set_corrupted(file->storage);
+		return -1;
+	}
+	if (rename(old_path, new_path) < 0) {
+		mail_storage_set_critical(&file->storage->storage.storage,
+					  "rename(%s, %s) failed: %m",
+					  old_path, new_path);
+		return -1;
+	}
+	mdbox_file_init_paths(file, new_fname,
+			      dbox_file_is_in_alt(&file->file));
+	file->file_id = file_id;
+	array_push_back(&file->storage->open_files, &file);
+	return 0;
+}
+
+static struct mdbox_file *
+mdbox_find_oldest_unused_file(struct mdbox_storage *storage,
+			      unsigned int *idx_r)
+{
+	struct mdbox_file *const *files, *oldest_file = NULL;
+	unsigned int i, count;
+
+	files = array_get(&storage->open_files, &count);
+	*idx_r = count;
+	for (i = 0; i < count; i++) {
+		if (files[i]->file.refcount == 0) {
+			if (oldest_file == NULL ||
+			    files[i]->close_time < oldest_file->close_time) {
+				oldest_file = files[i];
+				*idx_r = i;
+			}
+		}
+	}
+	return oldest_file;
+}
+
+static void mdbox_file_close_timeout(struct mdbox_storage *storage)
+{
+	struct mdbox_file *oldest;
+	unsigned int i;
+	time_t close_time = ioloop_time - MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS;
+
+	while ((oldest = mdbox_find_oldest_unused_file(storage, &i)) != NULL) {
+		if (oldest->close_time > close_time)
+			break;
+		array_delete(&storage->open_files, i, 1);
+		dbox_file_free(&oldest->file);
+	}
+
+	if (oldest == NULL)
+		timeout_remove(&storage->to_close_unused_files);
+}
+
+static void mdbox_file_close_later(struct mdbox_file *mfile)
+{
+	if (mfile->storage->to_close_unused_files == NULL) {
+		mfile->storage->to_close_unused_files =
+			timeout_add(MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS*1000,
+				    mdbox_file_close_timeout, mfile->storage);
+	}
+}
+
+void mdbox_file_unrefed(struct dbox_file *file)
+{
+	struct mdbox_file *mfile = (struct mdbox_file *)file;
+	struct mdbox_file *oldest_file;
+	unsigned int i, count;
+
+	/* don't cache metadata seeks while file isn't being referenced */
+	file->metadata_read_offset = UOFF_T_MAX;
+	mfile->close_time = ioloop_time;
+
+	if (mfile->file_id != 0) {
+		count = array_count(&mfile->storage->open_files);
+		if (count <= MDBOX_MAX_OPEN_UNUSED_FILES) {
+			/* we can leave this file open for now */
+			mdbox_file_close_later(mfile);
+			return;
+		}
+
+		/* close the oldest file with refcount=0 */
+		oldest_file = mdbox_find_oldest_unused_file(mfile->storage, &i);
+		i_assert(oldest_file != NULL);
+		array_delete(&mfile->storage->open_files, i, 1);
+		if (oldest_file != mfile) {
+			dbox_file_free(&oldest_file->file);
+			mdbox_file_close_later(mfile);
+			return;
+		}
+		/* have to close ourself */
+	}
+	dbox_file_free(file);
+}
+
+int mdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents)
+{
+	struct mdbox_file *mfile = (struct mdbox_file *)file;
+	struct mdbox_map *map = mfile->storage->map;
+	struct mailbox_permissions perm;
+	mode_t old_mask;
+	const char *p, *dir;
+	int fd;
+
+	mailbox_list_get_root_permissions(map->root_list, &perm);
+
+	old_mask = umask(0666 & ~perm.file_create_mode);
+	fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666);
+	umask(old_mask);
+	if (fd == -1 && errno == ENOENT && parents &&
+	    (p = strrchr(path, '/')) != NULL) {
+		dir = t_strdup_until(path, p);
+		if (mailbox_list_mkdir_root(map->root_list, dir,
+					    path != file->alt_path ?
+					    MAILBOX_LIST_PATH_TYPE_DIR :
+					    MAILBOX_LIST_PATH_TYPE_ALT_DIR) < 0) {
+			mail_storage_copy_list_error(&file->storage->storage,
+						     map->root_list);
+			return -1;
+		}
+		/* try again */
+		old_mask = umask(0666 & ~perm.file_create_mode);
+		fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666);
+		umask(old_mask);
+	}
+	if (fd == -1) {
+		mail_storage_set_critical(&file->storage->storage,
+			"open(%s, O_CREAT) failed: %m", path);
+	} else if (perm.file_create_gid == (gid_t)-1) {
+		/* no group change */
+	} else if (fchown(fd, (uid_t)-1, perm.file_create_gid) < 0) {
+		if (errno == EPERM) {
+			mail_storage_set_critical(&file->storage->storage, "%s",
+				eperm_error_get_chgrp("fchown", path,
+					perm.file_create_gid,
+					perm.file_create_gid_origin));
+		} else {
+			mail_storage_set_critical(&file->storage->storage,
+				"fchown(%s, -1, %ld) failed: %m",
+				path, (long)perm.file_create_gid);
+		}
+		/* continue anyway */
+	}
+	return fd;
+}
diff --git a/src/lib-storage/index/dbox-multi/mdbox-file.h b/src/lib-storage/index/dbox-multi/mdbox-file.h
new file mode 100644
index 0000000..9095cce
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-file.h
@@ -0,0 +1,29 @@
+#ifndef MDBOX_FILE_H
+#define MDBOX_FILE_H
+
+#include "dbox-file.h"
+
+struct mdbox_file {
+	struct dbox_file file;
+	struct mdbox_storage *storage;
+
+	uint32_t file_id;
+	time_t close_time;
+};
+
+struct dbox_file *
+mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id);
+struct dbox_file *
+mdbox_file_init_new_alt(struct mdbox_storage *storage);
+
+/* Assign file ID for a newly created file. */
+int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id);
+
+void mdbox_file_unrefed(struct dbox_file *file);
+int mdbox_file_create_fd(struct dbox_file *file, const char *path,
+			 bool parents);
+
+void mdbox_files_free(struct mdbox_storage *storage);
+void mdbox_files_sync_input(struct mdbox_storage *storage);
+
+#endif
diff --git a/src/lib-storage/index/dbox-multi/mdbox-mail.c b/src/lib-storage/index/dbox-multi/mdbox-mail.c
new file mode 100644
index 0000000..ed3fe06
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c
@@ -0,0 +1,265 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "str.h"
+#include "index-mail.h"
+#include "dbox-mail.h"
+#include "mdbox-storage.h"
+#include "mdbox-sync.h"
+#include "mdbox-map.h"
+#include "mdbox-file.h"
+
+#include <sys/stat.h>
+
+int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view,
+		      uint32_t seq, uint32_t *map_uid_r)
+{
+	const struct mdbox_mail_index_record *dbox_rec;
+	struct mdbox_index_header hdr;
+	const void *data;
+	uint32_t uid, cur_map_uid_validity;
+	bool need_resize;
+
+	mail_index_lookup_ext(view, seq, mbox->ext_id, &data, NULL);
+	dbox_rec = data;
+	if (dbox_rec == NULL || dbox_rec->map_uid == 0) {
+		mail_index_lookup_uid(view, seq, &uid);
+		mailbox_set_critical(&mbox->box,
+			"mdbox: map uid lost for uid %u", uid);
+		mdbox_storage_set_corrupted(mbox->storage);
+		return -1;
+	}
+
+	if (mbox->map_uid_validity == 0) {
+		if (mdbox_read_header(mbox, &hdr, &need_resize) < 0)
+			return -1;
+		mbox->map_uid_validity = hdr.map_uid_validity;
+	}
+	if (mdbox_map_open_or_create(mbox->storage->map) < 0)
+		return -1;
+
+	cur_map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map);
+	if (cur_map_uid_validity != mbox->map_uid_validity) {
+		mailbox_set_critical(&mbox->box,
+			"mdbox: map uidvalidity mismatch (%u vs %u)",
+			mbox->map_uid_validity, cur_map_uid_validity);
+		mdbox_storage_set_corrupted(mbox->storage);
+		return -1;
+	}
+	*map_uid_r = dbox_rec->map_uid;
+	return 0;
+}
+
+static void dbox_mail_set_expunged(struct dbox_mail *mail, uint32_t map_uid)
+{
+	struct mail *_mail = &mail->imail.mail.mail;
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->box);
+
+	mail_index_refresh(_mail->box->index);
+	if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
+		mail_set_expunged(_mail);
+		return;
+	}
+
+	mdbox_map_set_corrupted(mbox->storage->map,
+				"Unexpectedly lost %s uid=%u map_uid=%u",
+				mailbox_get_vname(_mail->box),
+				_mail->uid, map_uid);
+}
+
+static int dbox_mail_open_init(struct dbox_mail *mail, uint32_t map_uid)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->imail.mail.mail.box);
+	uint32_t file_id;
+	int ret;
+
+	if ((ret = mdbox_map_lookup(mbox->storage->map, map_uid,
+				    &file_id, &mail->offset)) <= 0) {
+		if (ret < 0)
+			return -1;
+
+		/* map_uid doesn't exist anymore. either it
+		   got just expunged or the map index is
+		   corrupted. */
+		dbox_mail_set_expunged(mail, map_uid);
+		return -1;
+	} else {
+		mail->open_file = mdbox_file_init(mbox->storage, file_id);
+	}
+	return 0;
+}
+
+int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r,
+		    struct dbox_file **file_r)
+{
+	struct mail *_mail = &mail->imail.mail.mail;
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->box);
+	uint32_t prev_file_id = 0, map_uid = 0;
+	bool deleted;
+
+	if (!mail_stream_access_start(_mail))
+		return -1;
+
+	do {
+		if (mail->open_file != NULL) {
+			/* already open */
+		} else if (!_mail->saving) {
+			if (mdbox_mail_lookup(mbox, _mail->transaction->view,
+					      _mail->seq, &map_uid) < 0)
+				return -1;
+			if (dbox_mail_open_init(mail, map_uid) < 0)
+				return -1;
+		} else {
+			/* mail is being saved in this transaction */
+			mail->open_file =
+				mdbox_save_file_get_file(_mail->transaction,
+							 _mail->seq,
+							 &mail->offset);
+		}
+
+		if (!dbox_file_is_open(mail->open_file))
+			_mail->transaction->stats.open_lookup_count++;
+		if (dbox_file_open(mail->open_file, &deleted) <= 0)
+			return -1;
+		if (deleted) {
+			/* either it's expunged now or moved to another file. */
+			struct mdbox_file *mfile =
+				(struct mdbox_file *)mail->open_file;
+
+			if (mfile->file_id == prev_file_id) {
+				dbox_mail_set_expunged(mail, map_uid);
+				return -1;
+			}
+			prev_file_id = mfile->file_id;
+			if (mdbox_map_refresh(mbox->storage->map) < 0)
+				return -1;
+			dbox_file_unref(&mail->open_file);
+		}
+	} while (mail->open_file == NULL);
+
+	*file_r = mail->open_file;
+	*offset_r = mail->offset;
+	return 0;
+}
+
+static int mdbox_mail_get_save_date(struct mail *mail, time_t *date_r)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->transaction->box);
+	const struct mdbox_mail_index_record *dbox_rec;
+	const void *data;
+
+	mail_index_lookup_ext(mail->transaction->view, mail->seq,
+			      mbox->ext_id, &data, NULL);
+	dbox_rec = data;
+	if (dbox_rec == NULL || dbox_rec->map_uid == 0) {
+		/* lost for some reason, use fallback */
+		return dbox_mail_get_save_date(mail, date_r);
+	}
+
+	*date_r = dbox_rec->save_date;
+	return 1;
+}
+
+static int
+mdbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+		       const char **value_r)
+{
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(_mail->transaction->box);
+	struct mdbox_map_mail_index_record rec;
+	uint32_t map_uid;
+	uint16_t refcount;
+
+	switch (field) {
+	case MAIL_FETCH_REFCOUNT:
+		if (mdbox_mail_lookup(mbox, _mail->transaction->view,
+				      _mail->seq, &map_uid) < 0)
+			return -1;
+		if (mdbox_map_lookup_full(mbox->storage->map, map_uid,
+					  &rec, &refcount) < 0)
+			return -1;
+		*value_r = p_strdup_printf(mail->imail.mail.data_pool, "%u",
+					   refcount);
+		return 0;
+	case MAIL_FETCH_REFCOUNT_ID:
+		if (mdbox_mail_lookup(mbox, _mail->transaction->view,
+				      _mail->seq, &map_uid) < 0)
+			return -1;
+		*value_r = p_strdup_printf(mail->imail.mail.data_pool, "%u",
+					   map_uid);
+		return 0;
+	case MAIL_FETCH_UIDL_BACKEND:
+		if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id,
+				offsetof(struct mdbox_index_header, flags),
+				DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) {
+			*value_r = "";
+			return 0;
+		}
+		break;
+	case MAIL_FETCH_POP3_ORDER:
+		if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id,
+				offsetof(struct mdbox_index_header, flags),
+				DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) {
+			*value_r = "";
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+	return dbox_mail_get_special(_mail, field, value_r);
+}
+
+static void
+mdbox_mail_update_flags(struct mail *mail, enum modify_type modify_type,
+			enum mail_flags flags)
+{
+	if ((flags & DBOX_INDEX_FLAG_ALT) != 0) {
+		mdbox_purge_alt_flag_change(mail, modify_type != MODIFY_REMOVE);
+		flags &= ENUM_NEGATE(DBOX_INDEX_FLAG_ALT);
+		if (flags == 0 && modify_type != MODIFY_REPLACE)
+			return;
+	}
+
+	index_mail_update_flags(mail, modify_type, flags);
+}
+
+struct mail_vfuncs mdbox_mail_vfuncs = {
+	dbox_mail_close,
+	index_mail_free,
+	index_mail_set_seq,
+	index_mail_set_uid,
+	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
+	index_mail_precache,
+	index_mail_add_temp_wanted_fields,
+
+	index_mail_get_flags,
+	index_mail_get_keywords,
+	index_mail_get_keyword_indexes,
+	index_mail_get_modseq,
+	index_mail_get_pvt_modseq,
+	index_mail_get_parts,
+	index_mail_get_date,
+	dbox_mail_get_received_date,
+	mdbox_mail_get_save_date,
+	dbox_mail_get_virtual_size,
+	dbox_mail_get_physical_size,
+	index_mail_get_first_header,
+	index_mail_get_headers,
+	index_mail_get_header_stream,
+	dbox_mail_get_stream,
+	index_mail_get_binary_stream,
+	mdbox_mail_get_special,
+	index_mail_get_backend_mail,
+	mdbox_mail_update_flags,
+	index_mail_update_keywords,
+	index_mail_update_modseq,
+	index_mail_update_pvt_modseq,
+	NULL,
+	index_mail_expunge,
+	index_mail_set_cache_corrupted,
+	index_mail_opened,
+};
diff --git a/src/lib-storage/index/dbox-multi/mdbox-map-private.h b/src/lib-storage/index/dbox-multi/mdbox-map-private.h
new file mode 100644
index 0000000..259d854
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-map-private.h
@@ -0,0 +1,64 @@
+#ifndef MDBOX_MAP_PRIVATE_H
+#define MDBOX_MAP_PRIVATE_H
+
+#include "mdbox-map.h"
+
+struct dbox_mail_lookup_rec {
+	uint32_t map_uid;
+	uint16_t refcount;
+	struct mdbox_map_mail_index_record rec;
+};
+
+struct mdbox_map {
+	struct mdbox_storage *storage;
+	const struct mdbox_settings *set;
+	char *path, *index_path;
+
+	struct mail_index *index;
+	struct mail_index_view *view;
+
+	uint32_t map_ext_id, ref_ext_id;
+
+	struct mailbox_list *root_list;
+
+	bool verify_existing_file_ids:1;
+};
+
+struct mdbox_map_append {
+	struct dbox_file_append_context *file_append;
+	uoff_t offset, size;
+};
+
+struct mdbox_map_append_context {
+	struct mdbox_map *map;
+	struct mdbox_map_atomic_context *atomic;
+	struct mail_index_transaction *trans;
+
+	ARRAY(struct dbox_file_append_context *) file_appends;
+	ARRAY(struct dbox_file *) files;
+	ARRAY(struct mdbox_map_append) appends;
+
+	uint32_t first_new_file_id;
+
+	unsigned int files_nonappendable_count;
+
+	bool failed:1;
+};
+
+struct mdbox_map_atomic_context {
+	struct mdbox_map *map;
+	struct mail_index_transaction *sync_trans;
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *sync_view;
+
+	bool map_refreshed:1;
+	bool locked:1;
+	bool success:1;
+	bool failed:1;
+};
+
+int mdbox_map_view_lookup_rec(struct mdbox_map *map,
+			      struct mail_index_view *view, uint32_t seq,
+			      struct dbox_mail_lookup_rec *rec_r);
+
+#endif
diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c
new file mode 100644
index 0000000..513db02
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.c
@@ -0,0 +1,1502 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "ostream.h"
+#include "mkdir-parents.h"
+#include "unlink-old-files.h"
+#include "mailbox-list-private.h"
+#include "mdbox-storage.h"
+#include "mdbox-file.h"
+#include "mdbox-map-private.h"
+
+#include <dirent.h>
+
+#define MAX_BACKWARDS_LOOKUPS 10
+
+#define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10)
+#define DBOX_FORCE_PURGE_MIN_RATIO 0.5
+
+#define MAP_STORAGE(map) (&(map)->storage->storage.storage)
+
+struct mdbox_map_transaction_context {
+	struct mdbox_map_atomic_context *atomic;
+	struct mail_index_transaction *trans;
+
+	bool changed:1;
+	bool committed:1;
+};
+
+static int mdbox_map_generate_uid_validity(struct mdbox_map *map);
+
+void mdbox_map_set_corrupted(struct mdbox_map *map, const char *format, ...)
+{
+	va_list args;
+
+	va_start(args, format);
+	mail_storage_set_critical(MAP_STORAGE(map),
+				  "mdbox map %s corrupted: %s",
+				  map->index->filepath,
+				  t_strdup_vprintf(format, args));
+	va_end(args);
+
+	mdbox_storage_set_corrupted(map->storage);
+}
+
+struct mdbox_map *
+mdbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list)
+{
+	struct mdbox_map *map;
+	const char *root, *index_root;
+
+	root = mailbox_list_get_root_forced(root_list, MAILBOX_LIST_PATH_TYPE_DIR);
+	index_root = mailbox_list_get_root_forced(root_list, MAILBOX_LIST_PATH_TYPE_INDEX);
+
+	map = i_new(struct mdbox_map, 1);
+	map->storage = storage;
+	map->set = storage->set;
+	map->path = i_strconcat(root, "/"MDBOX_GLOBAL_DIR_NAME, NULL);
+	map->index_path =
+		i_strconcat(index_root, "/"MDBOX_GLOBAL_DIR_NAME, NULL);
+	map->index = mail_index_alloc(storage->storage.storage.event,
+				      map->index_path,
+				      MDBOX_GLOBAL_INDEX_PREFIX);
+	mail_index_set_fsync_mode(map->index,
+		MAP_STORAGE(map)->set->parsed_fsync_mode, 0);
+	mail_index_set_lock_method(map->index,
+		MAP_STORAGE(map)->set->parsed_lock_method,
+		mail_storage_get_lock_timeout(MAP_STORAGE(map), UINT_MAX));
+	map->root_list = root_list;
+	map->map_ext_id = mail_index_ext_register(map->index, "map",
+				sizeof(struct mdbox_map_mail_index_header),
+				sizeof(struct mdbox_map_mail_index_record),
+				sizeof(uint32_t));
+	map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0,
+				sizeof(uint16_t), sizeof(uint16_t));
+	return map;
+}
+
+void mdbox_map_deinit(struct mdbox_map **_map)
+{
+	struct mdbox_map *map = *_map;
+
+	*_map = NULL;
+
+	if (map->view != NULL) {
+		mail_index_view_close(&map->view);
+		mail_index_close(map->index);
+	}
+	mail_index_free(&map->index);
+	i_free(map->index_path);
+	i_free(map->path);
+	i_free(map);
+}
+
+static int mdbox_map_mkdir_storage(struct mdbox_map *map)
+{
+	if (mailbox_list_mkdir_root(map->root_list, map->path,
+				    MAILBOX_LIST_PATH_TYPE_DIR) < 0) {
+		mail_storage_copy_list_error(MAP_STORAGE(map), map->root_list);
+		return -1;
+	}
+
+	if (strcmp(map->path, map->index_path) != 0 &&
+	    mailbox_list_mkdir_root(map->root_list, map->index_path,
+				    MAILBOX_LIST_PATH_TYPE_INDEX) < 0) {
+		mail_storage_copy_list_error(MAP_STORAGE(map), map->root_list);
+		return -1;
+	}
+	return 0;
+}
+
+static void mdbox_map_cleanup(struct mdbox_map *map)
+{
+	unsigned int interval =
+		MAP_STORAGE(map)->set->mail_temp_scan_interval;
+	struct stat st;
+
+	if (stat(map->path, &st) < 0)
+		return;
+
+	/* check once in a while if there are temp files to clean up */
+	if (interval == 0) {
+		/* disabled */
+	} else if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) {
+		/* there haven't been any changes to this directory since we
+		   last checked it. */
+	} else if (st.st_atime < ioloop_time - (time_t)interval) {
+		/* time to scan */
+		(void)unlink_old_files(map->path, DBOX_TEMP_FILE_PREFIX,
+				       ioloop_time - DBOX_TMP_DELETE_SECS);
+	}
+}
+
+static int mdbox_map_open_internal(struct mdbox_map *map, bool create_missing)
+{
+	enum mail_index_open_flags open_flags;
+	struct mailbox_permissions perm;
+	int ret = 0;
+
+	if (map->view != NULL) {
+		/* already opened */
+		return 1;
+	}
+
+	mailbox_list_get_root_permissions(map->root_list, &perm);
+	mail_index_set_permissions(map->index, perm.file_create_mode,
+				   perm.file_create_gid,
+				   perm.file_create_gid_origin);
+
+	open_flags = MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY |
+		mail_storage_settings_to_index_flags(MAP_STORAGE(map)->set);
+	if (create_missing) {
+		if ((ret = mdbox_map_mkdir_storage(map)) < 0)
+			return -1;
+		if (ret > 0) {
+			/* storage/ directory already existed.
+			   the index should exist also. */
+		} else {
+			open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
+		}
+	}
+	ret = mail_index_open(map->index, open_flags);
+	if (ret == 0 && create_missing) {
+		/* storage/ already existed, but indexes didn't. we'll need to
+		   take extra steps to make sure we won't overwrite any m.*
+		   files that may already exist. */
+		map->verify_existing_file_ids = TRUE;
+		open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
+		ret = mail_index_open(map->index, open_flags);
+	}
+	if (ret < 0) {
+		mail_storage_set_index_error(MAP_STORAGE(map), map->index);
+		return -1;
+	}
+	if (ret == 0) {
+		/* index not found - for now just return failure */
+		i_assert(!create_missing);
+		return 0;
+	}
+
+	map->view = mail_index_view_open(map->index);
+	mdbox_map_cleanup(map);
+
+	if (mail_index_get_header(map->view)->uid_validity == 0) {
+		if (mdbox_map_generate_uid_validity(map) < 0) {
+			mail_storage_set_index_error(MAP_STORAGE(map), map->index);
+			mail_index_close(map->index);
+			return -1;
+		}
+		if (mdbox_map_refresh(map) < 0) {
+			mail_index_close(map->index);
+			return -1;
+		}
+	}
+	return 1;
+}
+
+int mdbox_map_open(struct mdbox_map *map)
+{
+	return mdbox_map_open_internal(map, FALSE);
+}
+
+int mdbox_map_open_or_create(struct mdbox_map *map)
+{
+	return mdbox_map_open_internal(map, TRUE) <= 0 ? -1 : 0;
+}
+
+int mdbox_map_refresh(struct mdbox_map *map)
+{
+	struct mail_index_view_sync_ctx *ctx;
+	bool delayed_expunges, fscked;
+	int ret = 0;
+
+	/* some open files may have read partially written mails. now that
+	   map syncing makes the new mails visible, we need to make sure the
+	   partial data is flushed out of memory */
+	mdbox_files_sync_input(map->storage);
+
+	if (mail_index_refresh(map->view->index) < 0) {
+		mail_storage_set_index_error(MAP_STORAGE(map), map->index);
+		return -1;
+	}
+	if (mail_index_view_have_transactions(map->view)) {
+		/* can't sync when there are transactions */
+		return 0;
+	}
+
+	ctx = mail_index_view_sync_begin(map->view,
+				MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
+	fscked = mail_index_reset_fscked(map->view->index);
+	if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) {
+		mail_storage_set_index_error(MAP_STORAGE(map), map->index);
+		ret = -1;
+	}
+	if (fscked)
+		mdbox_storage_set_corrupted(map->storage);
+	return ret;
+}
+
+bool mdbox_map_is_fscked(struct mdbox_map *map)
+{
+	const struct mail_index_header *hdr;
+
+	if (map->view == NULL) {
+		/* map isn't opened yet. don't bother. */
+		return FALSE;
+	}
+
+	hdr = mail_index_get_header(map->view);
+	return (hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0;
+}
+
+static void
+mdbox_map_get_ext_hdr(struct mdbox_map *map, struct mail_index_view *view,
+		      struct mdbox_map_mail_index_header *hdr_r)
+{
+	const void *data;
+	size_t data_size;
+
+	mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
+	i_zero(hdr_r);
+	memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
+}
+
+uint32_t mdbox_map_get_rebuild_count(struct mdbox_map *map)
+{
+	struct mdbox_map_mail_index_header hdr;
+
+	mdbox_map_get_ext_hdr(map, map->view, &hdr);
+	return hdr.rebuild_count;
+}
+
+static int
+mdbox_map_lookup_seq(struct mdbox_map *map, uint32_t seq,
+		     const struct mdbox_map_mail_index_record **rec_r)
+{
+	const struct mdbox_map_mail_index_record *rec;
+	const void *data;
+	uint32_t uid;
+
+	mail_index_lookup_ext(map->view, seq, map->map_ext_id, &data, NULL);
+	rec = data;
+
+	if (rec == NULL || rec->file_id == 0) {
+		mail_index_lookup_uid(map->view, seq, &uid);
+		mdbox_map_set_corrupted(map, "file_id=0 for map_uid=%u", uid);
+		return -1;
+	}
+	*rec_r = rec;
+	return 0;
+}
+
+static int
+mdbox_map_get_seq(struct mdbox_map *map, uint32_t map_uid, uint32_t *seq_r)
+{
+	if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) {
+		/* not found - try again after a refresh */
+		if (mdbox_map_refresh(map) < 0)
+			return -1;
+		if (!mail_index_lookup_seq(map->view, map_uid, seq_r))
+			return 0;
+	}
+	return 1;
+}
+
+int mdbox_map_lookup(struct mdbox_map *map, uint32_t map_uid,
+		     uint32_t *file_id_r, uoff_t *offset_r)
+{
+	const struct mdbox_map_mail_index_record *rec;
+	uint32_t seq;
+	int ret;
+
+	if (mdbox_map_open_or_create(map) < 0)
+		return -1;
+
+	if ((ret = mdbox_map_get_seq(map, map_uid, &seq)) <= 0)
+		return ret;
+
+	if (mdbox_map_lookup_seq(map, seq, &rec) < 0)
+		return -1;
+	*file_id_r = rec->file_id;
+	*offset_r = rec->offset;
+	return 1;
+}
+
+int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid,
+			  struct mdbox_map_mail_index_record *rec_r,
+			  uint16_t *refcount_r)
+{
+	uint32_t seq;
+	int ret;
+
+	if (mdbox_map_open_or_create(map) < 0)
+		return -1;
+
+	if ((ret = mdbox_map_get_seq(map, map_uid, &seq)) <= 0)
+		return ret;
+
+	return mdbox_map_lookup_seq_full(map, seq, rec_r, refcount_r);
+}
+
+int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq,
+			      struct mdbox_map_mail_index_record *rec_r,
+			      uint16_t *refcount_r)
+{
+	const struct mdbox_map_mail_index_record *rec;
+	const uint16_t *ref16_p;
+	const void *data;
+
+	if (mdbox_map_lookup_seq(map, seq, &rec) < 0)
+		return -1;
+	*rec_r = *rec;
+
+	mail_index_lookup_ext(map->view, seq, map->ref_ext_id, &data, NULL);
+	if (data == NULL) {
+		mdbox_map_set_corrupted(map, "missing ref extension");
+		return -1;
+	}
+	ref16_p = data;
+	*refcount_r = *ref16_p;
+	return 1;
+}
+
+uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq)
+{
+	uint32_t uid;
+
+	mail_index_lookup_uid(map->view, seq, &uid);
+	return uid;
+}
+
+unsigned int mdbox_map_get_messages_count(struct mdbox_map *map)
+{
+	return mail_index_view_get_messages_count(map->view);
+}
+
+int mdbox_map_view_lookup_rec(struct mdbox_map *map,
+			      struct mail_index_view *view, uint32_t seq,
+			      struct dbox_mail_lookup_rec *rec_r)
+{
+	const uint16_t *ref16_p;
+	const void *data;
+
+	i_zero(rec_r);
+	mail_index_lookup_uid(view, seq, &rec_r->map_uid);
+
+	mail_index_lookup_ext(view, seq, map->map_ext_id, &data, NULL);
+	if (data == NULL) {
+		mdbox_map_set_corrupted(map, "missing map extension");
+		return -1;
+	}
+	memcpy(&rec_r->rec, data, sizeof(rec_r->rec));
+
+	mail_index_lookup_ext(view, seq, map->ref_ext_id, &data, NULL);
+	if (data == NULL) {
+		mdbox_map_set_corrupted(map, "missing ref extension");
+		return -1;
+	}
+	ref16_p = data;
+	rec_r->refcount = *ref16_p;
+	return 0;
+}
+
+int mdbox_map_get_file_msgs(struct mdbox_map *map, uint32_t file_id,
+			    ARRAY_TYPE(mdbox_map_file_msg) *recs)
+{
+	const struct mail_index_header *hdr;
+	struct dbox_mail_lookup_rec rec;
+	struct mdbox_map_file_msg msg;
+	uint32_t seq;
+
+	if (mdbox_map_refresh(map) < 0)
+		return -1;
+	hdr = mail_index_get_header(map->view);
+
+	i_zero(&msg);
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		if (mdbox_map_view_lookup_rec(map, map->view, seq, &rec) < 0)
+			return -1;
+
+		if (rec.rec.file_id == file_id) {
+			msg.map_uid = rec.map_uid;
+			msg.offset = rec.rec.offset;
+			msg.refcount = rec.refcount;
+			array_push_back(recs, &msg);
+		}
+	}
+	return 0;
+}
+
+int mdbox_map_get_zero_ref_files(struct mdbox_map *map,
+				 ARRAY_TYPE(seq_range) *file_ids_r)
+{
+	const struct mail_index_header *hdr;
+	const struct mdbox_map_mail_index_record *rec;
+	const uint16_t *ref16_p;
+	const void *data;
+	uint32_t seq;
+	bool expunged;
+	int ret;
+
+	if ((ret = mdbox_map_open(map)) <= 0) {
+		/* no map / internal error */
+		return ret;
+	}
+	if (mdbox_map_refresh(map) < 0)
+		return -1;
+
+	hdr = mail_index_get_header(map->view);
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
+				      &data, &expunged);
+		if (data != NULL && !expunged) {
+			ref16_p = data;
+			if (*ref16_p != 0)
+				continue;
+		}
+
+		mail_index_lookup_ext(map->view, seq, map->map_ext_id,
+				      &data, &expunged);
+		if (data != NULL && !expunged) {
+			rec = data;
+			seq_range_array_add(file_ids_r, rec->file_id);
+		}
+	}
+	return 0;
+}
+
+struct mdbox_map_atomic_context *mdbox_map_atomic_begin(struct mdbox_map *map)
+{
+	struct mdbox_map_atomic_context *atomic;
+
+	atomic = i_new(struct mdbox_map_atomic_context, 1);
+	atomic->map = map;
+	return atomic;
+}
+
+static void
+mdbox_map_sync_handle(struct mdbox_map *map,
+		      struct mail_index_sync_ctx *sync_ctx)
+{
+	struct mail_index_sync_rec sync_rec;
+	uint32_t seq1, seq2;
+	uoff_t offset1, offset2;
+
+	mail_index_sync_get_offsets(sync_ctx, &seq1, &offset1, &seq2, &offset2);
+	if (offset1 != offset2 || seq1 != seq2) {
+		/* something had crashed. need a full resync. */
+		i_warning("mdbox %s: Inconsistency in map index "
+			  "(%u,%"PRIuUOFF_T" != %u,%"PRIuUOFF_T")",
+			  map->path, seq1, offset1, seq2, offset2);
+		mdbox_storage_set_corrupted(map->storage);
+	}
+	while (mail_index_sync_next(sync_ctx, &sync_rec)) ;
+}
+
+int mdbox_map_atomic_lock(struct mdbox_map_atomic_context *atomic,
+			  const char *reason)
+{
+	int ret;
+
+	if (atomic->locked)
+		return 0;
+
+	if (mdbox_map_open_or_create(atomic->map) < 0)
+		return -1;
+
+	/* use syncing to lock the transaction log, so that we always see
+	   log's head_offset = tail_offset */
+	ret = mail_index_sync_begin(atomic->map->index, &atomic->sync_ctx,
+				    &atomic->sync_view, &atomic->sync_trans,
+				    MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET);
+	if (mail_index_reset_fscked(atomic->map->index))
+		mdbox_storage_set_corrupted(atomic->map->storage);
+	if (ret <= 0) {
+		i_assert(ret != 0);
+		mail_storage_set_index_error(MAP_STORAGE(atomic->map),
+					     atomic->map->index);
+		return -1;
+	}
+	mail_index_sync_set_reason(atomic->sync_ctx, reason);
+	atomic->locked = TRUE;
+	/* reset refresh state so that if it's wanted to be done locked,
+	   it gets the latest changes */
+	atomic->map_refreshed = FALSE;
+	mdbox_map_sync_handle(atomic->map, atomic->sync_ctx);
+	return 0;
+}
+
+bool mdbox_map_atomic_is_locked(struct mdbox_map_atomic_context *atomic)
+{
+	return atomic->locked;
+}
+
+void mdbox_map_atomic_set_failed(struct mdbox_map_atomic_context *atomic)
+{
+	atomic->success = FALSE;
+	atomic->failed = TRUE;
+}
+
+void mdbox_map_atomic_set_success(struct mdbox_map_atomic_context *atomic)
+{
+	if (!atomic->failed)
+		atomic->success = TRUE;
+}
+
+void mdbox_map_atomic_unset_fscked(struct mdbox_map_atomic_context *atomic)
+{
+	mail_index_unset_fscked(atomic->sync_trans);
+}
+
+int mdbox_map_atomic_finish(struct mdbox_map_atomic_context **_atomic)
+{
+	struct mdbox_map_atomic_context *atomic = *_atomic;
+	int ret = 0;
+
+	*_atomic = NULL;
+
+	if (atomic->sync_ctx == NULL) {
+		/* not locked */
+		i_assert(!atomic->locked);
+	} else if (atomic->success) {
+		if (mail_index_sync_commit(&atomic->sync_ctx) < 0) {
+			mail_storage_set_index_error(MAP_STORAGE(atomic->map),
+						     atomic->map->index);
+			ret = -1;
+		}
+	} else {
+		mail_index_sync_rollback(&atomic->sync_ctx);
+	}
+	i_free(atomic);
+	return ret;
+}
+
+struct mdbox_map_transaction_context *
+mdbox_map_transaction_begin(struct mdbox_map_atomic_context *atomic,
+			    bool external)
+{
+	struct mdbox_map_transaction_context *ctx;
+	enum mail_index_transaction_flags flags =
+		MAIL_INDEX_TRANSACTION_FLAG_FSYNC;
+	bool success;
+
+	if (external)
+		flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
+
+	ctx = i_new(struct mdbox_map_transaction_context, 1);
+	ctx->atomic = atomic;
+	if (atomic->locked && atomic->map_refreshed) {
+		/* already refreshed within a lock, don't do it again */
+		success = TRUE;
+	} else {
+		success = mdbox_map_open(atomic->map) > 0 &&
+			mdbox_map_refresh(atomic->map) == 0;
+	}
+
+	if (success) {
+		atomic->map_refreshed = TRUE;
+		ctx->trans = mail_index_transaction_begin(atomic->map->view,
+							  flags);
+	}
+	return ctx;
+}
+
+int mdbox_map_transaction_commit(struct mdbox_map_transaction_context *ctx,
+				 const char *reason)
+{
+	i_assert(!ctx->committed);
+
+	ctx->committed = TRUE;
+	if (!ctx->changed)
+		return 0;
+
+	if (mdbox_map_atomic_lock(ctx->atomic, reason) < 0)
+		return -1;
+
+	if (mail_index_transaction_commit(&ctx->trans) < 0) {
+		mail_storage_set_index_error(MAP_STORAGE(ctx->atomic->map),
+					     ctx->atomic->map->index);
+		return -1;
+	}
+	mdbox_map_atomic_set_success(ctx->atomic);
+	return 0;
+}
+
+void mdbox_map_transaction_free(struct mdbox_map_transaction_context **_ctx)
+{
+	struct mdbox_map_transaction_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+
+	if (ctx->trans != NULL)
+		mail_index_transaction_rollback(&ctx->trans);
+	i_free(ctx);
+}
+
+int mdbox_map_update_refcount(struct mdbox_map_transaction_context *ctx,
+			      uint32_t map_uid, int diff)
+{
+	struct mdbox_map *map = ctx->atomic->map;
+	const void *data;
+	uint32_t seq;
+	int old_diff, new_diff;
+
+	if (unlikely(ctx->trans == NULL))
+		return -1;
+
+	if (!mail_index_lookup_seq(map->view, map_uid, &seq)) {
+		/* we can't refresh map here since view has a
+		   transaction open. */
+		if (diff > 0) {
+			/* the message was probably just purged */
+			mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_EXPUNGED,
+				"Some of the requested messages no longer exist.");
+		} else {
+			mdbox_map_set_corrupted(map,
+				"refcount update lost map_uid=%u", map_uid);
+		}
+		return -1;
+	}
+	mail_index_lookup_ext(map->view, seq, map->ref_ext_id, &data, NULL);
+	old_diff = data == NULL ? 0 : *((const uint16_t *)data);
+	ctx->changed = TRUE;
+	new_diff = mail_index_atomic_inc_ext(ctx->trans, seq,
+					     map->ref_ext_id, diff);
+	if (old_diff + new_diff < 0) {
+		mdbox_map_set_corrupted(map, "map_uid=%u refcount too low",
+					map_uid);
+		return -1;
+	}
+	if (old_diff + new_diff >= 32768 && new_diff > 0) {
+		/* we're getting close to the 64k limit. fail early
+		   to make it less likely that two processes increase
+		   the refcount enough times to cross the limit */
+		mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_LIMIT,
+			t_strdup_printf("Message has been copied too many times (%d + %d)",
+					old_diff, new_diff));
+		return -1;
+	}
+	return 0;
+}
+
+int mdbox_map_update_refcounts(struct mdbox_map_transaction_context *ctx,
+			       const ARRAY_TYPE(uint32_t) *map_uids, int diff)
+{
+	const uint32_t *uidp;
+	unsigned int i, count;
+
+	if (unlikely(ctx->trans == NULL))
+		return -1;
+
+	count = array_count(map_uids);
+	for (i = 0; i < count; i++) {
+		uidp = array_idx(map_uids, i);
+		if (mdbox_map_update_refcount(ctx, *uidp, diff) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+int mdbox_map_remove_file_id(struct mdbox_map *map, uint32_t file_id)
+{
+	struct mdbox_map_atomic_context *atomic;
+	struct mdbox_map_transaction_context *map_trans;
+	const struct mail_index_header *hdr;
+	const struct mdbox_map_mail_index_record *rec;
+	const void *data;
+	uint32_t seq;
+	int ret = 0;
+
+	/* make sure the map is refreshed, otherwise we might be expunging
+	   messages that have already been moved to other files. */
+
+	/* we need a per-file transaction, otherwise we can't refresh the map */
+	atomic = mdbox_map_atomic_begin(map);
+	map_trans = mdbox_map_transaction_begin(atomic, TRUE);
+
+	hdr = mail_index_get_header(map->view);
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		mail_index_lookup_ext(map->view, seq, map->map_ext_id,
+				      &data, NULL);
+		if (data == NULL) {
+			mdbox_map_set_corrupted(map, "missing map extension");
+			ret = -1;
+			break;
+		}
+
+		rec = data;
+		if (rec->file_id == file_id) {
+			map_trans->changed = TRUE;
+			mail_index_expunge(map_trans->trans, seq);
+		}
+	}
+	if (ret == 0)
+		ret = mdbox_map_transaction_commit(map_trans, "removing file");
+	mdbox_map_transaction_free(&map_trans);
+	if (mdbox_map_atomic_finish(&atomic) < 0)
+		ret = -1;
+	return ret;
+}
+
+struct mdbox_map_append_context *
+mdbox_map_append_begin(struct mdbox_map_atomic_context *atomic)
+{
+	struct mdbox_map_append_context *ctx;
+
+	ctx = i_new(struct mdbox_map_append_context, 1);
+	ctx->atomic = atomic;
+	ctx->map = atomic->map;
+	ctx->first_new_file_id = (uint32_t)-1;
+	i_array_init(&ctx->file_appends, 64);
+	i_array_init(&ctx->files, 64);
+	i_array_init(&ctx->appends, 128);
+
+	if (mdbox_map_open_or_create(atomic->map) < 0)
+		ctx->failed = TRUE;
+	else {
+		/* refresh the map so we can try appending to the
+		   latest files */
+		if (mdbox_map_refresh(atomic->map) == 0)
+			atomic->map_refreshed = TRUE;
+		else
+			ctx->failed = TRUE;
+	}
+	return ctx;
+}
+
+static time_t day_begin_stamp(unsigned int interval)
+{
+	struct tm tm;
+	time_t stamp;
+	unsigned int unit = 1;
+
+	if (interval == 0)
+		return 0;
+
+	/* get the beginning of day/hour/minute depending on how large
+	   the interval is */
+	tm = *localtime(&ioloop_time);
+	if (interval >= 60) {
+		tm.tm_sec = 0;
+		unit = 60;
+		if (interval >= 3600) {
+			tm.tm_min = 0;
+			unit = 3600;
+			if (interval >= 3600*24) {
+				tm.tm_hour = 0;
+				unit = 3600*24;
+			}
+		}
+	}
+	stamp = mktime(&tm);
+	if (stamp == (time_t)-1)
+		i_panic("mktime(today) failed");
+
+	return stamp - (interval - unit);
+}
+
+static bool dbox_try_open(struct dbox_file *file, bool want_altpath)
+{
+	bool notfound;
+
+	if (want_altpath) {
+		if (dbox_file_open(file, &notfound) <= 0)
+			return FALSE;
+	} else {
+		if (dbox_file_open_primary(file, &notfound) <= 0)
+			return FALSE;
+	}
+	if (notfound)
+		return FALSE;
+
+	if (file->lock != NULL) {
+		/* already locked, we're possibly in the middle of purging it
+		   in which case we really don't want to write there. */
+		return FALSE;
+	}
+	if (dbox_file_is_in_alt(file) != want_altpath) {
+		/* different alt location than what we want, can't use it */
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static bool dbox_file_is_ok_at(struct dbox_file *file, uoff_t offset)
+{
+	bool last;
+	int ret;
+
+	if (dbox_file_seek(file, offset) == 0)
+		return FALSE;
+
+	while ((ret = dbox_file_seek_next(file, &offset, &last)) > 0);
+	if (ret == 0 && !last)
+		return FALSE;
+	return TRUE;
+}
+
+static bool
+mdbox_map_file_try_append(struct mdbox_map_append_context *ctx,
+			  bool want_altpath,
+			  const struct mdbox_map_mail_index_record *rec,
+			  time_t stamp, uoff_t mail_size,
+			  struct dbox_file_append_context **file_append_r,
+			  struct ostream **output_r, bool *retry_later_r)
+{
+	struct mdbox_map *map = ctx->map;
+	struct mdbox_storage *storage = map->storage;
+	struct dbox_file *file;
+	struct dbox_file_append_context *file_append;
+	struct stat st;
+	bool file_too_old = FALSE;
+	int ret;
+
+	*file_append_r = NULL;
+	*output_r = NULL;
+	*retry_later_r = FALSE;
+
+	file = mdbox_file_init(storage, rec->file_id);
+	if (!dbox_try_open(file, want_altpath)) {
+		dbox_file_unref(&file);
+		return TRUE;
+	}
+
+	if (file->create_time < stamp)
+		file_too_old = TRUE;
+	else if ((ret = dbox_file_try_lock(file)) <= 0) {
+		/* locking failed */
+		*retry_later_r = ret == 0;
+	} else if (stat(file->cur_path, &st) < 0) {
+		if (errno != ENOENT)
+			i_error("stat(%s) failed: %m", file->cur_path);
+		/* the file was unlinked between opening and locking it. */
+	} else if (st.st_size != rec->offset + rec->size &&
+		   /* check if there's any garbage at the end of file.
+		      note that there may be valid messages added by another
+		      session before we locked it (but after we refreshed
+		      map index). */
+		   !dbox_file_is_ok_at(file, rec->offset + rec->size)) {
+		/* error message was already logged */
+	} else {
+		file_append = dbox_file_append_init(file);
+		if (dbox_file_get_append_stream(file_append, output_r) <= 0) {
+			/* couldn't append to this file */
+		} else if ((*output_r)->offset + mail_size > map->set->mdbox_rotate_size) {
+			/* file was too large after all */
+		} else {
+			/* success */
+			*file_append_r = file_append;
+			return TRUE;
+		}
+		dbox_file_append_rollback(&file_append);
+	}
+
+	/* failure */
+	dbox_file_unlock(file);
+	dbox_file_unref(&file);
+	return !file_too_old;
+}
+
+static bool
+mdbox_map_is_appending(struct mdbox_map_append_context *ctx, uint32_t file_id)
+{
+	struct dbox_file_append_context *const *file_appends;
+	unsigned int i, count;
+
+	/* there shouldn't be many files open, don't bother with anything
+	   faster. */
+	file_appends = array_get(&ctx->file_appends, &count);
+	for (i = 0; i < count; i++) {
+		struct mdbox_file *mfile =
+			(struct mdbox_file *)file_appends[i]->file;
+
+		if (mfile->file_id == file_id)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static struct dbox_file_append_context *
+mdbox_map_find_existing_append(struct mdbox_map_append_context *ctx,
+			       uoff_t mail_size, bool want_altpath,
+			       struct ostream **output_r)
+{
+	struct mdbox_map *map = ctx->map;
+	struct dbox_file_append_context *const *file_appends, *append;
+	struct mdbox_file *mfile;
+	unsigned int i, count;
+	uoff_t append_offset;
+
+	/* first try to use files already used in this append */
+	file_appends = array_get(&ctx->file_appends, &count);
+	for (i = count; i > ctx->files_nonappendable_count; i--) {
+		append = file_appends[i-1];
+
+		if (dbox_file_is_in_alt(append->file) != want_altpath)
+			continue;
+		if (append->file->fd == -1) {
+			/* already closed it (below). we might be able to still
+			   fit some small mail there, but that's too much
+			   trouble */
+			continue;
+		}
+
+		append_offset = append->output->offset;
+		if (append_offset + mail_size <= map->set->mdbox_rotate_size &&
+		    dbox_file_get_append_stream(append, output_r) > 0)
+			return append;
+
+		/* can't append to this file anymore. if we created this file,
+		   close it so we don't waste fds. if we didn't, we can't close
+		   it without also losing our lock too early. */
+		mfile = (struct mdbox_file *)append->file;
+		if (mfile->file_id == 0 && dbox_file_append_flush(append) == 0)
+			dbox_file_close(append->file);
+	}
+	ctx->files_nonappendable_count = count;
+	return NULL;
+}
+
+static int
+mdbox_map_find_primary_files(struct mdbox_map_append_context *ctx,
+			     ARRAY_TYPE(seq_range) *file_ids_r)
+{
+	struct mdbox_storage *dstorage = ctx->map->storage;
+	struct mail_storage *storage = &dstorage->storage.storage;
+	DIR *dir;
+	struct dirent *d;
+	uint32_t file_id;
+	int ret = 0;
+
+	/* we want to quickly find the latest alt file, but we also want to
+	   avoid accessing the alt storage as much as possible. typically most
+	   of the older mails would be in alt storage, so we'll just put the
+	   few m.* files in primary storage to checked_file_ids array. other
+	   files are then known to exist in alt storage. */
+	dir = opendir(dstorage->storage_dir);
+	if (dir == NULL) {
+		mail_storage_set_critical(storage,
+			"opendir(%s) failed: %m", dstorage->storage_dir);
+		return -1;
+	}
+
+	for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
+		if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
+			    strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
+			continue;
+		if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
+				  &file_id) < 0)
+			continue;
+
+		seq_range_array_add(file_ids_r, file_id);
+	}
+	if (errno != 0) {
+		mail_storage_set_critical(storage,
+			"readdir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	if (closedir(dir) < 0) {
+		mail_storage_set_critical(storage,
+			"closedir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	return ret;
+}
+
+static int
+mdbox_map_find_appendable_file(struct mdbox_map_append_context *ctx,
+			       uoff_t mail_size, bool want_altpath,
+			       struct dbox_file_append_context **file_append_r,
+			       struct ostream **output_r)
+{
+	struct mdbox_map *map = ctx->map;
+	ARRAY_TYPE(seq_range) checked_file_ids;
+	const struct mail_index_header *hdr;
+	const struct mdbox_map_mail_index_record *rec;
+	unsigned int backwards_lookup_count;
+	uint32_t seq, seq1, uid;
+	time_t stamp;
+	bool retry_later;
+
+	if (mail_size >= map->set->mdbox_rotate_size)
+		return 0;
+
+	/* try to find an existing appendable file */
+	stamp = day_begin_stamp(map->set->mdbox_rotate_interval);
+	hdr = mail_index_get_header(map->view);
+
+	backwards_lookup_count = 0;
+	t_array_init(&checked_file_ids, 16);
+
+	if (want_altpath) {
+		/* we want to save to alt storage. */
+		if (mdbox_map_find_primary_files(ctx, &checked_file_ids) < 0)
+			return -1;
+	}
+
+	for (seq = hdr->messages_count; seq > 0; seq--) {
+		if (mdbox_map_lookup_seq(map, seq, &rec) < 0)
+			return -1;
+
+		if (seq_range_exists(&checked_file_ids, rec->file_id))
+			continue;
+		seq_range_array_add(&checked_file_ids, rec->file_id);
+
+		if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) {
+			/* we've wasted enough time here */
+			break;
+		}
+
+		/* first lookup: this should be enough usually, but we can't
+		   be sure until after locking. also if messages were recently
+		   moved, this message might not be the last one in the file. */
+		if (rec->offset + rec->size + mail_size >=
+		    			map->set->mdbox_rotate_size)
+			continue;
+
+		if (mdbox_map_is_appending(ctx, rec->file_id)) {
+			/* already checked this */
+			continue;
+		}
+
+		mail_index_lookup_uid(map->view, seq, &uid);
+		if (!mdbox_map_file_try_append(ctx, want_altpath, rec,
+					       stamp, mail_size, file_append_r,
+					       output_r, &retry_later)) {
+			/* file is too old. the rest of the files are too. */
+			break;
+		}
+		/* NOTE: we've now refreshed map view. there are no guarantees
+		   about sequences anymore. */
+		if (*file_append_r != NULL)
+			return 1;
+		/* FIXME: use retry_later somehow */
+		if (uid == 1 ||
+		    !mail_index_lookup_seq_range(map->view, 1, uid-1,
+						 &seq1, &seq))
+			break;
+		seq++;
+	}
+	return 0;
+}
+
+int mdbox_map_append_next(struct mdbox_map_append_context *ctx,
+			  uoff_t mail_size, enum mdbox_map_append_flags flags,
+			  struct dbox_file_append_context **file_append_ctx_r,
+			  struct ostream **output_r)
+{
+	struct dbox_file *file;
+	struct mdbox_map_append *append;
+	struct dbox_file_append_context *file_append;
+	bool existing, want_altpath;
+	int ret;
+
+	if (ctx->failed)
+		return -1;
+
+	want_altpath = (flags & DBOX_MAP_APPEND_FLAG_ALT) != 0;
+	file_append = mdbox_map_find_existing_append(ctx, mail_size,
+						     want_altpath, output_r);
+	if (file_append != NULL) {
+		ret = 1;
+		existing = TRUE;
+	} else {
+		ret = mdbox_map_find_appendable_file(ctx, mail_size, want_altpath,
+						     &file_append, output_r);
+		existing = FALSE;
+	}
+	if (ret > 0)
+		file = file_append->file;
+	else if (ret < 0)
+		return -1;
+	else {
+		/* create a new file */
+		file = (flags & DBOX_MAP_APPEND_FLAG_ALT) == 0 ?
+			mdbox_file_init(ctx->map->storage, 0) :
+			mdbox_file_init_new_alt(ctx->map->storage);
+		file_append = dbox_file_append_init(file);
+
+		ret = dbox_file_get_append_stream(file_append, output_r);
+		if (ret <= 0) {
+			i_assert(ret < 0);
+			dbox_file_append_rollback(&file_append);
+			dbox_file_unref(&file);
+			return -1;
+		}
+	}
+
+	append = array_append_space(&ctx->appends);
+	append->file_append = file_append;
+	append->offset = (*output_r)->offset;
+	append->size = (uint32_t)-1;
+	if (!existing) {
+		i_assert(file_append->first_append_offset == 0);
+		file_append->first_append_offset = file_append->output->offset;
+		array_push_back(&ctx->file_appends, &file_append);
+		array_push_back(&ctx->files, &file);
+	}
+	*file_append_ctx_r = file_append;
+	return 0;
+}
+
+static void
+mdbox_map_append_close_if_unneeded(struct mdbox_map *map,
+				   struct dbox_file_append_context *append_ctx)
+{
+	struct mdbox_file *mfile =
+		(struct mdbox_file *)append_ctx->file;
+	uoff_t end_offset = append_ctx->output->offset;
+
+	/* if this file is now large enough not to fit any other
+	   mails and we created it, close its fd since it's not
+	   needed anymore. */
+	if (end_offset > map->set->mdbox_rotate_size &&
+	    mfile->file_id == 0 &&
+	    dbox_file_append_flush(append_ctx) == 0)
+		dbox_file_close(append_ctx->file);
+}
+
+void mdbox_map_append_finish(struct mdbox_map_append_context *ctx)
+{
+	struct mdbox_map_append *appends, *last;
+	unsigned int count;
+	uoff_t cur_offset;
+
+	appends = array_get_modifiable(&ctx->appends, &count);
+	i_assert(count > 0);
+	last = &appends[count-1];
+	i_assert(last->size == (uint32_t)-1);
+
+	cur_offset = last->file_append->output->offset;
+	i_assert(cur_offset >= last->offset);
+	last->size = cur_offset - last->offset;
+	dbox_file_append_checkpoint(last->file_append);
+
+	mdbox_map_append_close_if_unneeded(ctx->map, last->file_append);
+}
+
+void mdbox_map_append_abort(struct mdbox_map_append_context *ctx)
+{
+	struct mdbox_map_append *appends;
+	unsigned int count;
+
+	appends = array_get_modifiable(&ctx->appends, &count);
+	i_assert(count > 0 && appends[count-1].size == (uint32_t)-1);
+	array_delete(&ctx->appends, count-1, 1);
+}
+
+static int
+mdbox_find_highest_file_id(struct mdbox_map *map, uint32_t *file_id_r)
+{
+	const size_t prefix_len = strlen(MDBOX_MAIL_FILE_PREFIX);
+	DIR *dir;
+	struct dirent *d;
+	unsigned int id, highest_id = 0;
+
+	dir = opendir(map->path);
+	if (dir == NULL) {
+		i_error("opendir(%s) failed: %m", map->path);
+		return -1;
+	}
+	while ((d = readdir(dir)) != NULL) {
+		if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX, prefix_len) == 0 &&
+		    str_to_uint(d->d_name + prefix_len, &id) == 0) {
+			if (highest_id < id)
+				highest_id = id;
+		}
+	}
+	(void)closedir(dir);
+
+	*file_id_r = highest_id;
+	return 0;
+}
+
+static int
+mdbox_map_assign_file_ids(struct mdbox_map_append_context *ctx,
+			  bool separate_transaction, const char *reason)
+{
+	struct dbox_file_append_context *const *file_appends;
+	unsigned int i, count;
+	struct mdbox_map_mail_index_header hdr;
+	uint32_t first_file_id, file_id, existing_id;
+
+	/* start the syncing. we'll need it even if there are no file ids to
+	   be assigned. */
+	if (mdbox_map_atomic_lock(ctx->atomic, reason) < 0)
+		return -1;
+
+	mdbox_map_get_ext_hdr(ctx->map, ctx->atomic->sync_view, &hdr);
+	file_id = hdr.highest_file_id + 1;
+
+	if (ctx->map->verify_existing_file_ids) {
+		/* storage/ directory had been already created but
+		   without indexes. scan to see if there exists a higher
+		   m.* file id than what is in header, so we won't
+		   accidentally overwrite any existing files. */
+		if (mdbox_find_highest_file_id(ctx->map, &existing_id) < 0)
+			return -1;
+		if (file_id < existing_id+1)
+			file_id = existing_id+1;
+	}
+
+	/* assign file_ids for newly created files */
+	first_file_id = file_id;
+	file_appends = array_get(&ctx->file_appends, &count);
+	for (i = 0; i < count; i++) {
+		struct mdbox_file *mfile =
+			(struct mdbox_file *)file_appends[i]->file;
+
+		if (dbox_file_append_flush(file_appends[i]) < 0)
+			return -1;
+
+		if (mfile->file_id == 0) {
+			if (mdbox_file_assign_file_id(mfile, file_id++) < 0)
+				return -1;
+		}
+	}
+
+	ctx->trans = !separate_transaction ? NULL :
+		mail_index_transaction_begin(ctx->map->view,
+					MAIL_INDEX_TRANSACTION_FLAG_FSYNC);
+
+	/* update the highest used file_id */
+	if (first_file_id != file_id) {
+		file_id--;
+		mail_index_update_header_ext(ctx->trans != NULL ? ctx->trans :
+					     ctx->atomic->sync_trans,
+					     ctx->map->map_ext_id,
+					     0, &file_id, sizeof(file_id));
+	}
+	return 0;
+}
+
+int mdbox_map_append_assign_map_uids(struct mdbox_map_append_context *ctx,
+				     uint32_t *first_map_uid_r,
+				     uint32_t *last_map_uid_r)
+{
+	const struct mdbox_map_append *appends;
+	const struct mail_index_header *hdr;
+	struct mdbox_map_mail_index_record rec;
+	unsigned int i, count;
+	ARRAY_TYPE(seq_range) uids;
+	const struct seq_range *range;
+	uint32_t seq;
+	uint16_t ref16;
+	int ret = 0;
+
+	if (array_count(&ctx->appends) == 0) {
+		*first_map_uid_r = 0;
+		*last_map_uid_r = 0;
+		return 0;
+	}
+
+	if (mdbox_map_assign_file_ids(ctx, TRUE, "saving - assign uids") < 0)
+		return -1;
+
+	/* append map records to index */
+	i_zero(&rec);
+	ref16 = 1;
+	appends = array_get(&ctx->appends, &count);
+	for (i = 0; i < count; i++) {
+		struct mdbox_file *mfile =
+			(struct mdbox_file *)appends[i].file_append->file;
+
+		i_assert(appends[i].offset <= (uint32_t)-1);
+		i_assert(appends[i].size <= (uint32_t)-1);
+
+		rec.file_id = mfile->file_id;
+		rec.offset = appends[i].offset;
+		rec.size = appends[i].size;
+
+		mail_index_append(ctx->trans, 0, &seq);
+		mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id,
+				      &rec, NULL);
+		mail_index_update_ext(ctx->trans, seq, ctx->map->ref_ext_id,
+				      &ref16, NULL);
+	}
+
+	/* assign map UIDs for appended records */
+	hdr = mail_index_get_header(ctx->atomic->sync_view);
+	t_array_init(&uids, 1);
+	mail_index_append_finish_uids(ctx->trans, hdr->next_uid, &uids);
+	range = array_front(&uids);
+	i_assert(range[0].seq2 - range[0].seq1 + 1 == count);
+
+	if (hdr->uid_validity == 0) {
+		/* we don't really care about uidvalidity, but it can't be 0 */
+		uint32_t uid_validity = ioloop_time;
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+
+	if (mail_index_transaction_commit(&ctx->trans) < 0) {
+		mail_storage_set_index_error(MAP_STORAGE(ctx->map),
+					     ctx->map->index);
+		return -1;
+	}
+
+	*first_map_uid_r = range[0].seq1;
+	*last_map_uid_r = range[0].seq2;
+	return ret;
+}
+
+int mdbox_map_append_move(struct mdbox_map_append_context *ctx,
+			  const ARRAY_TYPE(uint32_t) *map_uids,
+			  const ARRAY_TYPE(seq_range) *expunge_map_uids)
+{
+	const struct mdbox_map_append *appends;
+	struct mdbox_map_mail_index_record rec;
+	struct seq_range_iter iter;
+	const uint32_t *uids;
+	unsigned int i, j, map_uids_count, appends_count;
+	uint32_t uid, seq, next_uid;
+
+	/* map is locked by this call */
+	if (mdbox_map_assign_file_ids(ctx, FALSE, "purging - update uids") < 0)
+		return -1;
+
+	i_zero(&rec);
+	appends = array_get(&ctx->appends, &appends_count);
+
+	next_uid = mail_index_get_header(ctx->atomic->sync_view)->next_uid;
+	uids = array_get(map_uids, &map_uids_count);
+	for (i = j = 0; i < map_uids_count; i++) {
+		struct mdbox_file *mfile =
+			(struct mdbox_file *)appends[j].file_append->file;
+
+		i_assert(j < appends_count);
+		rec.file_id = mfile->file_id;
+		rec.offset = appends[j].offset;
+		rec.size = appends[j].size;
+		j++;
+
+		if (!mail_index_lookup_seq(ctx->atomic->sync_view,
+					   uids[i], &seq)) {
+			/* We wrote the email to the new m.* file, but another
+			   process already expunged it and purged it. Deleting
+			   the email from the new m.* file would be problematic
+			   at this point, so just add the mail back to the map
+			   with refcount=0 and the next purge will remove it. */
+			mail_index_append(ctx->atomic->sync_trans,
+					  next_uid++, &seq);
+		}
+		mail_index_update_ext(ctx->atomic->sync_trans, seq,
+				      ctx->map->map_ext_id, &rec, NULL);
+	}
+
+	seq_range_array_iter_init(&iter, expunge_map_uids); i = 0;
+	while (seq_range_array_iter_nth(&iter, i++, &uid)) {
+		if (!mail_index_lookup_seq(ctx->atomic->sync_view, uid, &seq))
+			i_unreached();
+		mail_index_expunge(ctx->atomic->sync_trans, seq);
+	}
+	return 0;
+}
+
+int mdbox_map_append_flush(struct mdbox_map_append_context *ctx)
+{
+	struct dbox_file_append_context **file_appends;
+	unsigned int i, count;
+
+	i_assert(ctx->trans == NULL);
+
+	file_appends = array_get_modifiable(&ctx->file_appends, &count);
+	for (i = 0; i < count; i++) {
+		if (dbox_file_append_flush(file_appends[i]) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+int mdbox_map_append_commit(struct mdbox_map_append_context *ctx)
+{
+	struct dbox_file_append_context **file_appends;
+	unsigned int i, count;
+
+	i_assert(ctx->trans == NULL);
+
+	file_appends = array_get_modifiable(&ctx->file_appends, &count);
+	for (i = 0; i < count; i++) {
+		if (dbox_file_append_commit(&file_appends[i]) < 0)
+			return -1;
+	}
+	mdbox_map_atomic_set_success(ctx->atomic);
+	return 0;
+}
+
+void mdbox_map_append_free(struct mdbox_map_append_context **_ctx)
+{
+	struct mdbox_map_append_context *ctx = *_ctx;
+	struct dbox_file_append_context **file_appends;
+	struct dbox_file **files;
+	unsigned int i, count;
+
+	*_ctx = NULL;
+
+	if (ctx->trans != NULL)
+		mail_index_transaction_rollback(&ctx->trans);
+
+	file_appends = array_get_modifiable(&ctx->file_appends, &count);
+	for (i = 0; i < count; i++) {
+		if (file_appends[i] != NULL)
+			dbox_file_append_rollback(&file_appends[i]);
+	}
+
+	files = array_get_modifiable(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		dbox_file_unlock(files[i]);
+		dbox_file_unref(&files[i]);
+	}
+
+	array_free(&ctx->appends);
+	array_free(&ctx->file_appends);
+	array_free(&ctx->files);
+	i_free(ctx);
+}
+
+static int mdbox_map_generate_uid_validity(struct mdbox_map *map)
+{
+	const struct mail_index_header *hdr;
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	uint32_t uid_validity;
+	int ret;
+
+	/* do this inside syncing, so that we're locked and there are no
+	   race conditions */
+	ret = mail_index_sync_begin(map->index, &sync_ctx, &view, &trans,
+				    MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET);
+	if (ret <= 0) {
+		i_assert(ret != 0);
+		return -1;
+	}
+	mdbox_map_sync_handle(map, sync_ctx);
+
+	hdr = mail_index_get_header(map->view);
+	if (hdr->uid_validity != 0) {
+		/* someone else beat us to it */
+	} else {
+		uid_validity = ioloop_time;
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+	mail_index_sync_set_reason(sync_ctx, "uidvalidity initialization");
+	return mail_index_sync_commit(&sync_ctx);
+}
+
+uint32_t mdbox_map_get_uid_validity(struct mdbox_map *map)
+{
+	uint32_t uid_validity;
+
+	i_assert(map->view != NULL);
+
+	uid_validity = mail_index_get_header(map->view)->uid_validity;
+	if (uid_validity == 0)
+		mdbox_map_set_corrupted(map, "lost uidvalidity");
+	return uid_validity;
+}
diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.h b/src/lib-storage/index/dbox-multi/mdbox-map.h
new file mode 100644
index 0000000..9571f0e
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.h
@@ -0,0 +1,144 @@
+#ifndef MDBOX_MAP_H
+#define MDBOX_MAP_H
+
+#include "seq-range-array.h"
+
+struct dbox_file_append_context;
+struct mdbox_map_append_context;
+struct mdbox_storage;
+
+enum mdbox_map_append_flags {
+	DBOX_MAP_APPEND_FLAG_ALT	= 0x01
+};
+
+struct mdbox_map_mail_index_header {
+	uint32_t highest_file_id;
+	/* increased every time storage is rebuilt */
+	uint32_t rebuild_count;
+};
+
+struct mdbox_map_mail_index_record {
+	uint32_t file_id;
+	uint32_t offset;
+	uint32_t size; /* including pre/post metadata */
+};
+
+struct mdbox_map_file_msg {
+	uint32_t map_uid;
+	uint32_t offset;
+	uint32_t refcount;
+};
+ARRAY_DEFINE_TYPE(mdbox_map_file_msg, struct mdbox_map_file_msg);
+
+struct mdbox_map *
+mdbox_map_init(struct mdbox_storage *storage, struct mailbox_list *root_list);
+void mdbox_map_deinit(struct mdbox_map **map);
+
+/* Open the map. Returns 1 if ok, 0 if map doesn't exist, -1 if error. */
+int mdbox_map_open(struct mdbox_map *map);
+/* Open or create the map. This is done automatically for most operations.
+   Returns 0 if ok, -1 if error. */
+int mdbox_map_open_or_create(struct mdbox_map *map);
+/* Refresh the map. Returns 0 if ok, -1 if error. */
+int mdbox_map_refresh(struct mdbox_map *map);
+/* Returns TRUE if map has been fsck'd. */
+bool mdbox_map_is_fscked(struct mdbox_map *map);
+
+/* Return the current rebuild counter */
+uint32_t mdbox_map_get_rebuild_count(struct mdbox_map *map);
+
+/* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID
+   is already expunged, -1 if error. */
+int mdbox_map_lookup(struct mdbox_map *map, uint32_t map_uid,
+		     uint32_t *file_id_r, uoff_t *offset_r);
+/* Like mdbox_map_lookup(), but look up everything. */
+int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid,
+			  struct mdbox_map_mail_index_record *rec_r,
+			  uint16_t *refcount_r);
+/* Like mdbox_map_lookup_full(), but look up with sequence. */
+int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq,
+			      struct mdbox_map_mail_index_record *rec_r,
+			      uint16_t *refcount_r);
+/* Return map UID for the map sequence. */
+uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq);
+/* Returns the total number of messages in the map. */
+unsigned int mdbox_map_get_messages_count(struct mdbox_map *map);
+
+/* Get all messages from file */
+int mdbox_map_get_file_msgs(struct mdbox_map *map, uint32_t file_id,
+			    ARRAY_TYPE(mdbox_map_file_msg) *recs);
+
+/* Begin atomic context. There can be multiple transactions/appends within the
+   same atomic context. */
+struct mdbox_map_atomic_context *mdbox_map_atomic_begin(struct mdbox_map *map);
+/* Lock the map immediately. */
+int mdbox_map_atomic_lock(struct mdbox_map_atomic_context *atomic,
+			  const char *reason);
+/* Returns TRUE if map is locked */
+bool mdbox_map_atomic_is_locked(struct mdbox_map_atomic_context *atomic);
+/* When finish() is called, rollback the changes. If data was already written
+   to map's transaction log, this desyncs the map and causes a rebuild */
+void mdbox_map_atomic_set_failed(struct mdbox_map_atomic_context *atomic);
+/* Mark this atomic as having succeeded. This is internally done if
+   transaction or append is committed within this atomic, but not when the
+   atomic is used standalone. */
+void mdbox_map_atomic_set_success(struct mdbox_map_atomic_context *atomic);
+/* Remove fsck'd flag. */
+void mdbox_map_atomic_unset_fscked(struct mdbox_map_atomic_context *atomic);
+/* Commit/rollback changes within this atomic context. */
+int mdbox_map_atomic_finish(struct mdbox_map_atomic_context **atomic);
+
+struct mdbox_map_transaction_context *
+mdbox_map_transaction_begin(struct mdbox_map_atomic_context *atomic,
+			    bool external);
+/* Write transaction to map and leave it locked. Call _free() to update tail
+   offset and unlock. */
+int mdbox_map_transaction_commit(struct mdbox_map_transaction_context *ctx,
+				 const char *reason);
+void mdbox_map_transaction_free(struct mdbox_map_transaction_context **ctx);
+
+int mdbox_map_update_refcount(struct mdbox_map_transaction_context *ctx,
+			      uint32_t map_uid, int diff);
+int mdbox_map_update_refcounts(struct mdbox_map_transaction_context *ctx,
+			       const ARRAY_TYPE(uint32_t) *map_uids, int diff);
+int mdbox_map_remove_file_id(struct mdbox_map *map, uint32_t file_id);
+
+/* Return all files containing messages with zero refcount. */
+int mdbox_map_get_zero_ref_files(struct mdbox_map *map,
+				 ARRAY_TYPE(seq_range) *file_ids_r);
+
+struct mdbox_map_append_context *
+mdbox_map_append_begin(struct mdbox_map_atomic_context *atomic);
+/* Request file for saving a new message with given size (if available). If an
+   existing file can be used, the record is locked and updated in index.
+   Returns 0 if ok, -1 if error. */
+int mdbox_map_append_next(struct mdbox_map_append_context *ctx, uoff_t mail_size,
+			  enum mdbox_map_append_flags flags,
+			  struct dbox_file_append_context **file_append_ctx_r,
+			  struct ostream **output_r);
+/* Finished saving the last mail. Saves the message size. */
+void mdbox_map_append_finish(struct mdbox_map_append_context *ctx);
+/* Abort saving the last mail. */
+void mdbox_map_append_abort(struct mdbox_map_append_context *ctx);
+/* Assign map UIDs to all appended msgs to multi-files. */
+int mdbox_map_append_assign_map_uids(struct mdbox_map_append_context *ctx,
+				     uint32_t *first_map_uid_r,
+				     uint32_t *last_map_uid_r);
+/* The appends are existing messages that were simply moved to a new file.
+   map_uids contains the moved messages' map UIDs. */
+int mdbox_map_append_move(struct mdbox_map_append_context *ctx,
+			  const ARRAY_TYPE(uint32_t) *map_uids,
+			  const ARRAY_TYPE(seq_range) *expunge_map_uids);
+/* Flush/fsync appends. */
+int mdbox_map_append_flush(struct mdbox_map_append_context *ctx);
+/* Returns 0 if ok, -1 if error. */
+int mdbox_map_append_commit(struct mdbox_map_append_context *ctx);
+void mdbox_map_append_free(struct mdbox_map_append_context **ctx);
+
+/* Returns map's uidvalidity */
+uint32_t mdbox_map_get_uid_validity(struct mdbox_map *map);
+
+void mdbox_map_set_corrupted(struct mdbox_map *map, const char *format, ...)
+	ATTR_FORMAT(2, 3);
+
+#endif
diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c
new file mode 100644
index 0000000..8461074
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c
@@ -0,0 +1,690 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "str.h"
+#include "hash.h"
+#include "dbox-attachment.h"
+#include "mdbox-storage.h"
+#include "mdbox-storage-rebuild.h"
+#include "mdbox-file.h"
+#include "mdbox-map.h"
+#include "mdbox-sync.h"
+
+#include <dirent.h>
+
+/*
+   Altmoving works like:
+
+   1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox
+      code and map UID's alt-refcount is updated. It won't be written to disk.
+   2. mdbox_purge() is called, which checks if map UID's refcount equals
+      to its alt-refcount. If it does, it's moved to alt storage. Moving to
+      primary storage is done if _ALT flag was removed from any message.
+*/
+
+enum mdbox_msg_action {
+	MDBOX_MSG_ACTION_MOVE_TO_ALT = 1,
+	MDBOX_MSG_ACTION_MOVE_FROM_ALT
+};
+
+struct mdbox_purge_context {
+	pool_t pool;
+	struct mdbox_storage *storage;
+
+	uint32_t lowest_primary_file_id;
+	/* list of file_ids that exist in primary storage. this list is looked
+	   up while there is no locking, so it may not be accurate anymore by
+	   the time it's used. */
+	ARRAY_TYPE(seq_range) primary_file_ids;
+	/* list of file_ids that we need to purge */
+	ARRAY_TYPE(seq_range) purge_file_ids;
+
+	/* uint32_t map_uid => enum mdbox_msg_action action */
+	HASH_TABLE(void *, void *) altmoves;
+	bool have_altmoves;
+
+	struct mdbox_map_atomic_context *atomic;
+	struct mdbox_map_append_context *append_ctx;
+};
+
+static int mdbox_map_file_msg_offset_cmp(const struct mdbox_map_file_msg *m1,
+					 const struct mdbox_map_file_msg *m2)
+{
+	if (m1->offset < m2->offset)
+		return -1;
+	else if (m1->offset > m2->offset)
+		return 1;
+	else
+		return 0;
+}
+
+static int
+mdbox_file_read_metadata_hdr(struct dbox_file *file,
+			     struct dbox_metadata_header *meta_hdr_r)
+{
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	ret = i_stream_read_bytes(file->input, &data, &size,
+				  sizeof(*meta_hdr_r));
+	if (ret <= 0) {
+		i_assert(ret == -1);
+		if (file->input->stream_errno == 0) {
+			dbox_file_set_corrupted(file, "missing metadata");
+			return 0;
+		}
+		mail_storage_set_critical(&file->storage->storage,
+			"read(%s) failed: %s", file->cur_path,
+			i_stream_get_error(file->input));
+		return -1;
+	}
+
+	memcpy(meta_hdr_r, data, sizeof(*meta_hdr_r));
+	if (memcmp(meta_hdr_r->magic_post, DBOX_MAGIC_POST,
+		   sizeof(meta_hdr_r->magic_post)) != 0) {
+		dbox_file_set_corrupted(file, "invalid metadata magic");
+		return 0;
+	}
+	i_stream_skip(file->input, sizeof(*meta_hdr_r));
+	return 1;
+}
+
+static int
+mdbox_file_metadata_copy(struct dbox_file *file, struct ostream *output)
+{
+	struct dbox_metadata_header meta_hdr;
+	const char *line;
+	size_t buf_size;
+	int ret;
+
+	if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0)
+		return ret;
+
+	o_stream_nsend(output, &meta_hdr, sizeof(meta_hdr));
+	buf_size = i_stream_get_max_buffer_size(file->input);
+	/* use unlimited line length for metadata */
+	i_stream_set_max_buffer_size(file->input, SIZE_MAX);
+	while ((line = i_stream_read_next_line(file->input)) != NULL) {
+		if (*line == '\0') {
+			/* end of metadata */
+			break;
+		}
+		o_stream_nsend_str(output, line);
+		o_stream_nsend(output, "\n", 1);
+	}
+	i_stream_set_max_buffer_size(file->input, buf_size);
+
+	if (line == NULL) {
+		dbox_file_set_corrupted(file, "missing end-of-metadata line");
+		return 0;
+	}
+	o_stream_nsend(output, "\n", 1);
+	return 1;
+}
+
+static int
+mdbox_metadata_get_extrefs(struct dbox_file *file, pool_t ext_refs_pool,
+			   ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+	struct dbox_metadata_header meta_hdr;
+	const char *line;
+	size_t buf_size;
+	int ret;
+
+	/* skip and ignore the header */
+	if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0)
+		return ret;
+
+	buf_size = i_stream_get_max_buffer_size(file->input);
+	/* use unlimited line length for metadata */
+	i_stream_set_max_buffer_size(file->input, SIZE_MAX);
+	while ((line = i_stream_read_next_line(file->input)) != NULL) {
+		if (*line == '\0') {
+			/* end of metadata */
+			break;
+		}
+		if (*line == DBOX_METADATA_EXT_REF) T_BEGIN {
+			if (!index_attachment_parse_extrefs(line+1, ext_refs_pool,
+							    extrefs)) {
+				i_warning("%s: Ignoring corrupted extref: %s",
+					  file->cur_path, line);
+			}
+		} T_END;
+	}
+	i_stream_set_max_buffer_size(file->input, buf_size);
+
+	if (line == NULL) {
+		dbox_file_set_corrupted(file, "missing end-of-metadata line");
+		return 0;
+	}
+	return 1;
+}
+
+static bool
+mdbox_purge_want_altpath(struct mdbox_purge_context *ctx,
+			 struct dbox_file *file, uint32_t map_uid)
+{
+	enum mdbox_msg_action action;
+	void *value;
+
+	if (dbox_file_is_in_alt(file))
+		return TRUE;
+
+	if (!ctx->have_altmoves)
+		return FALSE;
+
+	value = hash_table_lookup(ctx->altmoves, POINTER_CAST(map_uid));
+	action = POINTER_CAST_TO(value, enum mdbox_msg_action);
+	return action == MDBOX_MSG_ACTION_MOVE_TO_ALT;
+}
+
+static int
+mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file,
+		     const struct mdbox_map_file_msg *msg)
+{
+	struct dbox_file_append_context *out_file_append;
+	struct istream *input;
+	struct ostream *output;
+	enum mdbox_map_append_flags append_flags;
+	uoff_t msg_size;
+	int ret;
+
+	if (ctx->append_ctx == NULL)
+		ctx->append_ctx = mdbox_map_append_begin(ctx->atomic);
+
+	append_flags = !mdbox_purge_want_altpath(ctx, file, msg->map_uid) ? 0 :
+		DBOX_MAP_APPEND_FLAG_ALT;
+	msg_size = file->msg_header_size + file->cur_physical_size;
+	if (mdbox_map_append_next(ctx->append_ctx, file->cur_physical_size,
+				  append_flags, &out_file_append, &output) < 0)
+		return -1;
+
+	i_assert(file != out_file_append->file);
+
+	input = i_stream_create_limit(file->input, msg_size);
+	o_stream_nsend_istream(output, input);
+	if (o_stream_flush(output) < 0) {
+		mail_storage_set_critical(&file->storage->storage,
+					  "write(%s) failed: %s",
+					  out_file_append->file->cur_path,
+					  o_stream_get_error(output));
+		ret = -1;
+	} else if (input->v_offset != msg_size) {
+		i_assert(input->v_offset < msg_size);
+		i_assert(i_stream_read_eof(file->input));
+
+		dbox_file_set_corrupted(file, "truncated message at EOF");
+		ret = 0;
+	} else {
+		ret = 1;
+	}
+	i_stream_unref(&input);
+
+	if (ret > 0) {
+		/* copy metadata */
+		if ((ret = mdbox_file_metadata_copy(file, output)) <= 0)
+			return ret;
+
+		mdbox_map_append_finish(ctx->append_ctx);
+	}
+	return ret;
+}
+
+static int
+mdbox_file_purge_check_refcounts(struct mdbox_purge_context *ctx,
+				 const ARRAY_TYPE(mdbox_map_file_msg) *msgs_arr)
+{
+	struct mdbox_map *map = ctx->storage->map;
+	struct mdbox_map_mail_index_record rec;
+	uint16_t refcount;
+	const struct mdbox_map_file_msg *msgs;
+	unsigned int i, count;
+	int ret;
+
+	if (mdbox_map_atomic_lock(ctx->atomic, "purging check") < 0)
+		return -1;
+
+	msgs = array_get(msgs_arr, &count);
+	for (i = 0; i < count; i++) {
+		if (msgs[i].refcount != 0)
+			continue;
+
+		ret = mdbox_map_lookup_full(map, msgs[i].map_uid, &rec,
+					    &refcount);
+		if (ret <= 0) {
+			if (ret < 0)
+				return -1;
+			mdbox_map_set_corrupted(map,
+				"Purging unexpectedly lost map_uid=%u",
+				msgs[i].map_uid);
+			return -1;
+		}
+		if (refcount > 0)
+			return 0;
+	}
+	return 1;
+}
+
+static int
+mdbox_purge_attachments(struct mdbox_purge_context *ctx,
+			const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr)
+{
+	struct dbox_storage *storage = &ctx->storage->storage;
+	const struct mail_attachment_extref *extref;
+	int ret = 0;
+
+	array_foreach(extrefs_arr, extref) {
+		if (index_attachment_delete(&storage->storage,
+					    storage->attachment_fs,
+					    extref->path) < 0)
+			ret = -1;
+	}
+	return ret;
+}
+
+static int
+mdbox_file_purge(struct mdbox_purge_context *ctx, struct dbox_file *file,
+		 uint32_t file_id)
+{
+	struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage;
+	struct stat st;
+	ARRAY_TYPE(mdbox_map_file_msg) msgs_arr;
+	const struct mdbox_map_file_msg *msgs;
+	ARRAY_TYPE(seq_range) expunged_map_uids;
+	ARRAY_TYPE(uint32_t) copied_map_uids;
+	ARRAY_TYPE(mail_attachment_extref) ext_refs;
+	pool_t ext_refs_pool;
+	unsigned int i, count;
+	uoff_t offset;
+	int ret;
+
+	i_assert(ctx->atomic == NULL);
+	i_assert(ctx->append_ctx == NULL);
+
+	if ((ret = dbox_file_try_lock(file)) <= 0)
+		return ret;
+
+	/* make sure the file still exists. another process may have already
+	   deleted it. */
+	if (stat(file->cur_path, &st) < 0) {
+		dbox_file_unlock(file);
+		if (errno == ENOENT)
+			return 0;
+
+		mail_storage_set_critical(&file->storage->storage,
+			"stat(%s) failed: %m", file->cur_path);
+		return -1;
+	}
+
+	/* get list of map UIDs that exist in this file (again has to be done
+	   after locking) */
+	i_array_init(&msgs_arr, 128);
+	if (mdbox_map_get_file_msgs(dstorage->map, file_id,
+				    &msgs_arr) < 0) {
+		array_free(&msgs_arr);
+		dbox_file_unlock(file);
+		return -1;
+	}
+	/* sort messages by their offset */
+	array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp);
+
+	ext_refs_pool = pool_alloconly_create("mdbox purge ext refs", 1024);
+	ctx->atomic = mdbox_map_atomic_begin(ctx->storage->map);
+	msgs = array_get(&msgs_arr, &count);
+	i_array_init(&ext_refs, 32);
+	i_array_init(&copied_map_uids, I_MIN(count, 1));
+	i_array_init(&expunged_map_uids, I_MIN(count, 1));
+	offset = file->file_header_size;
+	for (i = 0; i < count; i++) {
+		if ((ret = dbox_file_seek(file, offset)) <= 0)
+			break;
+
+		if (msgs[i].offset != offset) {
+			/* map doesn't match file's actual contents */
+			dbox_file_set_corrupted(file,
+				"purging found mismatched offsets "
+				"(%"PRIuUOFF_T" vs %u, %u/%u)",
+				offset, msgs[i].offset, i, count);
+			ret = 0;
+			break;
+		}
+
+		if (msgs[i].refcount == 0) {
+			/* skip over expunged message */
+			i_stream_seek(file->input, offset +
+				      file->msg_header_size +
+				      file->cur_physical_size);
+			/* skip metadata */
+			ret = mdbox_metadata_get_extrefs(file, ext_refs_pool,
+							 &ext_refs);
+			if (ret <= 0)
+				break;
+			seq_range_array_add(&expunged_map_uids,
+					    msgs[i].map_uid);
+		} else {
+			/* non-expunged message. write it to output file. */
+			i_stream_seek(file->input, offset);
+			ret = mdbox_purge_save_msg(ctx, file, &msgs[i]);
+			if (ret <= 0)
+				break;
+			array_push_back(&copied_map_uids, &msgs[i].map_uid);
+		}
+		offset = file->input->v_offset;
+	}
+	if (offset != (uoff_t)st.st_size && ret > 0) {
+		/* file has more messages than what map tells us */
+		dbox_file_set_corrupted(file,
+			"more messages available than in map "
+			"(%"PRIuUOFF_T" < %"PRIuUOFF_T")", offset, st.st_size);
+		ret = 0;
+	}
+	if (ret > 0 && ctx->append_ctx != NULL) {
+		/* flush writes before locking the map */
+		if (mdbox_map_append_flush(ctx->append_ctx) < 0)
+			ret = -1;
+	}
+
+	if (ret <= 0)
+		ret = -1;
+	else {
+		/* it's possible that one of the messages we purged was
+		   just copied to another mailbox. the only way to prevent that
+		   would be to keep map locked during the purge, but that could
+		   keep it locked for too long. instead we'll check here if
+		   there are such copies, and if there are cancel this file's
+		   purge. */
+		ret = mdbox_file_purge_check_refcounts(ctx, &msgs_arr);
+	}
+	array_free(&msgs_arr); msgs = NULL;
+
+	if (ret <= 0) {
+		/* failed */
+	} else if (ctx->append_ctx == NULL) {
+		/* everything purged from this file */
+		ret = 1;
+	} else {
+		/* assign new file_id + offset to moved messages */
+		if (mdbox_map_append_move(ctx->append_ctx, &copied_map_uids,
+					  &expunged_map_uids) < 0 ||
+		    mdbox_map_append_commit(ctx->append_ctx) < 0)
+			ret = -1;
+		else
+			ret = 1;
+	}
+	if (ctx->append_ctx != NULL)
+		mdbox_map_append_free(&ctx->append_ctx);
+	(void)mdbox_map_atomic_finish(&ctx->atomic);
+
+	/* unlink only after unlocking map, so readers don't see it
+	   temporarily vanished */
+	if (ret > 0) {
+		(void)dbox_file_unlink(file);
+		if (mdbox_map_remove_file_id(ctx->storage->map, file_id) < 0)
+			ret = -1;
+	} else {
+		dbox_file_unlock(file);
+	}
+	array_free(&copied_map_uids);
+	array_free(&expunged_map_uids);
+
+	(void)mdbox_purge_attachments(ctx, &ext_refs);
+	array_free(&ext_refs);
+	pool_unref(&ext_refs_pool);
+	return ret;
+}
+
+void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(mail->box);
+	ARRAY_TYPE(uint32_t) *dest;
+	uint32_t map_uid;
+
+	/* we'll assume here that alt flag won't be changed multiple times
+	   for the same mail. it shouldn't happen with current code, and
+	   checking for it would just slow down the code.
+
+	   so the way it works currently is just that map_uids are added to
+	   an array, which is later sorted and processed further. note that
+	   it's still possible that the same map_uid exists in the array
+	   multiple times. */
+	if (mdbox_mail_lookup(mbox, mbox->box.view, mail->seq, &map_uid) < 0)
+		return;
+
+	dest = move_to_alt ? &mbox->storage->move_to_alt_map_uids :
+		&mbox->storage->move_from_alt_map_uids;
+
+	if (!array_is_created(dest))
+		i_array_init(dest, 256);
+	array_push_back(dest, &map_uid);
+}
+
+static struct mdbox_purge_context *
+mdbox_purge_alloc(struct mdbox_storage *storage)
+{
+	struct mdbox_purge_context *ctx;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mdbox purge context", 1024*32);
+	ctx = p_new(pool, struct mdbox_purge_context, 1);
+	ctx->pool = pool;
+	ctx->storage = storage;
+	ctx->lowest_primary_file_id = (uint32_t)-1;
+	i_array_init(&ctx->primary_file_ids, 64);
+	i_array_init(&ctx->purge_file_ids, 64);
+	hash_table_create_direct(&ctx->altmoves, pool, 0);
+	return ctx;
+}
+
+static void mdbox_purge_free(struct mdbox_purge_context **_ctx)
+{
+	struct mdbox_purge_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+
+	hash_table_destroy(&ctx->altmoves);
+	array_free(&ctx->primary_file_ids);
+	array_free(&ctx->purge_file_ids);
+	pool_unref(&ctx->pool);
+}
+
+static int mdbox_purge_get_primary_files(struct mdbox_purge_context *ctx)
+{
+	struct mdbox_storage *dstorage = ctx->storage;
+	struct mail_storage *storage = &dstorage->storage.storage;
+	DIR *dir;
+	struct dirent *d;
+	string_t *path;
+	unsigned int file_id;
+	size_t dir_len;
+	int ret = 0;
+
+	if (!array_is_created(&dstorage->move_to_alt_map_uids) &&
+	    !array_is_created(&dstorage->move_from_alt_map_uids)) {
+		/* we don't need to do alt moving, don't bother getting list
+		   of primary files */
+		return 0;
+	}
+
+	dir = opendir(dstorage->storage_dir);
+	if (dir == NULL) {
+		if (errno == ENOENT) {
+			/* no storage directory at all yet */
+			return 0;
+		}
+		mail_storage_set_critical(storage,
+			"opendir(%s) failed: %m", dstorage->storage_dir);
+		return -1;
+	}
+
+	path = t_str_new(256);
+	str_append(path, dstorage->storage_dir);
+	str_append_c(path, '/');
+	dir_len = str_len(path);
+
+	for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
+		if (!str_begins(d->d_name, MDBOX_MAIL_FILE_PREFIX))
+			continue;
+		if (str_to_uint32(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
+				  &file_id) < 0)
+			continue;
+
+		str_truncate(path, dir_len);
+		str_append(path, d->d_name);
+		seq_range_array_add(&ctx->primary_file_ids, file_id);
+	}
+	if (array_count(&ctx->primary_file_ids) > 0) {
+		const struct seq_range *range =
+			array_front(&ctx->primary_file_ids);
+		ctx->lowest_primary_file_id = range[0].seq1;
+	}
+	if (errno != 0) {
+		mail_storage_set_critical(storage,
+			"readdir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	if (closedir(dir) < 0) {
+		mail_storage_set_critical(storage,
+			"closedir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	return ret;
+}
+
+static int uint32_t_cmp(const uint32_t *u1, const uint32_t *u2)
+{
+	if (*u1 < *u2)
+		return -1;
+	if (*u1 > *u2)
+		return 1;
+	return 0;
+}
+
+static int mdbox_altmove_add_files(struct mdbox_purge_context *ctx)
+{
+	struct mdbox_storage *dstorage = ctx->storage;
+	const uint32_t *map_uids;
+	unsigned int i, count, alt_refcount = 0;
+	struct mdbox_map_mail_index_record cur_rec;
+	enum mdbox_msg_action action;
+	uint32_t cur_map_uid;
+	uint16_t cur_refcount = 0;
+	uoff_t offset;
+	int ret = 0;
+
+	/* first add move-to-alt actions */
+	if (array_is_created(&dstorage->move_to_alt_map_uids)) {
+		array_sort(&dstorage->move_to_alt_map_uids, uint32_t_cmp);
+		map_uids = array_get(&dstorage->move_to_alt_map_uids, &count);
+	} else {
+		map_uids = NULL;
+		count = 0;
+	}
+	cur_map_uid = 0;
+	for (i = 0; i < count; i++) {
+		if (cur_map_uid != map_uids[i]) {
+			cur_map_uid = map_uids[i];
+			if (mdbox_map_lookup_full(dstorage->map, cur_map_uid,
+						  &cur_rec, &cur_refcount) < 0) {
+				cur_refcount = (uint16_t)-1;
+				ret = -1;
+			}
+			alt_refcount = 1;
+		} else {
+			alt_refcount++;
+		}
+
+		if (alt_refcount == cur_refcount &&
+		    seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) {
+			/* all instances marked as moved to alt storage */
+			action = MDBOX_MSG_ACTION_MOVE_TO_ALT;
+			hash_table_insert(ctx->altmoves,
+					  POINTER_CAST(cur_map_uid),
+					  POINTER_CAST(action));
+			seq_range_array_add(&ctx->purge_file_ids,
+					    cur_rec.file_id);
+		}
+	}
+
+	/* next add move-from-alt actions. they override move-to-alt actions
+	   in case there happen to be any conflicts (shouldn't). only a single
+	   move-from-alt record is needed to do the move. */
+	if (array_is_created(&dstorage->move_from_alt_map_uids))
+		map_uids = array_get(&dstorage->move_from_alt_map_uids, &count);
+	else {
+		map_uids = NULL;
+		count = 0;
+	}
+	cur_map_uid = 0;
+	for (i = 0; i < count; i++) {
+		if (cur_map_uid == map_uids[i])
+			continue;
+		cur_map_uid = map_uids[i];
+
+		if (mdbox_map_lookup(dstorage->map, cur_map_uid,
+				     &cur_rec.file_id, &offset) < 0) {
+			ret = -1;
+			continue;
+		}
+		if (seq_range_exists(&ctx->primary_file_ids, cur_rec.file_id)) {
+			/* already in primary storage */
+			continue;
+		}
+
+		action = MDBOX_MSG_ACTION_MOVE_FROM_ALT;
+		hash_table_update(ctx->altmoves, POINTER_CAST(cur_map_uid),
+				  POINTER_CAST(action));
+		seq_range_array_add(&ctx->purge_file_ids, cur_rec.file_id);
+	}
+	ctx->have_altmoves = hash_table_count(ctx->altmoves) > 0;
+	return ret;
+}
+
+int mdbox_purge(struct mail_storage *_storage)
+{
+	struct mdbox_storage *storage = (struct mdbox_storage *)_storage;
+	struct mdbox_purge_context *ctx;
+	struct dbox_file *file;
+	struct seq_range_iter iter;
+	unsigned int i = 0;
+	uint32_t file_id;
+	bool deleted;
+	int ret;
+
+	ctx = mdbox_purge_alloc(storage);
+	ret = mdbox_map_get_zero_ref_files(storage->map, &ctx->purge_file_ids);
+	if (storage->alt_storage_dir != NULL) {
+		if (mdbox_purge_get_primary_files(ctx) < 0)
+			ret = -1;
+		else {
+			/* add files that can be altmoved */
+			if (mdbox_altmove_add_files(ctx) < 0)
+				ret = -1;
+		}
+	}
+
+	seq_range_array_iter_init(&iter, &ctx->purge_file_ids); i = 0;
+	while (ret == 0 &&
+	       seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN {
+		file = mdbox_file_init(storage, file_id);
+		if (dbox_file_open(file, &deleted) > 0 && !deleted) {
+			if (mdbox_file_purge(ctx, file, file_id) < 0)
+				ret = -1;
+		} else {
+			if (mdbox_map_remove_file_id(storage->map, file_id) < 0)
+				ret = -1;
+		}
+		dbox_file_unref(&file);
+	} T_END;
+	mdbox_purge_free(&ctx);
+
+	if (storage->corrupted) {
+		/* purging found corrupted files */
+		(void)mdbox_storage_rebuild(storage);
+		ret = -1;
+	}
+	return ret;
+}
diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c
new file mode 100644
index 0000000..c0dcf5b
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-save.c
@@ -0,0 +1,491 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "fdatasync-path.h"
+#include "hex-binary.h"
+#include "hex-dec.h"
+#include "str.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "ostream.h"
+#include "write-full.h"
+#include "index-mail.h"
+#include "index-pop3-uidl.h"
+#include "mail-copy.h"
+#include "dbox-save.h"
+#include "mdbox-storage.h"
+#include "mdbox-map.h"
+#include "mdbox-file.h"
+#include "mdbox-sync.h"
+
+
+struct dbox_save_mail {
+	struct dbox_file_append_context *file_append;
+	uint32_t seq;
+	uint32_t append_offset;
+	time_t save_date;
+	bool written_to_disk;
+};
+
+struct mdbox_save_context {
+	struct dbox_save_context ctx;
+
+	struct mdbox_mailbox *mbox;
+	struct mdbox_sync_context *sync_ctx;
+
+	struct dbox_file *cur_file;
+	struct dbox_file_append_context *cur_file_append;
+	struct mdbox_map_append_context *append_ctx;
+
+	ARRAY_TYPE(uint32_t) copy_map_uids;
+	struct mdbox_map_atomic_context *atomic;
+	struct mdbox_map_transaction_context *map_trans;
+
+	ARRAY(struct dbox_save_mail) mails;
+};
+
+#define MDBOX_SAVECTX(s)	container_of(DBOX_SAVECTX(s), struct mdbox_save_context, ctx)
+
+static struct dbox_file *
+mdbox_copy_file_get_file(struct mailbox_transaction_context *t,
+			 uint32_t seq, uoff_t *offset_r)
+{
+	struct mdbox_save_context *ctx = MDBOX_SAVECTX(t->save_ctx);
+	const struct mdbox_mail_index_record *rec;
+	const void *data;
+	uint32_t file_id;
+
+	mail_index_lookup_ext(t->view, seq, ctx->mbox->ext_id, &data, NULL);
+	rec = data;
+
+	if (mdbox_map_lookup(ctx->mbox->storage->map, rec->map_uid,
+			     &file_id, offset_r) < 0)
+		i_unreached();
+
+	return mdbox_file_init(ctx->mbox->storage, file_id);
+}
+
+struct dbox_file *
+mdbox_save_file_get_file(struct mailbox_transaction_context *t,
+			 uint32_t seq, uoff_t *offset_r)
+{
+	struct mdbox_save_context *ctx = MDBOX_SAVECTX(t->save_ctx);
+	const struct dbox_save_mail *mails, *mail;
+	unsigned int count;
+
+	mails = array_get(&ctx->mails, &count);
+	i_assert(count > 0);
+	i_assert(seq >= mails[0].seq);
+
+	mail = &mails[seq - mails[0].seq];
+	i_assert(mail->seq == seq);
+
+	if (mail->file_append == NULL) {
+		/* copied mail */
+		return mdbox_copy_file_get_file(t, seq, offset_r);
+	}
+
+	/* saved mail */
+	i_assert(mail->written_to_disk);
+	if (dbox_file_append_flush(mail->file_append) < 0)
+		ctx->ctx.failed = TRUE;
+
+	mail->file_append->file->refcount++;
+	*offset_r = mail->append_offset;
+	return mail->file_append->file;
+}
+
+struct mail_save_context *
+mdbox_save_alloc(struct mailbox_transaction_context *t)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(t->box);
+	struct mdbox_save_context *ctx;
+
+	i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	if (t->save_ctx != NULL) {
+		/* use the existing allocated structure */
+		ctx = MDBOX_SAVECTX(t->save_ctx);
+		ctx->cur_file = NULL;
+		ctx->ctx.failed = FALSE;
+		ctx->ctx.finished = FALSE;
+		ctx->ctx.dbox_output = NULL;
+		ctx->cur_file_append = NULL;
+		return &ctx->ctx.ctx;
+	}
+
+	ctx = i_new(struct mdbox_save_context, 1);
+	ctx->ctx.ctx.transaction = t;
+	ctx->ctx.trans = t->itrans;
+	ctx->mbox = mbox;
+	ctx->atomic = mdbox_map_atomic_begin(mbox->storage->map);
+	ctx->append_ctx = mdbox_map_append_begin(ctx->atomic);
+	i_array_init(&ctx->mails, 32);
+	t->save_ctx = &ctx->ctx.ctx;
+	return t->save_ctx;
+}
+
+int mdbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
+{
+	struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx);
+	struct dbox_save_mail *save_mail;
+	uoff_t mail_size, append_offset;
+
+	/* get the size of the mail to be saved, if possible */
+	if (i_stream_get_size(input, TRUE, &mail_size) <= 0) {
+		/* we couldn't find out the exact size. fallback to non-exact,
+		   maybe it'll give something useful. the mail size is used
+		   only to figure out if it's causing mdbox file to grow
+		   too large. */
+		if (i_stream_get_size(input, FALSE, &mail_size) <= 0)
+			mail_size = 0;
+	}
+	if (mdbox_map_append_next(ctx->append_ctx, mail_size, 0,
+				  &ctx->cur_file_append,
+				  &ctx->ctx.dbox_output) < 0) {
+		ctx->ctx.failed = TRUE;
+		return -1;
+	}
+	i_assert(ctx->ctx.dbox_output->offset <= (uint32_t)-1);
+	append_offset = ctx->ctx.dbox_output->offset;
+
+	ctx->cur_file = ctx->cur_file_append->file;
+	dbox_save_begin(&ctx->ctx, input);
+
+	save_mail = array_append_space(&ctx->mails);
+	save_mail->file_append = ctx->cur_file_append;
+	save_mail->seq = ctx->ctx.seq;
+	save_mail->append_offset = append_offset;
+	return ctx->ctx.failed ? -1 : 0;
+}
+
+static int mdbox_save_mail_write_metadata(struct mdbox_save_context *ctx,
+					  struct dbox_save_mail *mail)
+{
+	struct dbox_file *file = mail->file_append->file;
+	struct dbox_message_header dbox_msg_hdr;
+	uoff_t message_size;
+	guid_128_t guid_128;
+
+	i_assert(file->msg_header_size == sizeof(dbox_msg_hdr));
+
+	message_size = ctx->ctx.dbox_output->offset -
+		mail->append_offset - mail->file_append->file->msg_header_size;
+
+	dbox_save_write_metadata(&ctx->ctx.ctx, ctx->ctx.dbox_output,
+				 message_size, ctx->mbox->box.name, guid_128);
+	/* save the 128bit GUID to index so if the map index gets corrupted
+	   we can still find the message */
+	mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
+			      ctx->mbox->guid_ext_id, guid_128, NULL);
+
+	dbox_msg_header_fill(&dbox_msg_hdr, message_size);
+	if (o_stream_pwrite(ctx->ctx.dbox_output, &dbox_msg_hdr,
+			    sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
+		dbox_file_set_syscall_error(file, "pwrite()");
+		return -1;
+	}
+	mail->written_to_disk = TRUE;
+	mail->save_date = ctx->ctx.ctx.data.save_date;
+	return 0;
+}
+
+static int mdbox_save_finish_write(struct mail_save_context *_ctx)
+{
+	struct mdbox_save_context *ctx = (struct mdbox_save_context *)_ctx;
+	struct dbox_save_mail *mail;
+
+	ctx->ctx.finished = TRUE;
+	if (ctx->ctx.dbox_output == NULL)
+		return -1;
+
+	dbox_save_end(&ctx->ctx);
+
+	mail = array_back_modifiable(&ctx->mails);
+	if (!ctx->ctx.failed) T_BEGIN {
+		if (mdbox_save_mail_write_metadata(ctx, mail) < 0)
+			ctx->ctx.failed = TRUE;
+		else
+			mdbox_map_append_finish(ctx->append_ctx);
+	} T_END;
+
+	if (mail->file_append->file->input != NULL) {
+		/* if we try to read the saved mail before unlocking file,
+		   make sure the input stream doesn't have stale data */
+		i_stream_sync(mail->file_append->file->input);
+	}
+	i_stream_unref(&ctx->ctx.input);
+
+	if (ctx->ctx.failed) {
+		index_storage_save_abort_last(&ctx->ctx.ctx, ctx->ctx.seq);
+		mdbox_map_append_abort(ctx->append_ctx);
+		array_pop_back(&ctx->mails);
+		return -1;
+	}
+	return 0;
+}
+
+int mdbox_save_finish(struct mail_save_context *ctx)
+{
+	int ret;
+
+	ret = mdbox_save_finish_write(ctx);
+	index_save_context_free(ctx);
+	return ret;
+}
+
+void mdbox_save_cancel(struct mail_save_context *_ctx)
+{
+	struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx);
+
+	ctx->failed = TRUE;
+	(void)mdbox_save_finish(_ctx);
+}
+
+static void
+mdbox_save_set_map_uids(struct mdbox_save_context *ctx,
+			uint32_t first_map_uid, uint32_t last_map_uid)
+{
+	struct mdbox_mailbox *mbox = ctx->mbox;
+	struct mail_index_view *view = ctx->ctx.ctx.transaction->view;
+	const struct mdbox_mail_index_record *old_rec;
+	struct mdbox_mail_index_record rec;
+	const struct dbox_save_mail *mails;
+	unsigned int i, count;
+	const void *data;
+	uint32_t next_map_uid = first_map_uid;
+
+	mdbox_update_header(mbox, ctx->ctx.trans, NULL);
+
+	i_zero(&rec);
+	mails = array_get(&ctx->mails, &count);
+	for (i = 0; i < count; i++) {
+		mail_index_lookup_ext(view, mails[i].seq, mbox->ext_id,
+				      &data, NULL);
+		old_rec = data;
+		if (old_rec != NULL && old_rec->map_uid != 0) {
+			/* message was copied. keep the existing map uid */
+			continue;
+		}
+
+		if (mails[i].save_date > 0)
+			rec.save_date = mails[i].save_date;
+		else
+			rec.save_date = ioloop_time;
+		rec.map_uid = next_map_uid++;
+		mail_index_update_ext(ctx->ctx.trans, mails[i].seq,
+				      mbox->ext_id, &rec, NULL);
+	}
+	i_assert(next_map_uid == last_map_uid + 1);
+}
+
+int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
+{
+	struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx);
+	struct mailbox_transaction_context *_t = _ctx->transaction;
+	const struct mail_index_header *hdr;
+	uint32_t first_map_uid, last_map_uid;
+
+	i_assert(ctx->ctx.finished);
+
+	/* flush/fsync writes to m.* files before locking the map */
+	if (mdbox_map_append_flush(ctx->append_ctx) < 0) {
+		mdbox_transaction_save_rollback(_ctx);
+		return -1;
+	}
+
+	/* make sure the map gets locked */
+	if (mdbox_map_atomic_lock(ctx->atomic, "saving") < 0) {
+		mdbox_transaction_save_rollback(_ctx);
+		return -1;
+	}
+	/* lock the mailbox after map to avoid deadlocks. if we've noticed
+	   any corruption, deal with it later, otherwise we won't have
+	   up-to-date atomic->sync_view */
+	if (mdbox_sync_begin(ctx->mbox, MDBOX_SYNC_FLAG_NO_PURGE |
+			     MDBOX_SYNC_FLAG_FORCE |
+			     MDBOX_SYNC_FLAG_FSYNC |
+			     MDBOX_SYNC_FLAG_NO_REBUILD, ctx->atomic,
+			     &ctx->sync_ctx) < 0) {
+		mdbox_transaction_save_rollback(_ctx);
+		return -1;
+	}
+	i_assert(ctx->sync_ctx != NULL);
+
+	/* assign map UIDs for newly saved messages after we've successfully
+	   acquired all the locks. the transaction is now very unlikely to
+	   fail. the UIDs are written to the transaction log immediately within
+	   this function, but the map is left locked. */
+	if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid,
+					     &last_map_uid) < 0) {
+		mdbox_transaction_save_rollback(_ctx);
+		return -1;
+	}
+
+	/* update dbox header flags */
+	dbox_save_update_header_flags(&ctx->ctx, ctx->sync_ctx->sync_view,
+		ctx->mbox->hdr_ext_id, offsetof(struct mdbox_index_header, flags));
+
+	/* assign UIDs for new messages */
+	hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
+	mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
+				      &_t->changes->saved_uids);
+
+	if (ctx->ctx.highest_pop3_uidl_seq != 0) {
+		const struct dbox_save_mail *mails;
+		struct seq_range_iter iter;
+		unsigned int highest_pop3_uidl_idx;
+		uint32_t uid;
+
+		mails = array_front(&ctx->mails);
+		highest_pop3_uidl_idx =
+			ctx->ctx.highest_pop3_uidl_seq - mails[0].seq;
+		i_assert(mails[highest_pop3_uidl_idx].seq == ctx->ctx.highest_pop3_uidl_seq);
+
+		seq_range_array_iter_init(&iter, &_t->changes->saved_uids);
+		if (!seq_range_array_iter_nth(&iter, highest_pop3_uidl_idx, &uid))
+			i_unreached();
+		index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid);
+	}
+
+	/* save map UIDs to mailbox index */
+	if (first_map_uid != 0)
+		mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid);
+
+	/* increase map's refcount for copied mails */
+	if (array_is_created(&ctx->copy_map_uids)) {
+		ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE);
+		if (mdbox_map_update_refcounts(ctx->map_trans,
+					       &ctx->copy_map_uids, 1) < 0) {
+			mdbox_transaction_save_rollback(_ctx);
+			return -1;
+		}
+		mail_index_sync_set_reason(ctx->sync_ctx->index_sync_ctx, "copying");
+	} else {
+		mail_index_sync_set_reason(ctx->sync_ctx->index_sync_ctx, "saving");
+	}
+
+	_t->changes->uid_validity = hdr->uid_validity;
+	return 0;
+}
+
+void mdbox_transaction_save_commit_post(struct mail_save_context *_ctx,
+					struct mail_index_transaction_commit_result *result)
+{
+	struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx);
+	struct mail_storage *storage = _ctx->transaction->box->storage;
+
+	_ctx->transaction = NULL; /* transaction is already freed */
+
+	mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx,
+					  result);
+
+	/* finish writing the mailbox APPENDs */
+	if (mdbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) {
+		/* commit refcount increases for copied mails */
+		if (ctx->map_trans != NULL) {
+			if (mdbox_map_transaction_commit(ctx->map_trans, "copy refcount updates") < 0)
+				mdbox_map_atomic_set_failed(ctx->atomic);
+		}
+		/* flush file append writes */
+		if (mdbox_map_append_commit(ctx->append_ctx) < 0)
+			mdbox_map_atomic_set_failed(ctx->atomic);
+	}
+	mdbox_map_append_free(&ctx->append_ctx);
+	/* update the sync tail offset, everything else
+	   was already written at this point. */
+	(void)mdbox_map_atomic_finish(&ctx->atomic);
+
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
+		const char *box_path = mailbox_get_path(&ctx->mbox->box);
+
+		if (fdatasync_path(box_path) < 0) {
+			mail_set_critical(_ctx->dest_mail,
+				"fdatasync_path(%s) failed: %m", box_path);
+		}
+	}
+	mdbox_transaction_save_rollback(_ctx);
+}
+
+void mdbox_transaction_save_rollback(struct mail_save_context *_ctx)
+{
+	struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx);
+
+	if (!ctx->ctx.finished)
+		mdbox_save_cancel(&ctx->ctx.ctx);
+	if (ctx->append_ctx != NULL)
+		mdbox_map_append_free(&ctx->append_ctx);
+	if (ctx->map_trans != NULL)
+		mdbox_map_transaction_free(&ctx->map_trans);
+	if (ctx->atomic != NULL)
+		(void)mdbox_map_atomic_finish(&ctx->atomic);
+	if (array_is_created(&ctx->copy_map_uids))
+		array_free(&ctx->copy_map_uids);
+
+	if (ctx->sync_ctx != NULL)
+		(void)mdbox_sync_finish(&ctx->sync_ctx, FALSE);
+
+	array_free(&ctx->mails);
+	i_free(ctx);
+}
+
+int mdbox_copy(struct mail_save_context *_ctx, struct mail *mail)
+{
+	struct mdbox_save_context *ctx = MDBOX_SAVECTX(_ctx);
+	struct dbox_save_mail *save_mail;
+	struct mdbox_mailbox *src_mbox;
+	struct mdbox_mail_index_record rec;
+	const void *guid_data;
+	guid_128_t wanted_guid;
+
+	ctx->ctx.finished = TRUE;
+
+	if (mail->box->storage != _ctx->transaction->box->storage ||
+	    _ctx->transaction->box->disable_reflink_copy_to)
+		return mail_storage_copy(_ctx, mail);
+	src_mbox = MDBOX_MAILBOX(mail->box);
+
+	i_zero(&rec);
+	rec.save_date = ioloop_time;
+	if (mdbox_mail_lookup(src_mbox, mail->transaction->view, mail->seq,
+			      &rec.map_uid) < 0) {
+		index_save_context_free(_ctx);
+		return -1;
+	}
+
+	mail_index_lookup_ext(mail->transaction->view, mail->seq,
+			      src_mbox->guid_ext_id, &guid_data, NULL);
+	if (guid_data == NULL || guid_128_is_empty(guid_data)) {
+		/* missing GUID, something's broken. don't copy using
+		   refcounting. */
+		return mail_storage_copy(_ctx, mail);
+	} else if (_ctx->data.guid != NULL &&
+		   (guid_128_from_string(_ctx->data.guid, wanted_guid) < 0 ||
+		    memcmp(guid_data, wanted_guid, sizeof(wanted_guid)) != 0)) {
+		/* GUID change requested. we can't do it with refcount
+		   copying */
+		return mail_storage_copy(_ctx, mail);
+	}
+
+	/* remember the map_uid so we can later increase its refcount */
+	if (!array_is_created(&ctx->copy_map_uids))
+		i_array_init(&ctx->copy_map_uids, 32);
+	array_push_back(&ctx->copy_map_uids, &rec.map_uid);
+
+	/* add message to mailbox index */
+	dbox_save_add_to_index(&ctx->ctx);
+	mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
+			      ctx->mbox->ext_id, &rec, NULL);
+
+	mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq,
+			      ctx->mbox->guid_ext_id, guid_data, NULL);
+	index_copy_cache_fields(_ctx, mail, ctx->ctx.seq);
+
+	save_mail = array_append_space(&ctx->mails);
+	save_mail->seq = ctx->ctx.seq;
+
+	mail_set_seq_saving(_ctx->dest_mail, ctx->ctx.seq);
+	index_save_context_free(_ctx);
+	return 0;
+}
diff --git a/src/lib-storage/index/dbox-multi/mdbox-settings.c b/src/lib-storage/index/dbox-multi/mdbox-settings.c
new file mode 100644
index 0000000..dc9e989
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-settings.c
@@ -0,0 +1,43 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "settings-parser.h"
+#include "mail-storage-settings.h"
+#include "mdbox-settings.h"
+
+#include <stddef.h>
+
+#undef DEF
+#define DEF(type, name) \
+	SETTING_DEFINE_STRUCT_##type(#name, name, struct mdbox_settings)
+
+static const struct setting_define mdbox_setting_defines[] = {
+	DEF(BOOL, mdbox_preallocate_space),
+	DEF(SIZE, mdbox_rotate_size),
+	DEF(TIME, mdbox_rotate_interval),
+
+	SETTING_DEFINE_LIST_END
+};
+
+static const struct mdbox_settings mdbox_default_settings = {
+	.mdbox_preallocate_space = FALSE,
+	.mdbox_rotate_size = 10*1024*1024,
+	.mdbox_rotate_interval = 0
+};
+
+static const struct setting_parser_info mdbox_setting_parser_info = {
+	.module_name = "mdbox",
+	.defines = mdbox_setting_defines,
+	.defaults = &mdbox_default_settings,
+
+	.type_offset = SIZE_MAX,
+	.struct_size = sizeof(struct mdbox_settings),
+
+	.parent_offset = SIZE_MAX,
+	.parent = &mail_user_setting_parser_info
+};
+
+const struct setting_parser_info *mdbox_get_setting_parser_info(void)
+{
+	return &mdbox_setting_parser_info;
+}
diff --git a/src/lib-storage/index/dbox-multi/mdbox-settings.h b/src/lib-storage/index/dbox-multi/mdbox-settings.h
new file mode 100644
index 0000000..353da69
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-settings.h
@@ -0,0 +1,12 @@
+#ifndef MDBOX_SETTINGS_H
+#define MDBOX_SETTINGS_H
+
+struct mdbox_settings {
+	bool mdbox_preallocate_space;
+	uoff_t mdbox_rotate_size;
+	unsigned int mdbox_rotate_interval;
+};
+
+const struct setting_parser_info *mdbox_get_setting_parser_info(void);
+
+#endif
diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
new file mode 100644
index 0000000..4277a1a
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
@@ -0,0 +1,1005 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "hash.h"
+#include "str.h"
+#include "mail-cache.h"
+#include "index-rebuild.h"
+#include "mail-namespace.h"
+#include "mailbox-list-private.h"
+#include "mdbox-storage.h"
+#include "mdbox-file.h"
+#include "mdbox-map-private.h"
+#include "mdbox-sync.h"
+#include "mdbox-storage-rebuild.h"
+
+#include <dirent.h>
+#include <unistd.h>
+
+#define REBUILD_MAX_REFCOUNT 32768
+
+struct mdbox_rebuild_msg {
+	struct mdbox_rebuild_msg *guid_hash_next;
+
+	guid_128_t guid_128;
+	uint32_t file_id;
+	uint32_t offset;
+	uint32_t rec_size;
+	uoff_t mail_size;
+	uint32_t map_uid;
+
+	uint16_t refcount;
+	bool seen_zero_ref_in_map:1;
+};
+
+struct rebuild_msg_mailbox {
+	struct mailbox *box;
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	uint32_t next_uid;
+};
+
+struct mdbox_storage_rebuild_context {
+	struct mdbox_storage *storage;
+	struct mdbox_map_atomic_context *atomic;
+	pool_t pool;
+
+	struct mdbox_map_mail_index_header orig_map_hdr;
+	HASH_TABLE(uint8_t *, struct mdbox_rebuild_msg *) guid_hash;
+	ARRAY(struct mdbox_rebuild_msg *) msgs;
+	ARRAY_TYPE(seq_range) seen_file_ids;
+
+	uint32_t rebuild_count;
+	uint32_t highest_file_id;
+
+	struct mailbox_list *default_list;
+
+	struct rebuild_msg_mailbox prev_msg;
+
+	bool have_pop3_uidls:1;
+	bool have_pop3_orders:1;
+};
+
+static struct mdbox_storage_rebuild_context *
+mdbox_storage_rebuild_init(struct mdbox_storage *storage,
+			   struct mdbox_map_atomic_context *atomic)
+{
+	struct mdbox_storage_rebuild_context *ctx;
+
+	i_assert(!storage->rebuilding_storage);
+
+	ctx = i_new(struct mdbox_storage_rebuild_context, 1);
+	ctx->storage = storage;
+	ctx->atomic = atomic;
+	ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256);
+	hash_table_create(&ctx->guid_hash, ctx->pool, 0,
+			  guid_128_hash, guid_128_cmp);
+	i_array_init(&ctx->msgs, 512);
+	i_array_init(&ctx->seen_file_ids, 128);
+
+	ctx->storage->rebuilding_storage = TRUE;
+	return ctx;
+}
+
+static void
+mdbox_storage_rebuild_deinit(struct mdbox_storage_rebuild_context *ctx)
+{
+	i_assert(ctx->storage->rebuilding_storage);
+
+	ctx->storage->rebuilding_storage = FALSE;
+
+	hash_table_destroy(&ctx->guid_hash);
+	pool_unref(&ctx->pool);
+	array_free(&ctx->seen_file_ids);
+	array_free(&ctx->msgs);
+	i_free(ctx);
+}
+
+static int
+mdbox_rebuild_msg_offset_cmp(struct mdbox_rebuild_msg *const *m1,
+			     struct mdbox_rebuild_msg *const *m2)
+{
+	if ((*m1)->file_id < (*m2)->file_id)
+		return -1;
+	if ((*m1)->file_id > (*m2)->file_id)
+		return 1;
+
+	if ((*m1)->offset < (*m2)->offset)
+		return -1;
+	if ((*m1)->offset > (*m2)->offset)
+		return 1;
+
+	if ((*m1)->rec_size < (*m2)->rec_size)
+		return -1;
+	if ((*m1)->rec_size > (*m2)->rec_size)
+		return 1;
+	return 0;
+}
+
+static int mdbox_rebuild_msg_uid_cmp(struct mdbox_rebuild_msg *const *m1,
+				     struct mdbox_rebuild_msg *const *m2)
+{
+	if ((*m1)->map_uid < (*m2)->map_uid)
+		return -1;
+	if ((*m1)->map_uid > (*m2)->map_uid)
+		return 1;
+	return 0;
+}
+
+static void rebuild_scan_metadata(struct mdbox_storage_rebuild_context *ctx,
+				  struct dbox_file *file)
+{
+	if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_UIDL) != NULL)
+		ctx->have_pop3_uidls = TRUE;
+	if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_ORDER) != NULL)
+		ctx->have_pop3_orders = TRUE;
+}
+
+static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
+			      struct dbox_file *file, uint32_t file_id)
+{
+	const char *guid;
+	uint8_t *guid_p;
+	struct mdbox_rebuild_msg *rec, *old_rec;
+	uoff_t offset, prev_offset;
+	bool last, first, fixed = FALSE;
+	int ret;
+
+	dbox_file_seek_rewind(file);
+	prev_offset = 0;
+	while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) {
+		if (ret > 0) {
+			if ((ret = dbox_file_metadata_read(file)) < 0)
+				break;
+		}
+
+		if (ret == 0) {
+			/* file is corrupted. fix it and retry. */
+			if (fixed || last)
+				break;
+			first = prev_offset == 0;
+			if (prev_offset == 0) {
+				/* use existing file header if it was ok */
+				prev_offset = offset;
+			}
+			if ((ret = dbox_file_fix(file, prev_offset)) < 0)
+				break;
+			if (ret == 0) {
+				/* file was deleted */
+				return 1;
+			}
+			fixed = TRUE;
+			if (!first) {
+				/* seek to the offset where we last left off */
+				ret = dbox_file_seek(file, prev_offset);
+				if (ret <= 0)
+					break;
+			}
+			continue;
+		}
+		prev_offset = offset;
+
+		guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID);
+		if (guid == NULL || *guid == '\0') {
+			dbox_file_set_corrupted(file,
+						"Message is missing GUID");
+			ret = 0;
+			break;
+		}
+		rebuild_scan_metadata(ctx, file);
+
+		rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1);
+		rec->file_id = file_id;
+		rec->offset = offset;
+		rec->rec_size = file->input->v_offset - offset;
+		rec->mail_size = dbox_file_get_plaintext_size(file);
+		mail_generate_guid_128_hash(guid, rec->guid_128);
+		i_assert(!guid_128_is_empty(rec->guid_128));
+		array_push_back(&ctx->msgs, &rec);
+
+		guid_p = rec->guid_128;
+		old_rec = hash_table_lookup(ctx->guid_hash, guid_p);
+		if (old_rec == NULL)
+			hash_table_insert(ctx->guid_hash, guid_p, rec);
+		else if (rec->mail_size == old_rec->mail_size) {
+			/* two mails' GUID and size are the same, which quite
+			   likely means that their contents are the same as
+			   well. we'll compare the mail sizes instead of the
+			   record sizes, because the records' metadata may
+			   differ.
+
+			   save this duplicate mail with refcount=0 to the map,
+			   so it will eventually be purged. */
+			rec->seen_zero_ref_in_map = TRUE;
+		} else {
+			/* duplicate GUID, but not a duplicate message. */
+			i_error("mdbox %s: Duplicate GUID %s in "
+				"m.%u:%u (size=%"PRIuUOFF_T") and m.%u:%u "
+				"(size=%"PRIuUOFF_T")",
+				ctx->storage->storage_dir, guid,
+				old_rec->file_id, old_rec->offset, old_rec->mail_size,
+				rec->file_id, rec->offset, rec->mail_size);
+			rec->guid_hash_next = old_rec->guid_hash_next;
+			old_rec->guid_hash_next = rec;
+		}
+	}
+	if (ret < 0)
+		return -1;
+	else if (ret == 0 && !last)
+		return 0;
+	else
+		return 1;
+}
+
+static int
+rebuild_rename_file(struct mdbox_storage_rebuild_context *ctx,
+		    const char *dir, const char **fname_p, uint32_t *file_id_r)
+{
+	const char *old_path, *new_path, *fname = *fname_p;
+
+	old_path = t_strconcat(dir, "/", fname, NULL);
+	do {
+		new_path = t_strdup_printf("%s/"MDBOX_MAIL_FILE_FORMAT,
+					   dir, ++ctx->highest_file_id);
+		/* use link()+unlink() instead of rename() to make sure we
+		   don't overwrite any files. */
+		if (link(old_path, new_path) == 0) {
+			i_unlink(old_path);
+			*fname_p = strrchr(new_path, '/') + 1;
+			*file_id_r = ctx->highest_file_id;
+			return 0;
+		}
+	} while (errno == EEXIST);
+
+	i_error("link(%s, %s) failed: %m", old_path, new_path);
+	return -1;
+}
+
+static int rebuild_add_file(struct mdbox_storage_rebuild_context *ctx,
+			    const char *dir, const char *fname)
+{
+	struct dbox_file *file;
+	uint32_t file_id;
+	const char *id_str, *ext;
+	bool deleted;
+	int ret = 0;
+
+	id_str = fname + strlen(MDBOX_MAIL_FILE_PREFIX);
+	if (str_to_uint32(id_str, &file_id) < 0 || file_id == 0) {
+		/* m.*.broken files are created by file fixing
+		   m.*.lock files are created if flock() isn't available */
+		ext = strrchr(id_str, '.');
+		if (ext == NULL || (strcmp(ext, ".broken") != 0 &&
+				    strcmp(ext, ".lock") != 0)) {
+			i_warning("mdbox rebuild: "
+				  "Skipping file with missing ID: %s/%s",
+				  dir, fname);
+		}
+		return 0;
+	}
+	if (!seq_range_exists(&ctx->seen_file_ids, file_id)) {
+		if (ctx->highest_file_id < file_id)
+			ctx->highest_file_id = file_id;
+	} else {
+		/* duplicate file. either readdir() returned it twice
+		   (unlikely) or it exists in both alt and primary storage.
+		   to make sure we don't lose any mails from either of the
+		   files, give this file a new ID and rename it. */
+		if (rebuild_rename_file(ctx, dir, &fname, &file_id) < 0)
+			return -1;
+	}
+	seq_range_array_add(&ctx->seen_file_ids, file_id);
+
+	file = mdbox_file_init(ctx->storage, file_id);
+	if ((ret = dbox_file_open(file, &deleted)) > 0 && !deleted)
+		ret = rebuild_file_mails(ctx, file, file_id);
+	if (ret == 0)
+		i_error("mdbox rebuild: Failed to fix file %s/%s", dir, fname);
+	dbox_file_unref(&file);
+	return ret < 0 ? -1 : 0;
+}
+
+static void
+rebuild_add_missing_map_uids(struct mdbox_storage_rebuild_context *ctx,
+			     uint32_t next_uid)
+{
+	struct mdbox_rebuild_msg **msgs;
+	struct mdbox_map_mail_index_record rec;
+	unsigned int i, count;
+	uint32_t seq;
+
+	i_zero(&rec);
+	msgs = array_get_modifiable(&ctx->msgs, &count);
+	for (i = 0; i < count; i++) {
+		if (msgs[i]->map_uid != 0)
+			continue;
+
+		rec.file_id = msgs[i]->file_id;
+		rec.offset = msgs[i]->offset;
+		rec.size = msgs[i]->rec_size;
+
+		msgs[i]->map_uid = next_uid++;
+		mail_index_append(ctx->atomic->sync_trans,
+				  msgs[i]->map_uid, &seq);
+		mail_index_update_ext(ctx->atomic->sync_trans, seq,
+				      ctx->storage->map->map_ext_id,
+				      &rec, NULL);
+	}
+}
+
+static void rebuild_apply_map(struct mdbox_storage_rebuild_context *ctx)
+{
+	struct mdbox_map *map = ctx->storage->map;
+	const struct mail_index_header *hdr;
+	struct mdbox_rebuild_msg **pos;
+	struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
+	struct dbox_mail_lookup_rec rec;
+	uint32_t seq;
+
+	array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp);
+	/* msgs now contains a list of all messages that exists in m.* files,
+	   sorted by file_id,offset */
+
+	hdr = mail_index_get_header(ctx->atomic->sync_view);
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		if (mdbox_map_view_lookup_rec(map, ctx->atomic->sync_view,
+					      seq, &rec) < 0) {
+			/* map or ref extension is missing from the index.
+			   Just ignore the file entirely. (Don't try to
+			   continue with other records, since they'll fail
+			   as well, and each failure logs the same error.) */
+			i_assert(seq == 1);
+			break;
+		}
+
+		/* look up the rebuild msg record for this message based on
+		   the (file_id, offset, size) triplet */
+		search_msg.file_id = rec.rec.file_id;
+		search_msg.offset = rec.rec.offset;
+		search_msg.rec_size = rec.rec.size;
+		pos = array_bsearch(&ctx->msgs, &search_msgp,
+				    mdbox_rebuild_msg_offset_cmp);
+		if (pos == NULL || (*pos)->map_uid != 0) {
+			/* map record points to nonexistent or
+			   a duplicate message. */
+			mail_index_expunge(ctx->atomic->sync_trans, seq);
+		} else {
+			/* remember this message's map_uid */
+			(*pos)->map_uid = rec.map_uid;
+			if (rec.refcount == 0)
+				(*pos)->seen_zero_ref_in_map = TRUE;
+		}
+	}
+	rebuild_add_missing_map_uids(ctx, hdr->next_uid);
+
+	/* afterwards we're interested in looking up map_uids.
+	   re-sort the messages to make it easier. */
+	array_sort(&ctx->msgs, mdbox_rebuild_msg_uid_cmp);
+}
+
+static struct mdbox_rebuild_msg *
+rebuild_lookup_map_uid(struct mdbox_storage_rebuild_context *ctx,
+		       uint32_t map_uid)
+{
+	struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
+	struct mdbox_rebuild_msg **pos;
+
+	search_msg.map_uid = map_uid;
+	pos = array_bsearch(&ctx->msgs, &search_msgp,
+			    mdbox_rebuild_msg_uid_cmp);
+	return pos == NULL ? NULL : *pos;
+}
+
+static bool
+guid_hash_have_map_uid(struct mdbox_rebuild_msg **recp, uint32_t map_uid)
+{
+	struct mdbox_rebuild_msg *rec;
+
+	for (rec = *recp; rec != NULL; rec = rec->guid_hash_next) {
+		if (rec->map_uid == map_uid) {
+			*recp = rec;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static void
+rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx,
+		      struct index_rebuild_context *rebuild_ctx,
+		      struct mdbox_mailbox *mbox,
+		      struct mail_index_view *view,
+		      struct mail_index_transaction *trans)
+{
+	struct mdbox_mail_index_record new_dbox_rec;
+	const struct mail_index_header *hdr;
+	struct mdbox_rebuild_msg *rec;
+	const void *data;
+	const uint8_t *guid_p;
+	uint32_t old_seq, new_seq, uid, map_uid;
+
+	/* Rebuild the mailbox's index. Note that index is reset at this point,
+	   so although we can still access the old messages, we'll need to
+	   append anything we want to keep as new messages. */
+	hdr = mail_index_get_header(view);
+	for (old_seq = 1; old_seq <= hdr->messages_count; old_seq++) {
+		mail_index_lookup_ext(view, old_seq, mbox->ext_id,
+				      &data, NULL);
+		if (data == NULL) {
+			i_zero(&new_dbox_rec);
+			map_uid = 0;
+		} else {
+			memcpy(&new_dbox_rec, data, sizeof(new_dbox_rec));
+			map_uid = new_dbox_rec.map_uid;
+		}
+
+		mail_index_lookup_ext(view, old_seq, mbox->guid_ext_id,
+				      &data, NULL);
+		guid_p = data;
+
+		/* see if we can find this message based on
+		   1) GUID, 2) map_uid */
+		rec = guid_p == NULL ? NULL :
+			hash_table_lookup(ctx->guid_hash, guid_p);
+		if (rec == NULL) {
+			/* multi-dbox message that wasn't found with GUID.
+			   either it's lost or GUID has been corrupted. we can
+			   still try to look it up using map_uid. */
+			rec = map_uid == 0 ? NULL :
+				rebuild_lookup_map_uid(ctx, map_uid);
+			map_uid = rec == NULL ? 0 : rec->map_uid;
+		} else if (!guid_hash_have_map_uid(&rec, map_uid)) {
+			/* message's GUID and map_uid point to different
+			   physical messages. assume that GUID is correct and
+			   map_uid is wrong. */
+			map_uid = rec->map_uid;
+		} else {
+			/* everything was ok. use this specific record's
+			   map_uid to avoid duplicating mails in case the same
+			   GUID exists multiple times */
+		}
+
+		if (rec != NULL &&
+		    rec->refcount < REBUILD_MAX_REFCOUNT) T_BEGIN {
+			/* keep this message. add it to mailbox index. */
+			i_assert(map_uid != 0);
+			rec->refcount++;
+
+			mail_index_lookup_uid(view, old_seq, &uid);
+			mail_index_append(trans, uid, &new_seq);
+			index_rebuild_index_metadata(rebuild_ctx,
+						     new_seq, uid);
+
+			new_dbox_rec.map_uid = map_uid;
+			mail_index_update_ext(trans, new_seq, mbox->ext_id,
+					      &new_dbox_rec, NULL);
+			mail_index_update_ext(trans, new_seq, mbox->guid_ext_id,
+					      rec->guid_128, NULL);
+		} T_END;
+	}
+}
+
+static void
+mdbox_rebuild_get_header(struct mail_index_view *view, uint32_t hdr_ext_id,
+			 struct mdbox_index_header *hdr_r, bool *need_resize_r)
+{
+	const void *data;
+	size_t data_size;
+
+	mail_index_get_header_ext(view, hdr_ext_id, &data, &data_size);
+	i_zero(hdr_r);
+	memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
+	*need_resize_r = data_size < sizeof(*hdr_r);
+}
+
+static void mdbox_header_update(struct mdbox_storage_rebuild_context *ctx,
+				struct index_rebuild_context *rebuild_ctx,
+				struct mdbox_mailbox *mbox)
+{
+	struct mdbox_index_header hdr, backup_hdr;
+	bool need_resize, need_resize_backup;
+
+	mdbox_rebuild_get_header(rebuild_ctx->view, mbox->hdr_ext_id,
+				 &hdr, &need_resize);
+	if (rebuild_ctx->backup_view == NULL) {
+		i_zero(&backup_hdr);
+		need_resize = TRUE;
+	} else {
+		mdbox_rebuild_get_header(rebuild_ctx->backup_view,
+					 mbox->hdr_ext_id, &backup_hdr,
+					 &need_resize_backup);
+	}
+
+	/* make sure we have valid mailbox guid */
+	if (guid_128_is_empty(hdr.mailbox_guid)) {
+		if (!guid_128_is_empty(backup_hdr.mailbox_guid)) {
+			memcpy(hdr.mailbox_guid, backup_hdr.mailbox_guid,
+			       sizeof(hdr.mailbox_guid));
+		} else {
+			guid_128_generate(hdr.mailbox_guid);
+		}
+	}
+
+	/* update map's uid-validity */
+	hdr.map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map);
+
+	if (ctx->have_pop3_uidls)
+		hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS;
+	if (ctx->have_pop3_orders)
+		hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS;
+
+	/* and write changes */
+	if (need_resize) {
+		mail_index_ext_resize_hdr(rebuild_ctx->trans, mbox->hdr_ext_id,
+					  sizeof(hdr));
+	}
+	mail_index_update_header_ext(rebuild_ctx->trans, mbox->hdr_ext_id, 0,
+				     &hdr, sizeof(hdr));
+}
+
+static int
+rebuild_mailbox(struct mdbox_storage_rebuild_context *ctx,
+		struct mail_namespace *ns, const char *vname)
+{
+	struct mailbox *box;
+	struct mdbox_mailbox *mbox;
+        struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	struct index_rebuild_context *rebuild_ctx;
+	enum mail_error error;
+	int ret;
+
+	box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY |
+			    MAILBOX_FLAG_IGNORE_ACLS);
+	if (box->storage != &ctx->storage->storage.storage) {
+		/* the namespace has multiple storages. */
+		mailbox_free(&box);
+		return 0;
+	}
+	if (mailbox_open(box) < 0) {
+		error = mailbox_get_last_mail_error(box);
+		i_error("Couldn't open mailbox '%s': %s",
+			vname, mailbox_get_last_internal_error(box, NULL));
+		mailbox_free(&box);
+		if (error == MAIL_ERROR_TEMP)
+			return -1;
+		/* non-temporary error, ignore */
+		return 0;
+	}
+	mbox = MDBOX_MAILBOX(box);
+
+	ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans,
+				    MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES);
+	if (ret <= 0) {
+		i_assert(ret != 0);
+		mailbox_set_index_error(box);
+		mailbox_free(&box);
+		return -1;
+	}
+
+	rebuild_ctx = index_index_rebuild_init(&mbox->box, view, trans);
+	mdbox_header_update(ctx, rebuild_ctx, mbox);
+	rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
+	index_index_rebuild_deinit(&rebuild_ctx, dbox_get_uidvalidity_next);
+	mail_index_unset_fscked(trans);
+
+	mail_index_sync_set_reason(sync_ctx, "mdbox storage rebuild");
+	if (mail_index_sync_commit(&sync_ctx) < 0) {
+		mailbox_set_index_error(box);
+		ret = -1;
+	}
+
+	mailbox_free(&box);
+	return ret < 0 ? -1 : 0;
+}
+
+static int
+rebuild_namespace_mailboxes(struct mdbox_storage_rebuild_context *ctx,
+			    struct mail_namespace *ns)
+{
+	struct mailbox_list_iterate_context *iter;
+	const struct mailbox_info *info;
+	int ret = 0;
+
+	if (ctx->default_list == NULL ||
+	    (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0)
+		ctx->default_list = ns->list;
+
+	iter = mailbox_list_iter_init(ns->list, "*",
+				      MAILBOX_LIST_ITER_RAW_LIST |
+				      MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
+	while ((info = mailbox_list_iter_next(iter)) != NULL) {
+		if ((info->flags & (MAILBOX_NONEXISTENT |
+				    MAILBOX_NOSELECT)) == 0) {
+			T_BEGIN {
+				ret = rebuild_mailbox(ctx, ns, info->vname);
+			} T_END;
+			if (ret < 0) {
+				ret = -1;
+				break;
+			}
+		}
+	}
+	if (mailbox_list_iter_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+static int rebuild_mailboxes(struct mdbox_storage_rebuild_context *ctx)
+{
+	struct mail_storage *storage = &ctx->storage->storage.storage;
+	struct mail_namespace *ns;
+
+	for (ns = storage->user->namespaces; ns != NULL; ns = ns->next) {
+		if (ns->storage == storage && ns->alias_for == NULL) {
+			if (rebuild_namespace_mailboxes(ctx, ns) < 0)
+				return -1;
+		}
+	}
+	if (ctx->default_list == NULL)
+		i_panic("No namespace found for storage=%s", storage->name);
+	return 0;
+}
+
+static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg)
+{
+	mail_index_sync_set_reason(msg->sync_ctx, "mdbox storage rebuild");
+	if (mail_index_sync_commit(&msg->sync_ctx) < 0)
+		return -1;
+	mailbox_free(&msg->box);
+	i_zero(msg);
+	return 0;
+}
+
+static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx,
+			       struct mdbox_rebuild_msg *msg)
+{
+	struct mail_storage *storage = &ctx->storage->storage.storage;
+	struct dbox_file *file;
+	const struct mail_index_header *hdr;
+	struct mdbox_mail_index_record dbox_rec;
+	const char *mailbox = NULL;
+	struct mailbox *box;
+	struct mdbox_mailbox *mbox;
+	enum mail_error error;
+	bool deleted, created;
+	int ret;
+	uint32_t seq;
+
+	/* first see if message contains the mailbox it was originally
+	   saved to */
+	file = mdbox_file_init(ctx->storage, msg->file_id);
+	ret = dbox_file_open(file, &deleted);
+	if (ret > 0 && !deleted)
+		ret = dbox_file_seek(file, msg->offset);
+	if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) {
+		mailbox = dbox_file_metadata_get(file,
+						 DBOX_METADATA_ORIG_MAILBOX);
+		if (mailbox != NULL) {
+			mailbox = mailbox_list_get_vname(ctx->default_list, mailbox);
+			mailbox = t_strdup(mailbox);
+		}
+		rebuild_scan_metadata(ctx, file);
+	}
+	dbox_file_unref(&file);
+	if (ret <= 0 || deleted) {
+		if (ret < 0)
+			return -1;
+		/* we shouldn't get here, so apparently we couldn't fix
+		   something. just ignore the mail.. */
+		return 0;
+	}
+
+	if (mailbox == NULL)
+		mailbox = "INBOX";
+
+	/* we have the destination mailbox. now open it and add the message
+	   there. */
+	created = FALSE;
+	box = ctx->prev_msg.box != NULL &&
+		strcmp(mailbox, ctx->prev_msg.box->vname) == 0 ?
+		ctx->prev_msg.box : NULL;
+	while (box == NULL) {
+		box = mailbox_alloc(ctx->default_list, mailbox,
+				    MAILBOX_FLAG_READONLY |
+				    MAILBOX_FLAG_IGNORE_ACLS);
+		i_assert(box->storage == storage);
+		if (mailbox_open(box) == 0)
+			break;
+
+		error = mailbox_get_last_mail_error(box);
+		if (error == MAIL_ERROR_NOTFOUND && !created) {
+			/* mailbox doesn't exist currently? see if creating
+			   it helps. */
+			created = TRUE;
+			(void)mailbox_create(box, NULL, FALSE);
+			mailbox_free(&box);
+			continue;
+		}
+
+		mailbox_free(&box);
+		if (error == MAIL_ERROR_TEMP)
+			return -1;
+
+		if (strcmp(mailbox, "INBOX") != 0) {
+			/* see if we can save to INBOX instead. */
+			mailbox = "INBOX";
+		} else {
+			/* this shouldn't happen */
+			return -1;
+		}
+	}
+	mbox = MDBOX_MAILBOX(box);
+
+	/* switch the mailbox cache if necessary */
+	if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) {
+		if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
+			return -1;
+	}
+	if (ctx->prev_msg.box == NULL) {
+		ret = mail_index_sync_begin(box->index,
+					    &ctx->prev_msg.sync_ctx,
+					    &ctx->prev_msg.view,
+					    &ctx->prev_msg.trans, 0);
+		if (ret <= 0) {
+			i_assert(ret != 0);
+			mailbox_set_index_error(box);
+			mailbox_free(&box);
+			return -1;
+		}
+		ctx->prev_msg.box = box;
+		hdr = mail_index_get_header(ctx->prev_msg.view);
+		ctx->prev_msg.next_uid = hdr->next_uid;
+	}
+
+	/* add the new message */
+	i_zero(&dbox_rec);
+	dbox_rec.map_uid = msg->map_uid;
+	dbox_rec.save_date = ioloop_time;
+	mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq);
+	mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->ext_id,
+			      &dbox_rec, NULL);
+	mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id,
+			      msg->guid_128, NULL);
+
+	i_assert(msg->refcount == 0);
+	msg->refcount++;
+	return 0;
+}
+
+static int rebuild_handle_zero_refs(struct mdbox_storage_rebuild_context *ctx)
+{
+	struct mdbox_rebuild_msg **msgs;
+	unsigned int i, count;
+
+	/* if we have messages at this point which have refcount=0, they're
+	   either already expunged or they were somehow lost for some reason.
+	   we'll need to figure out what to do about them. */
+	msgs = array_get_modifiable(&ctx->msgs, &count);
+	for (i = 0; i < count; i++) {
+		if (msgs[i]->refcount != 0)
+			continue;
+
+		if (msgs[i]->seen_zero_ref_in_map) {
+			/* we've seen the map record, trust it. */
+			continue;
+		}
+		/* either map record was lost for this message or the message
+		   was lost from its mailbox. safest way to handle this is to
+		   restore the message. */
+		if (rebuild_restore_msg(ctx, msgs[i]) < 0)
+			return -1;
+	}
+	if (ctx->prev_msg.box != NULL) {
+		if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static void rebuild_update_refcounts(struct mdbox_storage_rebuild_context *ctx)
+{
+	const struct mail_index_header *hdr;
+	const void *data;
+	struct mdbox_rebuild_msg **msgs;
+	const uint16_t *ref16_p;
+	uint32_t seq, map_uid;
+	unsigned int i, count;
+
+	/* update refcounts for existing map records */
+	msgs = array_get_modifiable(&ctx->msgs, &count);
+	hdr = mail_index_get_header(ctx->atomic->sync_view);
+	for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) {
+		mail_index_lookup_uid(ctx->atomic->sync_view, seq, &map_uid);
+		if (map_uid != msgs[i]->map_uid) {
+			/* we've already expunged this map record */
+			i_assert(map_uid < msgs[i]->map_uid);
+			continue;
+		}
+
+		mail_index_lookup_ext(ctx->atomic->sync_view, seq,
+				      ctx->storage->map->ref_ext_id,
+				      &data, NULL);
+		ref16_p = data;
+		if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) {
+			mail_index_update_ext(ctx->atomic->sync_trans, seq,
+					      ctx->storage->map->ref_ext_id,
+					      &msgs[i]->refcount, NULL);
+		}
+		i++;
+	}
+
+	/* update refcounts for newly created map records */
+	for (; i < count; i++, seq++) {
+		mail_index_update_ext(ctx->atomic->sync_trans, seq,
+				      ctx->storage->map->ref_ext_id,
+				      &msgs[i]->refcount, NULL);
+	}
+}
+
+static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx)
+{
+	struct mdbox_map_mail_index_header map_hdr;
+
+	i_assert(ctx->default_list != NULL);
+
+	if (rebuild_handle_zero_refs(ctx) < 0)
+		return -1;
+	rebuild_update_refcounts(ctx);
+
+	/* update map header */
+	map_hdr = ctx->orig_map_hdr;
+	map_hdr.highest_file_id = ctx->highest_file_id;
+	map_hdr.rebuild_count = ++ctx->rebuild_count;
+
+	mail_index_update_header_ext(ctx->atomic->sync_trans,
+				     ctx->storage->map->map_ext_id,
+				     0, &map_hdr, sizeof(map_hdr));
+	return 0;
+}
+
+static int
+mdbox_storage_rebuild_scan_dir(struct mdbox_storage_rebuild_context *ctx,
+			       const char *storage_dir, bool alt)
+{
+	DIR *dir;
+	struct dirent *d;
+	int ret = 0;
+
+	dir = opendir(storage_dir);
+	if (dir == NULL) {
+		if (alt && errno == ENOENT)
+			return 0;
+
+		mail_storage_set_critical(&ctx->storage->storage.storage,
+			"opendir(%s) failed: %m", storage_dir);
+		return -1;
+	}
+	for (errno = 0; (d = readdir(dir)) != NULL && ret == 0; errno = 0) {
+		if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
+			    strlen(MDBOX_MAIL_FILE_PREFIX)) == 0) T_BEGIN {
+			ret = rebuild_add_file(ctx, storage_dir, d->d_name);
+		} T_END;
+	}
+	if (ret == 0 && errno != 0) {
+		mail_storage_set_critical(&ctx->storage->storage.storage,
+			"readdir(%s) failed: %m", storage_dir);
+		ret = -1;
+	}
+	if (closedir(dir) < 0) {
+		mail_storage_set_critical(&ctx->storage->storage.storage,
+			"closedir(%s) failed: %m", storage_dir);
+		ret = -1;
+	}
+	return ret;
+}
+
+static int
+mdbox_storage_rebuild_scan_prepare(struct mdbox_storage_rebuild_context *ctx)
+{
+	const void *data;
+	size_t data_size;
+
+	if (mdbox_map_open_or_create(ctx->storage->map) < 0)
+		return -1;
+
+	/* begin by locking the map, so that other processes can't try to
+	   rebuild at the same time. */
+	if (mdbox_map_atomic_lock(ctx->atomic, "mdbox storage rebuild") < 0)
+		return -1;
+
+	/* fsck the map just in case its UIDs are broken */
+	if (mail_index_fsck(ctx->storage->map->index) < 0) {
+		mail_storage_set_index_error(&ctx->storage->storage.storage,
+					     ctx->storage->map->index);
+		return -1;
+	}
+
+	/* get old map header */
+	mail_index_get_header_ext(ctx->atomic->sync_view,
+				  ctx->storage->map->map_ext_id,
+				  &data, &data_size);
+	i_zero(&ctx->orig_map_hdr);
+	memcpy(&ctx->orig_map_hdr, data,
+	       I_MIN(data_size, sizeof(ctx->orig_map_hdr)));
+	ctx->highest_file_id = ctx->orig_map_hdr.highest_file_id;
+
+	/* get storage rebuild counter after locking */
+	ctx->rebuild_count = mdbox_map_get_rebuild_count(ctx->storage->map);
+	if (ctx->rebuild_count != ctx->storage->corrupted_rebuild_count &&
+	    ctx->storage->corrupted) {
+		/* storage was already rebuilt by someone else */
+		return 0;
+	}
+	return 1;
+}
+
+static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx)
+{
+	i_warning("mdbox %s: rebuilding indexes", ctx->storage->storage_dir);
+
+	if (mdbox_storage_rebuild_scan_dir(ctx, ctx->storage->storage_dir,
+					   FALSE) < 0)
+		return -1;
+	if (ctx->storage->alt_storage_dir != NULL) {
+		if (mdbox_storage_rebuild_scan_dir(ctx,
+				ctx->storage->alt_storage_dir, TRUE) < 0)
+			return -1;
+	}
+
+	rebuild_apply_map(ctx);
+	if (rebuild_mailboxes(ctx) < 0 ||
+	    rebuild_finish(ctx) < 0) {
+		mdbox_map_atomic_set_failed(ctx->atomic);
+		return -1;
+	}
+	return 0;
+}
+
+int mdbox_storage_rebuild_in_context(struct mdbox_storage *storage,
+				     struct mdbox_map_atomic_context *atomic)
+{
+	struct mdbox_storage_rebuild_context *ctx;
+	int ret;
+
+	if (dbox_verify_alt_storage(storage->map->root_list) < 0) {
+		mail_storage_set_critical(&storage->storage.storage,
+			"mdbox rebuild: Alt storage %s not mounted, aborting",
+			storage->alt_storage_dir);
+		mdbox_map_atomic_set_failed(atomic);
+		return -1;
+	}
+
+	ctx = mdbox_storage_rebuild_init(storage, atomic);
+	if ((ret = mdbox_storage_rebuild_scan_prepare(ctx)) > 0) {
+		struct event_reason *reason = event_reason_begin("mdbox:rebuild");
+		ret = mdbox_storage_rebuild_scan(ctx);
+		event_reason_end(&reason);
+	}
+	mdbox_storage_rebuild_deinit(ctx);
+
+	if (ret == 0) {
+		storage->corrupted = FALSE;
+		storage->corrupted_rebuild_count = 0;
+	}
+	return ret;
+}
+
+int mdbox_storage_rebuild(struct mdbox_storage *storage)
+{
+	struct mdbox_map_atomic_context *atomic;
+	int ret;
+
+	atomic = mdbox_map_atomic_begin(storage->map);
+	ret = mdbox_storage_rebuild_in_context(storage, atomic);
+	mdbox_map_atomic_set_success(atomic);
+	mdbox_map_atomic_unset_fscked(atomic);
+	if (mdbox_map_atomic_finish(&atomic) < 0)
+		ret = -1;
+	return ret;
+}
diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h
new file mode 100644
index 0000000..1194d29
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.h
@@ -0,0 +1,10 @@
+#ifndef MDBOX_STORAGE_REBUILD_H
+#define MDBOX_STORAGE_REBUILD_H
+
+struct mdbox_map_atomic_context;
+
+int mdbox_storage_rebuild_in_context(struct mdbox_storage *storage,
+				     struct mdbox_map_atomic_context *atomic);
+int mdbox_storage_rebuild(struct mdbox_storage *storage);
+
+#endif
diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.c b/src/lib-storage/index/dbox-multi/mdbox-storage.c
new file mode 100644
index 0000000..e7e7f60
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c
@@ -0,0 +1,530 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "mkdir-parents.h"
+#include "master-service.h"
+#include "mail-index-modseq.h"
+#include "mail-index-alloc-cache.h"
+#include "mailbox-log.h"
+#include "mailbox-list-private.h"
+#include "index-pop3-uidl.h"
+#include "dbox-mail.h"
+#include "dbox-save.h"
+#include "mdbox-map.h"
+#include "mdbox-file.h"
+#include "mdbox-sync.h"
+#include "mdbox-storage-rebuild.h"
+#include "mdbox-storage.h"
+
+extern struct mail_storage mdbox_storage;
+extern struct mailbox mdbox_mailbox;
+
+static struct event_category event_category_mdbox = {
+	.name = "mdbox",
+	.parent = &event_category_storage,
+};
+
+static struct mail_storage *mdbox_storage_alloc(void)
+{
+	struct mdbox_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mdbox storage", 2048);
+	storage = p_new(pool, struct mdbox_storage, 1);
+	storage->storage.v = mdbox_dbox_storage_vfuncs;
+	storage->storage.storage = mdbox_storage;
+	storage->storage.storage.pool = pool;
+	return &storage->storage.storage;
+}
+
+int mdbox_storage_create(struct mail_storage *_storage,
+			 struct mail_namespace *ns, const char **error_r)
+{
+	struct mdbox_storage *storage = MDBOX_STORAGE(_storage);
+	const char *dir;
+
+	storage->set = mail_namespace_get_driver_settings(ns, _storage);
+	storage->preallocate_space = storage->set->mdbox_preallocate_space;
+
+	if (*ns->list->set.mailbox_dir_name == '\0') {
+		*error_r = "mdbox: MAILBOXDIR must not be empty";
+		return -1;
+	}
+
+	_storage->unique_root_dir =
+		p_strdup(_storage->pool, ns->list->set.root_dir);
+
+	dir = mailbox_list_get_root_forced(ns->list, MAILBOX_LIST_PATH_TYPE_DIR);
+	storage->storage_dir = p_strconcat(_storage->pool, dir,
+					   "/"MDBOX_GLOBAL_DIR_NAME, NULL);
+	if (ns->list->set.alt_dir != NULL) {
+		storage->alt_storage_dir = p_strconcat(_storage->pool,
+							ns->list->set.alt_dir,
+							"/"MDBOX_GLOBAL_DIR_NAME, NULL);
+	}
+	i_array_init(&storage->open_files, 64);
+
+	storage->map = mdbox_map_init(storage, ns->list);
+	return dbox_storage_create(_storage, ns, error_r);
+}
+
+void mdbox_storage_destroy(struct mail_storage *_storage)
+{
+	struct mdbox_storage *storage = MDBOX_STORAGE(_storage);
+
+	mdbox_files_free(storage);
+	mdbox_map_deinit(&storage->map);
+	timeout_remove(&storage->to_close_unused_files);
+
+	if (array_is_created(&storage->move_from_alt_map_uids))
+		array_free(&storage->move_from_alt_map_uids);
+	if (array_is_created(&storage->move_to_alt_map_uids))
+		array_free(&storage->move_to_alt_map_uids);
+	array_free(&storage->open_files);
+	dbox_storage_destroy(_storage);
+}
+
+static const char *
+mdbox_storage_find_root_dir(const struct mail_namespace *ns)
+{
+	bool debug = ns->mail_set->mail_debug;
+	const char *home, *path;
+
+	if (ns->owner != NULL &&
+	    mail_user_get_home(ns->owner, &home) > 0) {
+		path = t_strconcat(home, "/mdbox", NULL);
+		if (access(path, R_OK|W_OK|X_OK) == 0) {
+			if (debug)
+				i_debug("mdbox: root exists (%s)", path);
+			return path;
+		} 
+		if (debug)
+			i_debug("mdbox: access(%s, rwx): failed: %m", path);
+	}
+	return NULL;
+}
+
+static bool mdbox_storage_autodetect(const struct mail_namespace *ns,
+				     struct mailbox_list_settings *set)
+{
+	bool debug = ns->mail_set->mail_debug;
+	struct stat st;
+	const char *path, *root_dir;
+
+	if (set->root_dir != NULL)
+		root_dir = set->root_dir;
+	else {
+		root_dir = mdbox_storage_find_root_dir(ns);
+		if (root_dir == NULL) {
+			if (debug)
+				i_debug("mdbox: couldn't find root dir");
+			return FALSE;
+		}
+	}
+
+	path = t_strconcat(root_dir, "/"MDBOX_GLOBAL_DIR_NAME, NULL);
+	if (stat(path, &st) < 0) {
+		if (debug)
+			i_debug("mdbox autodetect: stat(%s) failed: %m", path);
+		return FALSE;
+	}
+
+	if (!S_ISDIR(st.st_mode)) {
+		if (debug)
+			i_debug("mdbox autodetect: %s not a directory", path);
+		return FALSE;
+	}
+
+	set->root_dir = root_dir;
+	dbox_storage_get_list_settings(ns, set);
+	return TRUE;
+}
+
+static struct mailbox *
+mdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
+		    const char *vname, enum mailbox_flags flags)
+{
+	struct mdbox_mailbox *mbox;
+	struct index_mailbox_context *ibox;
+	pool_t pool;
+
+	/* dbox can't work without index files */
+	flags &= ENUM_NEGATE(MAILBOX_FLAG_NO_INDEX_FILES);
+
+	pool = pool_alloconly_create("mdbox mailbox", 1024*3);
+	mbox = p_new(pool, struct mdbox_mailbox, 1);
+	mbox->box = mdbox_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.mail_vfuncs = &mdbox_mail_vfuncs;
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
+
+	ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
+	ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS |
+		MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY;
+
+	mbox->storage = MDBOX_STORAGE(storage);
+	return &mbox->box;
+}
+
+int mdbox_mailbox_open(struct mailbox *box)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box);
+	time_t path_ctime;
+
+	if (dbox_mailbox_check_existence(box, &path_ctime) < 0)
+		return -1;
+	if (dbox_mailbox_open(box, path_ctime) < 0)
+		return -1;
+
+	mbox->ext_id =
+		mail_index_ext_register(mbox->box.index, "mdbox", 0,
+					sizeof(struct mdbox_mail_index_record),
+					sizeof(uint32_t));
+	mbox->hdr_ext_id =
+		mail_index_ext_register(mbox->box.index, "mdbox-hdr",
+					sizeof(struct mdbox_index_header), 0, 0);
+	mbox->guid_ext_id =
+		mail_index_ext_register(mbox->box.index, "guid",
+					0, GUID_128_SIZE, 1);
+	return 0;
+}
+
+static void mdbox_mailbox_close(struct mailbox *box)
+{
+	struct mdbox_storage *mstorage = MDBOX_STORAGE(box->storage);
+
+	if (mstorage->corrupted && !mstorage->rebuilding_storage)
+		(void)mdbox_storage_rebuild(mstorage);
+
+	index_storage_mailbox_close(box);
+}
+
+int mdbox_read_header(struct mdbox_mailbox *mbox,
+		      struct mdbox_index_header *hdr, bool *need_resize_r)
+{
+	const void *data;
+	size_t data_size;
+
+	i_assert(mbox->box.opened);
+
+	mail_index_get_header_ext(mbox->box.view, mbox->hdr_ext_id,
+				  &data, &data_size);
+	if (data_size < MDBOX_INDEX_HEADER_MIN_SIZE &&
+	    (!mbox->creating || data_size != 0)) {
+		mailbox_set_critical(&mbox->box,
+			"mdbox: Invalid dbox header size: %zu",
+			data_size);
+		mdbox_storage_set_corrupted(mbox->storage);
+		return -1;
+	}
+	i_zero(hdr);
+	if (data_size > 0)
+		memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr)));
+	*need_resize_r = data_size < sizeof(*hdr);
+	return 0;
+}
+
+void mdbox_update_header(struct mdbox_mailbox *mbox,
+			 struct mail_index_transaction *trans,
+			 const struct mailbox_update *update)
+{
+	struct mdbox_index_header hdr, new_hdr;
+	bool need_resize;
+
+	if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) {
+		i_zero(&hdr);
+		need_resize = TRUE;
+	}
+
+	new_hdr = hdr;
+
+	if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) {
+		memcpy(new_hdr.mailbox_guid, update->mailbox_guid,
+		       sizeof(new_hdr.mailbox_guid));
+	} else if (guid_128_is_empty(new_hdr.mailbox_guid)) {
+		guid_128_generate(new_hdr.mailbox_guid);
+	}
+
+	new_hdr.map_uid_validity =
+		mdbox_map_get_uid_validity(mbox->storage->map);
+	if (need_resize) {
+		mail_index_ext_resize_hdr(trans, mbox->hdr_ext_id,
+					  sizeof(new_hdr));
+	}
+	if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) {
+		mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0,
+					     &new_hdr, sizeof(new_hdr));
+	}
+}
+
+static int ATTR_NULL(2, 3)
+mdbox_write_index_header(struct mailbox *box,
+			 const struct mailbox_update *update,
+			 struct mail_index_transaction *trans)
+{
+	struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box;
+	struct mail_index_transaction *new_trans = NULL;
+	struct mail_index_view *view;
+	const struct mail_index_header *hdr;
+	uint32_t uid_validity, uid_next;
+
+	if (mdbox_map_open_or_create(mbox->storage->map) < 0)
+		return -1;
+
+	if (trans == NULL) {
+		new_trans = mail_index_transaction_begin(box->view, 0);
+		trans = new_trans;
+	}
+
+	view = mail_index_view_open(box->index);
+	hdr = mail_index_get_header(view);
+	uid_validity = hdr->uid_validity;
+	if (update != NULL && update->uid_validity != 0)
+		uid_validity = update->uid_validity;
+	else if (uid_validity == 0) {
+		/* set uidvalidity */
+		uid_validity = dbox_get_uidvalidity_next(box->list);
+	}
+
+	if (hdr->uid_validity != uid_validity) {
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+	if (update != NULL && hdr->next_uid < update->min_next_uid) {
+		uid_next = update->min_next_uid;
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, next_uid),
+			&uid_next, sizeof(uid_next), TRUE);
+	}
+	if (update != NULL && update->min_first_recent_uid != 0 &&
+	    hdr->first_recent_uid < update->min_first_recent_uid) {
+		uint32_t first_recent_uid = update->min_first_recent_uid;
+
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, first_recent_uid),
+			&first_recent_uid, sizeof(first_recent_uid), FALSE);
+	}
+	if (update != NULL && update->min_highest_modseq != 0 &&
+	    mail_index_modseq_get_highest(view) < update->min_highest_modseq) {
+		mail_index_modseq_enable(box->index);
+		mail_index_update_highest_modseq(trans,
+						 update->min_highest_modseq);
+	}
+	mail_index_view_close(&view);
+
+	if (box->inbox_user && box->creating) {
+		/* initialize pop3-uidl header when creating mailbox
+		   (not on mailbox_update()) */
+		index_pop3_uidl_set_max_uid(box, trans, 0);
+	}
+
+	mdbox_update_header(mbox, trans, update);
+	if (new_trans != NULL) {
+		if (mail_index_transaction_commit(&new_trans) < 0) {
+			mailbox_set_index_error(box);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+int mdbox_mailbox_create_indexes(struct mailbox *box,
+				 const struct mailbox_update *update,
+				 struct mail_index_transaction *trans)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box);
+	int ret;
+
+	mbox->creating = TRUE;
+	ret = mdbox_write_index_header(box, update, trans);
+	mbox->creating = FALSE;
+	return ret;
+}
+
+void mdbox_storage_set_corrupted(struct mdbox_storage *storage)
+{
+	if (storage->corrupted) {
+		/* already set it corrupted (possibly recursing back here) */
+		return;
+	}
+
+	storage->corrupted = TRUE;
+	storage->corrupted_rebuild_count = (uint32_t)-1;
+
+	if (mdbox_map_open(storage->map) > 0 &&
+	    mdbox_map_refresh(storage->map) == 0) {
+		storage->corrupted_rebuild_count =
+			mdbox_map_get_rebuild_count(storage->map);
+	}
+}
+
+static const char *
+mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED)
+{
+	return "";
+}
+
+void mdbox_set_mailbox_corrupted(struct mailbox *box)
+{
+	struct mdbox_storage *mstorage = MDBOX_STORAGE(box->storage);
+
+	mdbox_storage_set_corrupted(mstorage);
+}
+
+void mdbox_set_file_corrupted(struct dbox_file *file)
+{
+	struct mdbox_storage *mstorage = MDBOX_DBOX_STORAGE(file->storage);
+
+	mdbox_storage_set_corrupted(mstorage);
+}
+
+static int
+mdbox_mailbox_get_guid(struct mdbox_mailbox *mbox, guid_128_t guid_r)
+{
+	const struct mail_index_header *idx_hdr;
+	struct mdbox_index_header hdr;
+	bool need_resize;
+	int ret = 0;
+
+	i_assert(!mbox->creating);
+
+	/* there's a race condition between mkdir and getting the mailbox GUID.
+	   normally this is handled by mdbox syncing, but GUID can be looked up
+	   without syncing. when we detect this situation we'll try to finish
+	   creating the indexes first, which usually means just waiting for
+	   the sync lock to get unlocked by the other process creating them. */
+	idx_hdr = mail_index_get_header(mbox->box.view);
+	if (idx_hdr->uid_validity == 0 && idx_hdr->next_uid == 1) {
+		if (dbox_mailbox_create_indexes(&mbox->box, NULL) < 0)
+			return -1;
+	}
+
+	if (mdbox_read_header(mbox, &hdr, &need_resize) < 0)
+		i_zero(&hdr);
+
+	if (guid_128_is_empty(hdr.mailbox_guid)) {
+		/* regenerate it */
+		if (mdbox_write_index_header(&mbox->box, NULL, NULL) < 0 ||
+		    mdbox_read_header(mbox, &hdr, &need_resize) < 0)
+			ret = -1;
+	}
+	if (ret == 0)
+		memcpy(guid_r, hdr.mailbox_guid, GUID_128_SIZE);
+	return ret;
+}
+
+static int
+mdbox_mailbox_get_metadata(struct mailbox *box,
+			   enum mailbox_metadata_items items,
+			   struct mailbox_metadata *metadata_r)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box);
+
+	if (index_mailbox_get_metadata(box, items, metadata_r) < 0)
+		return -1;
+	if ((items & MAILBOX_METADATA_GUID) != 0) {
+		if (mdbox_mailbox_get_guid(mbox, metadata_r->guid) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int
+mdbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update)
+{
+	if (!box->opened) {
+		if (mailbox_open(box) < 0)
+			return -1;
+	}
+	if (mdbox_write_index_header(box, update, NULL) < 0)
+		return -1;
+	return index_storage_mailbox_update_common(box, update);
+}
+
+struct mail_storage mdbox_storage = {
+	.name = MDBOX_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_BINARY_DATA,
+	.event_category = &event_category_mdbox,
+
+	.v = {
+                mdbox_get_setting_parser_info,
+		mdbox_storage_alloc,
+		mdbox_storage_create,
+		mdbox_storage_destroy,
+		NULL,
+		dbox_storage_get_list_settings,
+		mdbox_storage_autodetect,
+		mdbox_mailbox_alloc,
+		mdbox_purge,
+		mail_storage_list_index_rebuild,
+	}
+};
+
+struct mailbox mdbox_mailbox = {
+	.v = {
+		index_storage_is_readonly,
+		index_storage_mailbox_enable,
+		index_storage_mailbox_exists,
+		mdbox_mailbox_open,
+		mdbox_mailbox_close,
+		index_storage_mailbox_free,
+		dbox_mailbox_create,
+		mdbox_mailbox_update,
+		index_storage_mailbox_delete,
+		index_storage_mailbox_rename,
+		index_storage_get_status,
+		mdbox_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		index_storage_list_index_has_changed,
+		index_storage_list_index_update_sync,
+		mdbox_storage_sync_init,
+		index_mailbox_sync_next,
+		index_mailbox_sync_deinit,
+		NULL,
+		dbox_notify_changes,
+		index_transaction_begin,
+		index_transaction_commit,
+		index_transaction_rollback,
+		NULL,
+		dbox_mail_alloc,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		mdbox_save_alloc,
+		mdbox_save_begin,
+		dbox_save_continue,
+		mdbox_save_finish,
+		mdbox_save_cancel,
+		mdbox_copy,
+		mdbox_transaction_save_commit_pre,
+		mdbox_transaction_save_commit_post,
+		mdbox_transaction_save_rollback,
+		index_storage_is_inconsistent
+	}
+};
+
+struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs = {
+	mdbox_file_unrefed,
+	mdbox_file_create_fd,
+	mdbox_mail_open,
+	mdbox_mailbox_create_indexes,
+	mdbox_get_attachment_path_suffix,
+	mdbox_set_mailbox_corrupted,
+	mdbox_set_file_corrupted
+};
diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.h b/src/lib-storage/index/dbox-multi/mdbox-storage.h
new file mode 100644
index 0000000..ea99532
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h
@@ -0,0 +1,118 @@
+#ifndef MDBOX_STORAGE_H
+#define MDBOX_STORAGE_H
+
+#include "index-storage.h"
+#include "dbox-storage.h"
+#include "mdbox-settings.h"
+
+#define MDBOX_STORAGE_NAME "mdbox"
+#define MDBOX_DELETED_STORAGE_NAME "mdbox_deleted"
+#define MDBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index"
+#define MDBOX_GLOBAL_DIR_NAME "storage"
+#define MDBOX_MAIL_FILE_PREFIX "m."
+#define MDBOX_MAIL_FILE_FORMAT MDBOX_MAIL_FILE_PREFIX"%u"
+#define MDBOX_MAX_OPEN_UNUSED_FILES 2
+#define MDBOX_CLOSE_UNUSED_FILES_TIMEOUT_SECS 30
+
+#define MDBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t))
+struct mdbox_index_header {
+	uint32_t map_uid_validity;
+	guid_128_t mailbox_guid;
+	uint8_t flags; /* enum dbox_index_header_flags */
+	uint8_t unused[3];
+};
+
+struct mdbox_storage {
+	struct dbox_storage storage;
+	const struct mdbox_settings *set;
+
+	/* paths for storage directories */
+	const char *storage_dir, *alt_storage_dir;
+	struct mdbox_map *map;
+
+	ARRAY(struct mdbox_file *) open_files;
+	struct timeout *to_close_unused_files;
+
+	ARRAY_TYPE(uint32_t) move_to_alt_map_uids;
+	ARRAY_TYPE(uint32_t) move_from_alt_map_uids;
+
+	/* if non-zero, storage should be rebuilt (except if rebuild_count
+	   has changed from this value) */
+	uint32_t corrupted_rebuild_count;
+
+	bool corrupted:1;
+	bool rebuilding_storage:1;
+	bool preallocate_space:1;
+};
+
+struct mdbox_mail_index_record {
+	uint32_t map_uid;
+	/* UNIX timestamp of when the message was saved/copied to this
+	   mailbox */
+	uint32_t save_date;
+};
+
+struct mdbox_mailbox {
+	struct mailbox box;
+	struct mdbox_storage *storage;
+
+	uint32_t map_uid_validity;
+	uint32_t ext_id, hdr_ext_id, guid_ext_id;
+
+	bool mdbox_deleted_synced:1;
+	bool creating:1;
+};
+
+#define MDBOX_DBOX_STORAGE(s)	container_of(s, struct mdbox_storage, storage)
+#define MDBOX_STORAGE(s)	MDBOX_DBOX_STORAGE(DBOX_STORAGE(s))
+#define MDBOX_MAILBOX(s)	container_of(s, struct mdbox_mailbox, box)
+
+extern struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs;
+extern struct mail_vfuncs mdbox_mail_vfuncs;
+
+int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r,
+		    struct dbox_file **file_r);
+
+/* Get map_uid for wanted message. */
+int mdbox_mail_lookup(struct mdbox_mailbox *mbox, struct mail_index_view *view,
+		      uint32_t seq, uint32_t *map_uid_r);
+uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list);
+int mdbox_read_header(struct mdbox_mailbox *mbox,
+		      struct mdbox_index_header *hdr, bool *need_resize_r);
+void mdbox_update_header(struct mdbox_mailbox *mbox,
+			 struct mail_index_transaction *trans,
+			 const struct mailbox_update *update) ATTR_NULL(3);
+int mdbox_mailbox_create_indexes(struct mailbox *box,
+				 const struct mailbox_update *update,
+				 struct mail_index_transaction *trans);
+
+struct mail_save_context *
+mdbox_save_alloc(struct mailbox_transaction_context *_t);
+int mdbox_save_begin(struct mail_save_context *ctx, struct istream *input);
+int mdbox_save_finish(struct mail_save_context *ctx);
+void mdbox_save_cancel(struct mail_save_context *ctx);
+
+struct dbox_file *
+mdbox_save_file_get_file(struct mailbox_transaction_context *t,
+			 uint32_t seq, uoff_t *offset_r);
+
+int mdbox_transaction_save_commit_pre(struct mail_save_context *ctx);
+void mdbox_transaction_save_commit_post(struct mail_save_context *ctx,
+					struct mail_index_transaction_commit_result *result);
+void mdbox_transaction_save_rollback(struct mail_save_context *ctx);
+
+int mdbox_copy(struct mail_save_context *ctx, struct mail *mail);
+
+void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt);
+int mdbox_purge(struct mail_storage *storage);
+
+int mdbox_storage_create(struct mail_storage *_storage,
+			 struct mail_namespace *ns, const char **error_r);
+void mdbox_storage_destroy(struct mail_storage *_storage);
+int mdbox_mailbox_open(struct mailbox *box);
+
+void mdbox_storage_set_corrupted(struct mdbox_storage *storage);
+void mdbox_set_mailbox_corrupted(struct mailbox *box);
+void mdbox_set_file_corrupted(struct dbox_file *file);
+
+#endif
diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.c b/src/lib-storage/index/dbox-multi/mdbox-sync.c
new file mode 100644
index 0000000..0c2d2d4
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c
@@ -0,0 +1,377 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+/*
+   Expunging works like:
+
+   1. Lock map index by beginning a map sync.
+   2. Write map UID refcount changes to map index (=> tail != head).
+   3. Expunge messages from mailbox index.
+   4. Finish map sync, which updates tail=head and unlocks map index.
+
+   If something crashes after 2 but before 4 is finished, tail != head and
+   reader can do a full resync to figure out what got broken.
+*/
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "str.h"
+#include "mdbox-storage.h"
+#include "mdbox-storage-rebuild.h"
+#include "mdbox-map.h"
+#include "mdbox-file.h"
+#include "mdbox-sync.h"
+#include "mailbox-recent-flags.h"
+
+
+/* returns -1 on error, 1 on success, 0 if guid is empty/missing */
+static int
+dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq,
+			      const guid_128_t guid_128)
+{
+	const void *data;
+	uint32_t uid;
+
+	mail_index_lookup_uid(ctx->sync_view, seq, &uid);
+	mail_index_lookup_ext(ctx->sync_view, seq,
+			      ctx->mbox->guid_ext_id, &data, NULL);
+
+	if ((data == NULL) || guid_128_is_empty(data))
+		return 0;
+
+	if (guid_128_is_empty(guid_128) ||
+	    memcmp(data, guid_128, GUID_128_SIZE) == 0)
+		return 1;
+
+	mailbox_set_critical(&ctx->mbox->box,
+		"Expunged GUID mismatch for UID %u: %s vs %s",
+		uid, guid_128_to_string(data), guid_128_to_string(guid_128));
+	mdbox_storage_set_corrupted(ctx->mbox->storage);
+	return -1;
+}
+
+static int mdbox_sync_expunge(struct mdbox_sync_context *ctx, uint32_t seq,
+			      const guid_128_t guid_128)
+{
+	uint32_t map_uid;
+	int ret;
+
+	if (seq_range_array_add(&ctx->expunged_seqs, seq)) {
+		/* already marked as expunged in this sync */
+		return 0;
+	}
+
+	ret = dbox_sync_verify_expunge_guid(ctx, seq, guid_128);
+	if (ret <= 0)
+		return ret;
+	if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0)
+		return -1;
+	if (mdbox_map_update_refcount(ctx->map_trans, map_uid, -1) < 0)
+		return -1;
+	return 0;
+}
+
+static int mdbox_sync_rec(struct mdbox_sync_context *ctx,
+			  const struct mail_index_sync_rec *sync_rec)
+{
+	uint32_t seq, seq1, seq2;
+
+	if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
+		/* not interested */
+		return 0;
+	}
+
+	if (!mail_index_lookup_seq_range(ctx->sync_view,
+					 sync_rec->uid1, sync_rec->uid2,
+					 &seq1, &seq2)) {
+		/* already expunged everything. nothing to do. */
+		return 0;
+	}
+
+	for (seq = seq1; seq <= seq2; seq++) {
+		if (mdbox_sync_expunge(ctx, seq, sync_rec->guid_128) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int dbox_sync_mark_expunges(struct mdbox_sync_context *ctx)
+{
+	enum mail_index_transaction_flags flags =
+		MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
+	struct mailbox *box = &ctx->mbox->box;
+	struct mail_index_transaction *trans;
+	struct seq_range_iter iter;
+	unsigned int n;
+	const void *data;
+	uint32_t seq, uid;
+
+	/* use a separate transaction here so that we can commit the changes
+	   during map transaction */
+	trans = mail_index_transaction_begin(ctx->sync_view, flags);
+	seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0;
+	while (seq_range_array_iter_nth(&iter, n++, &seq)) {
+		mail_index_lookup_uid(ctx->sync_view, seq, &uid);
+		mail_index_lookup_ext(ctx->sync_view, seq,
+				      ctx->mbox->guid_ext_id, &data, NULL);
+		if ((data == NULL) || guid_128_is_empty(data))
+			mail_index_expunge(trans, seq);
+		else
+			mail_index_expunge_guid(trans, seq, data);
+	}
+	if (mail_index_transaction_commit(&trans) < 0)
+		return -1;
+
+	box->tmp_sync_view = ctx->sync_view;
+	seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0;
+	while (seq_range_array_iter_nth(&iter, n++, &seq)) {
+		mail_index_lookup_uid(ctx->sync_view, seq, &uid);
+		mailbox_sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
+	}
+	box->tmp_sync_view = NULL;
+	return 0;
+}
+
+static int mdbox_sync_index(struct mdbox_sync_context *ctx)
+{
+	struct mailbox *box = &ctx->mbox->box;
+	const struct mail_index_header *hdr;
+	struct mail_index_sync_rec sync_rec;
+	uint32_t seq1, seq2;
+	int ret = 0;
+
+	hdr = mail_index_get_header(ctx->sync_view);
+	if (hdr->uid_validity == 0) {
+		/* newly created index file */
+		if (hdr->next_uid == 1) {
+			/* could be just a race condition where we opened the
+			   mailbox between mkdir and index creation. fix this
+			   silently. */
+			if (mdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0)
+				return -1;
+			return 1;
+		}
+		mailbox_set_critical(box, "Broken index: missing UIDVALIDITY");
+		return 0;
+	}
+
+	/* mark the newly seen messages as recent */
+	if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
+					hdr->next_uid, &seq1, &seq2)) {
+		mailbox_recent_flags_set_seqs(&ctx->mbox->box, ctx->sync_view,
+					      seq1, seq2);
+	}
+
+	/* handle syncing records without map being locked. */
+	if (mdbox_map_atomic_is_locked(ctx->atomic)) {
+		ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE);
+		i_array_init(&ctx->expunged_seqs, 64);
+	}
+	while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
+		if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0)
+			break;
+	}
+
+	/* write refcount changes to map index. transaction commit updates the
+	   log head, while tail is left behind. */
+	if (mdbox_map_atomic_is_locked(ctx->atomic)) {
+		if (ret == 0)
+			ret = mdbox_map_transaction_commit(ctx->map_trans, "mdbox syncing");
+		/* write changes to mailbox index */
+		if (ret == 0)
+			ret = dbox_sync_mark_expunges(ctx);
+
+		/* finish the map changes and unlock the map. this also updates
+		   map's tail -> head. */
+		if (ret < 0)
+			mdbox_map_atomic_set_failed(ctx->atomic);
+		mdbox_map_transaction_free(&ctx->map_trans);
+		ctx->expunged_count = seq_range_count(&ctx->expunged_seqs);
+		array_free(&ctx->expunged_seqs);
+	}
+
+	mailbox_sync_notify(box, 0, 0);
+
+	return ret == 0 ? 1 :
+		(ctx->mbox->storage->corrupted ? 0 : -1);
+}
+
+static int mdbox_sync_try_begin(struct mdbox_sync_context *ctx,
+				enum mail_index_sync_flags sync_flags)
+{
+	struct mdbox_mailbox *mbox = ctx->mbox;
+	int ret;
+
+	ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx,
+						&ctx->sync_view, &ctx->trans, sync_flags);
+	if (mail_index_reset_fscked(mbox->box.index))
+		mdbox_storage_set_corrupted(mbox->storage);
+	if (ret <= 0)
+		return ret; /* error / nothing to do */
+
+	if (!mdbox_map_atomic_is_locked(ctx->atomic) &&
+	    mail_index_sync_has_expunges(ctx->index_sync_ctx)) {
+		/* we have expunges, so we need to write to map.
+		   it needs to be locked before mailbox index. */
+		mail_index_sync_set_reason(ctx->index_sync_ctx, "mdbox expunge check");
+		mail_index_sync_rollback(&ctx->index_sync_ctx);
+		index_storage_expunging_deinit(&ctx->mbox->box);
+
+		if (mdbox_map_atomic_lock(ctx->atomic, "mdbox syncing with expunges") < 0)
+			return -1;
+		return mdbox_sync_try_begin(ctx, sync_flags);
+	}
+	return 1;
+}
+
+int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
+		     struct mdbox_map_atomic_context *atomic,
+		     struct mdbox_sync_context **ctx_r)
+{
+	const struct mail_index_header *hdr =
+		mail_index_get_header(mbox->box.view);
+	struct mdbox_sync_context *ctx;
+	const char *reason;
+	enum mail_index_sync_flags sync_flags;
+	int ret;
+	bool rebuild, storage_rebuilt = FALSE;
+
+	*ctx_r = NULL;
+
+	/* avoid race conditions with mailbox creation, don't check for dbox
+	   headers until syncing has locked the mailbox */
+	rebuild = mbox->storage->corrupted ||
+		(hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0 ||
+		mdbox_map_is_fscked(mbox->storage->map) ||
+		(flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
+	if (rebuild && (flags & MDBOX_SYNC_FLAG_NO_REBUILD) == 0) {
+		if (mdbox_storage_rebuild_in_context(mbox->storage, atomic) < 0)
+			return -1;
+		mailbox_recent_flags_reset(&mbox->box);
+		storage_rebuilt = TRUE;
+	}
+
+	ctx = i_new(struct mdbox_sync_context, 1);
+	ctx->mbox = mbox;
+	ctx->flags = flags;
+	ctx->atomic = atomic;
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box);
+	if (!rebuild && (flags & MDBOX_SYNC_FLAG_FORCE) == 0)
+		sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
+	if ((flags & MDBOX_SYNC_FLAG_FSYNC) != 0)
+		sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC;
+	/* don't write unnecessary dirty flag updates */
+	sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;
+
+	ret = mdbox_sync_try_begin(ctx, sync_flags);
+	if (ret <= 0) {
+		/* failed / nothing to do */
+		index_storage_expunging_deinit(&mbox->box);
+		i_free(ctx);
+		return ret;
+	}
+
+	if ((ret = mdbox_sync_index(ctx)) <= 0) {
+		mail_index_sync_set_reason(ctx->index_sync_ctx,
+			ret < 0 ? "mdbox syncing failed" :
+			"mdbox syncing found corruption");
+		mail_index_sync_rollback(&ctx->index_sync_ctx);
+		index_storage_expunging_deinit(&mbox->box);
+		i_free_and_null(ctx);
+
+		if (ret < 0)
+			return -1;
+
+		/* corrupted */
+		if (storage_rebuilt) {
+			mailbox_set_critical(&mbox->box,
+				"mdbox: Storage keeps breaking");
+			return -1;
+		}
+
+		/* we'll need to rebuild storage.
+		   try again from the beginning. */
+		mdbox_storage_set_corrupted(mbox->storage);
+		if ((flags & MDBOX_SYNC_FLAG_NO_REBUILD) != 0) {
+			mailbox_set_critical(&mbox->box,
+				"mdbox: Can't rebuild storage");
+			return -1;
+		}
+		return mdbox_sync_begin(mbox, flags, atomic, ctx_r);
+	}
+	index_storage_expunging_deinit(&mbox->box);
+
+	if (!mdbox_map_atomic_is_locked(ctx->atomic))
+		reason = "mdbox synced";
+	else {
+		/* may be 0 msgs, but that still informs that the map
+		   was locked */
+		reason = t_strdup_printf("mdbox synced - %u msgs expunged",
+					 ctx->expunged_count);
+	}
+	mail_index_sync_set_reason(ctx->index_sync_ctx, reason);
+
+	*ctx_r = ctx;
+	return 0;
+}
+
+int mdbox_sync_finish(struct mdbox_sync_context **_ctx, bool success)
+{
+	struct mdbox_sync_context *ctx = *_ctx;
+	struct mail_storage *storage = &ctx->mbox->storage->storage.storage;
+	int ret = success ? 0 : -1;
+
+	*_ctx = NULL;
+
+	if (success) {
+		if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
+			mailbox_set_index_error(&ctx->mbox->box);
+			ret = -1;
+		}
+	} else {
+		mail_index_sync_rollback(&ctx->index_sync_ctx);
+	}
+
+	if (storage->rebuild_list_index)
+		ret = mail_storage_list_index_rebuild_and_set_uncorrupted(storage);
+
+	i_free(ctx);
+	return ret;
+}
+
+int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags)
+{
+	struct mdbox_sync_context *sync_ctx;
+	struct mdbox_map_atomic_context *atomic;
+	int ret;
+
+	atomic = mdbox_map_atomic_begin(mbox->storage->map);
+	ret = mdbox_sync_begin(mbox, flags, atomic, &sync_ctx);
+	if (ret == 0 && sync_ctx != NULL)
+		ret = mdbox_sync_finish(&sync_ctx, TRUE);
+	if (ret == 0)
+		mdbox_map_atomic_set_success(atomic);
+	if (mdbox_map_atomic_finish(&atomic) < 0)
+		ret = -1;
+	return ret;
+}
+
+struct mailbox_sync_context *
+mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box);
+	enum mdbox_sync_flags mdbox_sync_flags = 0;
+	int ret = 0;
+
+	if (mail_index_reset_fscked(box->index))
+		mdbox_storage_set_corrupted(mbox->storage);
+	if (index_mailbox_want_full_sync(&mbox->box, flags) ||
+	    mbox->storage->corrupted) {
+		if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)
+			mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD;
+		ret = mdbox_sync(mbox, mdbox_sync_flags);
+	}
+
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.h b/src/lib-storage/index/dbox-multi/mdbox-sync.h
new file mode 100644
index 0000000..a9bc565
--- /dev/null
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.h
@@ -0,0 +1,37 @@
+#ifndef MDBOX_SYNC_H
+#define MDBOX_SYNC_H
+
+struct mailbox;
+struct mdbox_mailbox;
+
+enum mdbox_sync_flags {
+	MDBOX_SYNC_FLAG_FORCE		= 0x01,
+	MDBOX_SYNC_FLAG_FSYNC		= 0x02,
+	MDBOX_SYNC_FLAG_FORCE_REBUILD	= 0x04,
+	MDBOX_SYNC_FLAG_NO_PURGE	= 0x08,
+	MDBOX_SYNC_FLAG_NO_REBUILD	= 0x10
+};
+
+struct mdbox_sync_context {
+	struct mdbox_mailbox *mbox;
+        struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *trans;
+	struct mdbox_map_transaction_context *map_trans;
+	struct mdbox_map_atomic_context *atomic;
+	enum mdbox_sync_flags flags;
+
+	ARRAY_TYPE(seq_range) expunged_seqs;
+	unsigned int expunged_count;
+};
+
+int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
+		     struct mdbox_map_atomic_context *atomic,
+		     struct mdbox_sync_context **ctx_r);
+int mdbox_sync_finish(struct mdbox_sync_context **ctx, bool success);
+int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags);
+
+struct mailbox_sync_context *
+mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+
+#endif
diff --git a/src/lib-storage/index/dbox-single/Makefile.am b/src/lib-storage/index/dbox-single/Makefile.am
new file mode 100644
index 0000000..b6f59bd
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/Makefile.am
@@ -0,0 +1,30 @@
+noinst_LTLIBRARIES = libstorage_dbox_single.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-fs \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/lib-storage/index/dbox-common
+
+libstorage_dbox_single_la_SOURCES = \
+	sdbox-copy.c \
+	sdbox-file.c \
+	sdbox-mail.c \
+	sdbox-save.c \
+	sdbox-sync.c \
+	sdbox-sync-rebuild.c \
+	sdbox-storage.c
+
+headers = \
+	sdbox-file.h \
+	sdbox-storage.h \
+	sdbox-sync.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/dbox-single/Makefile.in b/src/lib-storage/index/dbox-single/Makefile.in
new file mode 100644
index 0000000..f3e5076
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/Makefile.in
@@ -0,0 +1,848 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/dbox-single
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_dbox_single_la_LIBADD =
+am_libstorage_dbox_single_la_OBJECTS = sdbox-copy.lo sdbox-file.lo \
+	sdbox-mail.lo sdbox-save.lo sdbox-sync.lo \
+	sdbox-sync-rebuild.lo sdbox-storage.lo
+libstorage_dbox_single_la_OBJECTS =  \
+	$(am_libstorage_dbox_single_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/sdbox-copy.Plo \
+	./$(DEPDIR)/sdbox-file.Plo ./$(DEPDIR)/sdbox-mail.Plo \
+	./$(DEPDIR)/sdbox-save.Plo ./$(DEPDIR)/sdbox-storage.Plo \
+	./$(DEPDIR)/sdbox-sync-rebuild.Plo ./$(DEPDIR)/sdbox-sync.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_dbox_single_la_SOURCES)
+DIST_SOURCES = $(libstorage_dbox_single_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_dbox_single.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-fs \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/lib-storage/index/dbox-common
+
+libstorage_dbox_single_la_SOURCES = \
+	sdbox-copy.c \
+	sdbox-file.c \
+	sdbox-mail.c \
+	sdbox-save.c \
+	sdbox-sync.c \
+	sdbox-sync-rebuild.c \
+	sdbox-storage.c
+
+headers = \
+	sdbox-file.h \
+	sdbox-storage.h \
+	sdbox-sync.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/dbox-single/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/dbox-single/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_dbox_single.la: $(libstorage_dbox_single_la_OBJECTS) $(libstorage_dbox_single_la_DEPENDENCIES) $(EXTRA_libstorage_dbox_single_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_dbox_single_la_OBJECTS) $(libstorage_dbox_single_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-copy.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-file.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-save.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-sync-rebuild.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sdbox-sync.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/sdbox-copy.Plo
+	-rm -f ./$(DEPDIR)/sdbox-file.Plo
+	-rm -f ./$(DEPDIR)/sdbox-mail.Plo
+	-rm -f ./$(DEPDIR)/sdbox-save.Plo
+	-rm -f ./$(DEPDIR)/sdbox-storage.Plo
+	-rm -f ./$(DEPDIR)/sdbox-sync-rebuild.Plo
+	-rm -f ./$(DEPDIR)/sdbox-sync.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/sdbox-copy.Plo
+	-rm -f ./$(DEPDIR)/sdbox-file.Plo
+	-rm -f ./$(DEPDIR)/sdbox-mail.Plo
+	-rm -f ./$(DEPDIR)/sdbox-save.Plo
+	-rm -f ./$(DEPDIR)/sdbox-storage.Plo
+	-rm -f ./$(DEPDIR)/sdbox-sync-rebuild.Plo
+	-rm -f ./$(DEPDIR)/sdbox-sync.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/dbox-single/sdbox-copy.c b/src/lib-storage/index/dbox-single/sdbox-copy.c
new file mode 100644
index 0000000..48a3f59
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-copy.c
@@ -0,0 +1,185 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "nfs-workarounds.h"
+#include "fs-api.h"
+#include "dbox-save.h"
+#include "dbox-attachment.h"
+#include "sdbox-storage.h"
+#include "sdbox-file.h"
+#include "mail-copy.h"
+
+static int
+sdbox_file_copy_attachments(struct sdbox_file *src_file,
+			    struct sdbox_file *dest_file)
+{
+	struct dbox_storage *src_storage = src_file->file.storage;
+	struct dbox_storage *dest_storage = dest_file->file.storage;
+	struct fs_file *src_fsfile, *dest_fsfile;
+	ARRAY_TYPE(mail_attachment_extref) extrefs;
+	const struct mail_attachment_extref *extref;
+	const char *extrefs_line, *src, *dest, *dest_relpath;
+	pool_t pool;
+	int ret;
+
+	if (src_storage->attachment_dir == NULL) {
+		/* no attachments in source storage */
+		return 1;
+	}
+	if (dest_storage->attachment_dir == NULL ||
+	    strcmp(src_storage->attachment_dir,
+		   dest_storage->attachment_dir) != 0 ||
+	    strcmp(src_storage->storage.set->mail_attachment_fs,
+		   dest_storage->storage.set->mail_attachment_fs) != 0 ||
+	    strcmp(src_storage->storage.set->mail_attachment_hash,
+		   dest_storage->storage.set->mail_attachment_hash) != 0) {
+		/* different attachment dirs/settings between storages.
+		   have to copy the slow way. */
+		return 0;
+	}
+
+	if ((ret = sdbox_file_get_attachments(&src_file->file,
+					      &extrefs_line)) <= 0)
+		return ret < 0 ? -1 : 1;
+
+	pool = pool_alloconly_create("sdbox attachments copy", 1024);
+	p_array_init(&extrefs, pool, 16);
+	if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) {
+		mailbox_set_critical(&dest_file->mbox->box,
+			"Can't copy %s with corrupted extref metadata: %s",
+			src_file->file.cur_path, extrefs_line);
+		pool_unref(&pool);
+		return -1;
+	}
+
+	dest_file->attachment_pool =
+		pool_alloconly_create("sdbox attachment copy paths", 512);
+	p_array_init(&dest_file->attachment_paths, dest_file->attachment_pool,
+		     array_count(&extrefs));
+
+	ret = 1;
+	array_foreach(&extrefs, extref) T_BEGIN {
+		src = t_strdup_printf("%s/%s", dest_storage->attachment_dir,
+			sdbox_file_attachment_relpath(src_file, extref->path));
+		dest_relpath = p_strconcat(dest_file->attachment_pool,
+					   extref->path, "-",
+					   guid_generate(), NULL);
+		dest = t_strdup_printf("%s/%s", dest_storage->attachment_dir,
+				       dest_relpath);
+		/* we verified above that attachment_fs is compatible for
+		   src and dest, so it doesn't matter which storage's
+		   attachment_fs we use. in any case we need to use the same
+		   one or fs_copy() will crash with assert. */
+		src_fsfile = fs_file_init(dest_storage->attachment_fs, src,
+					  FS_OPEN_MODE_READONLY);
+		dest_fsfile = fs_file_init(dest_storage->attachment_fs, dest,
+					   FS_OPEN_MODE_READONLY);
+		if (fs_copy(src_fsfile, dest_fsfile) < 0) {
+			mailbox_set_critical(&dest_file->mbox->box, "%s",
+				fs_file_last_error(dest_fsfile));
+			ret = -1;
+		} else {
+			array_push_back(&dest_file->attachment_paths,
+					&dest_relpath);
+		}
+		fs_file_deinit(&src_fsfile);
+		fs_file_deinit(&dest_fsfile);
+	} T_END;
+	pool_unref(&pool);
+	return ret;
+}
+
+static int
+sdbox_copy_hardlink(struct mail_save_context *_ctx, struct mail *mail)
+{
+	struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx);
+	struct sdbox_mailbox *dest_mbox = SDBOX_MAILBOX(_ctx->transaction->box);
+	struct sdbox_mailbox *src_mbox;
+	struct dbox_file *src_file, *dest_file;
+	const char *src_path, *dest_path;
+	int ret;
+
+	if (strcmp(mail->box->storage->name, SDBOX_STORAGE_NAME) == 0)
+		src_mbox = SDBOX_MAILBOX(mail->box);
+	else {
+		/* Source storage isn't sdbox, can't hard link */
+		return 0;
+	}
+
+	src_file = sdbox_file_init(src_mbox, mail->uid);
+	dest_file = sdbox_file_init(dest_mbox, 0);
+
+	ctx->ctx.data.flags &= ENUM_NEGATE(DBOX_INDEX_FLAG_ALT);
+
+	src_path = src_file->primary_path;
+	dest_path = dest_file->primary_path;
+	ret = nfs_safe_link(src_path, dest_path, FALSE);
+	if (ret < 0 && errno == ENOENT && src_file->alt_path != NULL) {
+		src_path = src_file->alt_path;
+		if (dest_file->alt_path != NULL) {
+			dest_path = dest_file->cur_path = dest_file->alt_path;
+			ctx->ctx.data.flags |= DBOX_INDEX_FLAG_ALT;
+		}
+		ret = nfs_safe_link(src_path, dest_path, FALSE);
+	}
+	if (ret < 0) {
+		if (ECANTLINK(errno))
+			ret = 0;
+		else if (errno == ENOENT) {
+			/* try if the fallback copying code can still
+			   read the file (the mail could still have the
+			   stream open) */
+			ret = 0;
+		} else {
+			mail_set_critical(mail, "link(%s, %s) failed: %m",
+					  src_path, dest_path);
+		}
+		dbox_file_unref(&src_file);
+		dbox_file_unref(&dest_file);
+		return ret;
+	}
+
+	ret = sdbox_file_copy_attachments((struct sdbox_file *)src_file,
+					  (struct sdbox_file *)dest_file);
+	if (ret <= 0) {
+		(void)sdbox_file_unlink_aborted_save((struct sdbox_file *)dest_file);
+		dbox_file_unref(&src_file);
+		dbox_file_unref(&dest_file);
+		return ret;
+	}
+	((struct sdbox_file *)dest_file)->written_to_disk = TRUE;
+
+	dbox_save_add_to_index(ctx);
+	index_copy_cache_fields(_ctx, mail, ctx->seq);
+
+	sdbox_save_add_file(_ctx, dest_file);
+	mail_set_seq_saving(_ctx->dest_mail, ctx->seq);
+	dbox_file_unref(&src_file);
+	return 1;
+}
+
+int sdbox_copy(struct mail_save_context *_ctx, struct mail *mail)
+{
+	struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;
+	struct mailbox_transaction_context *_t = _ctx->transaction;
+	struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)_t->box;
+	int ret;
+
+	i_assert((_t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	ctx->finished = TRUE;
+	if (mail_storage_copy_can_use_hardlink(mail->box, &mbox->box) &&
+	    _ctx->data.guid == NULL) {
+		T_BEGIN {
+			ret = sdbox_copy_hardlink(_ctx, mail);
+		} T_END;
+
+		if (ret != 0) {
+			index_save_context_free(_ctx);
+			return ret > 0 ? 0 : -1;
+		}
+
+		/* non-fatal hardlinking failure, try the slow way */
+	}
+	return mail_storage_copy(_ctx, mail);
+}
diff --git a/src/lib-storage/index/dbox-single/sdbox-file.c b/src/lib-storage/index/dbox-single/sdbox-file.c
new file mode 100644
index 0000000..fed111a
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-file.c
@@ -0,0 +1,447 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "eacces-error.h"
+#include "fdatasync-path.h"
+#include "mkdir-parents.h"
+#include "istream.h"
+#include "ostream.h"
+#include "str.h"
+#include "fs-api.h"
+#include "dbox-attachment.h"
+#include "sdbox-storage.h"
+#include "sdbox-file.h"
+
+#include <stdio.h>
+#include <utime.h>
+
+static void sdbox_file_init_paths(struct sdbox_file *file, const char *fname)
+{
+	struct mailbox *box = &file->mbox->box;
+	const char *alt_path;
+
+	i_free(file->file.primary_path);
+	i_free(file->file.alt_path);
+	file->file.primary_path =
+		i_strdup_printf("%s/%s", mailbox_get_path(box), fname);
+	file->file.cur_path = file->file.primary_path;
+
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX,
+				&alt_path) > 0)
+		file->file.alt_path = i_strdup_printf("%s/%s", alt_path, fname);
+}
+
+struct dbox_file *sdbox_file_init(struct sdbox_mailbox *mbox, uint32_t uid)
+{
+	struct sdbox_file *file;
+	const char *fname;
+
+	file = i_new(struct sdbox_file, 1);
+	file->file.storage = &mbox->storage->storage;
+	file->mbox = mbox;
+	T_BEGIN {
+		if (uid != 0) {
+			fname = t_strdup_printf(SDBOX_MAIL_FILE_FORMAT, uid);
+			sdbox_file_init_paths(file, fname);
+			file->uid = uid;
+		} else {
+			sdbox_file_init_paths(file, dbox_generate_tmp_filename());
+		}
+	} T_END;
+	dbox_file_init(&file->file);
+	return &file->file;
+}
+
+struct dbox_file *sdbox_file_create(struct sdbox_mailbox *mbox)
+{
+	struct dbox_file *file;
+
+	file = sdbox_file_init(mbox, 0);
+	file->fd = file->storage->v.
+		file_create_fd(file, file->primary_path, FALSE);
+	return file;
+}
+
+void sdbox_file_free(struct dbox_file *file)
+{
+	struct sdbox_file *sfile = (struct sdbox_file *)file;
+
+	pool_unref(&sfile->attachment_pool);
+	dbox_file_free(file);
+}
+
+int sdbox_file_get_attachments(struct dbox_file *file, const char **extrefs_r)
+{
+	const char *line;
+	bool deleted;
+	int ret;
+
+	*extrefs_r = NULL;
+
+	/* read the metadata */
+	ret = dbox_file_open(file, &deleted);
+	if (ret > 0) {
+		if (deleted)
+			return 0;
+		if ((ret = dbox_file_seek(file, 0)) > 0)
+			ret = dbox_file_metadata_read(file);
+	}
+	if (ret <= 0) {
+		if (ret < 0)
+			return -1;
+		/* corrupted file. we're deleting it anyway. */
+		line = NULL;
+	} else {
+		line = dbox_file_metadata_get(file, DBOX_METADATA_EXT_REF);
+	}
+	if (line == NULL) {
+		/* no attachments */
+		return 0;
+	}
+	*extrefs_r = line;
+	return 1;
+}
+
+const char *
+sdbox_file_attachment_relpath(struct sdbox_file *file, const char *srcpath)
+{
+	const char *p;
+
+	p = strchr(srcpath, '-');
+	if (p == NULL) {
+		mailbox_set_critical(&file->mbox->box,
+			"sdbox attachment path in invalid format: %s", srcpath);
+	} else {
+		p = strchr(p+1, '-');
+	}
+	return t_strdup_printf("%s-%s-%u",
+			p == NULL ? srcpath : t_strdup_until(srcpath, p),
+			guid_128_to_string(file->mbox->mailbox_guid),
+			file->uid);
+}
+
+static int sdbox_file_rename_attachments(struct sdbox_file *file)
+{
+	struct dbox_storage *storage = file->file.storage;
+	struct fs_file *src_file, *dest_file;
+	const char *path, *src, *dest;
+	int ret = 0;
+
+	array_foreach_elem(&file->attachment_paths, path) T_BEGIN {
+		src = t_strdup_printf("%s/%s", storage->attachment_dir, path);
+		dest = t_strdup_printf("%s/%s", storage->attachment_dir,
+				sdbox_file_attachment_relpath(file, path));
+		src_file = fs_file_init(storage->attachment_fs, src,
+					FS_OPEN_MODE_READONLY);
+		dest_file = fs_file_init(storage->attachment_fs, dest,
+					FS_OPEN_MODE_READONLY);
+		if (fs_rename(src_file, dest_file) < 0) {
+			mailbox_set_critical(&file->mbox->box, "%s",
+				fs_file_last_error(dest_file));
+			ret = -1;
+		}
+		fs_file_deinit(&src_file);
+		fs_file_deinit(&dest_file);
+	} T_END;
+	return ret;
+}
+
+int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid)
+{
+	const char *p, *old_path, *dir, *new_fname, *new_path;
+	struct stat st;
+
+	i_assert(file->uid == 0);
+	i_assert(uid != 0);
+
+	old_path = file->file.cur_path;
+	p = strrchr(old_path, '/');
+	i_assert(p != NULL);
+	dir = t_strdup_until(old_path, p);
+
+	new_fname = t_strdup_printf(SDBOX_MAIL_FILE_FORMAT, uid);
+	new_path = t_strdup_printf("%s/%s", dir, new_fname);
+
+	if (stat(new_path, &st) == 0) {
+		mailbox_set_critical(&file->mbox->box,
+			"sdbox: %s already exists, rebuilding index", new_path);
+		sdbox_set_mailbox_corrupted(&file->mbox->box);
+		return -1;
+	}
+	if (rename(old_path, new_path) < 0) {
+		mailbox_set_critical(&file->mbox->box,
+				     "rename(%s, %s) failed: %m",
+				     old_path, new_path);
+		return -1;
+	}
+	sdbox_file_init_paths(file, new_fname);
+	file->uid = uid;
+
+	if (array_is_created(&file->attachment_paths)) {
+		if (sdbox_file_rename_attachments(file) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int sdbox_file_unlink_aborted_save_attachments(struct sdbox_file *file)
+{
+	struct dbox_storage *storage = file->file.storage;
+	struct fs *fs = storage->attachment_fs;
+	struct fs_file *fs_file;
+	const char *path, *att_path;
+	int ret = 0;
+
+	array_foreach_elem(&file->attachment_paths, att_path) T_BEGIN {
+		/* we don't know if we aborted before renaming this attachment,
+		   so try deleting both source and dest path. the source paths
+		   point to temporary files (not to source messages'
+		   attachment paths), so it's safe to delete them. */
+		path = t_strdup_printf("%s/%s", storage->attachment_dir,
+				       att_path);
+		fs_file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+		if (fs_delete(fs_file) < 0 &&
+		    errno != ENOENT) {
+			mailbox_set_critical(&file->mbox->box, "%s",
+					     fs_file_last_error(fs_file));
+			ret = -1;
+		}
+		fs_file_deinit(&fs_file);
+
+		path = t_strdup_printf("%s/%s", storage->attachment_dir,
+				sdbox_file_attachment_relpath(file, att_path));
+		fs_file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+		if (fs_delete(fs_file) < 0 &&
+		    errno != ENOENT) {
+			mailbox_set_critical(&file->mbox->box, "%s",
+					     fs_file_last_error(fs_file));
+			ret = -1;
+		}
+		fs_file_deinit(&fs_file);
+	} T_END;
+	return ret;
+}
+
+int sdbox_file_unlink_aborted_save(struct sdbox_file *file)
+{
+	int ret = 0;
+
+	if (unlink(file->file.cur_path) < 0) {
+		mailbox_set_critical(&file->mbox->box,
+			"unlink(%s) failed: %m", file->file.cur_path);
+		ret = -1;
+	}
+	if (array_is_created(&file->attachment_paths)) {
+		if (sdbox_file_unlink_aborted_save_attachments(file) < 0)
+			ret = -1;
+	}
+	return ret;
+}
+
+int sdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents)
+{
+	struct sdbox_file *sfile = (struct sdbox_file *)file;
+	struct mailbox *box = &sfile->mbox->box;
+	const struct mailbox_permissions *perm = mailbox_get_permissions(box);
+	const char *p, *dir;
+	mode_t old_mask;
+	int fd;
+
+	old_mask = umask(0666 & ~perm->file_create_mode);
+	fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666);
+	umask(old_mask);
+	if (fd == -1 && errno == ENOENT && parents &&
+	    (p = strrchr(path, '/')) != NULL) {
+		dir = t_strdup_until(path, p);
+		if (mkdir_parents_chgrp(dir, perm->dir_create_mode,
+					perm->file_create_gid,
+					perm->file_create_gid_origin) < 0 &&
+		   errno != EEXIST) {
+			mailbox_set_critical(box,
+				"mkdir_parents(%s) failed: %m", dir);
+			return -1;
+		}
+		/* try again */
+		old_mask = umask(0666 & ~perm->file_create_mode);
+		fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666);
+		umask(old_mask);
+	}
+	if (fd == -1) {
+		mailbox_set_critical(box, "open(%s, O_CREAT) failed: %m", path);
+	} else if (perm->file_create_gid == (gid_t)-1) {
+		/* no group change */
+	} else if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) {
+		if (errno == EPERM) {
+			mailbox_set_critical(box, "%s",
+				eperm_error_get_chgrp("fchown", path,
+					perm->file_create_gid,
+					perm->file_create_gid_origin));
+		} else {
+			mailbox_set_critical(box,
+				"fchown(%s, -1, %ld) failed: %m",
+				path, (long)perm->file_create_gid);
+		}
+		/* continue anyway */
+	}
+	return fd;
+}
+
+int sdbox_file_move(struct dbox_file *file, bool alt_path)
+{
+	struct mail_storage *storage = &file->storage->storage;
+	struct ostream *output;
+	const char *dest_dir, *temp_path, *dest_path, *p;
+	struct stat st;
+	struct utimbuf ut;
+	bool deleted;
+	int out_fd, ret = 0;
+
+	i_assert(file->input != NULL);
+
+	if (dbox_file_is_in_alt(file) == alt_path)
+		return 0;
+	if (file->alt_path == NULL)
+		return 0;
+
+	if (stat(file->cur_path, &st) < 0 && errno == ENOENT) {
+		/* already expunged/moved by another session */
+		return 0;
+	}
+
+	dest_path = !alt_path ? file->primary_path : file->alt_path;
+
+	i_assert(dest_path != NULL);
+
+	p = strrchr(dest_path, '/');
+	i_assert(p != NULL);
+	dest_dir = t_strdup_until(dest_path, p);
+	temp_path = t_strdup_printf("%s/%s", dest_dir,
+				    dbox_generate_tmp_filename());
+
+	/* first copy the file. make sure to catch every possible error
+	   since we really don't want to break the file. */
+	out_fd = file->storage->v.file_create_fd(file, temp_path, TRUE);
+	if (out_fd == -1)
+		return -1;
+
+	output = o_stream_create_fd_file(out_fd, 0, FALSE);
+	i_stream_seek(file->input, 0);
+	o_stream_nsend_istream(output, file->input);
+	if (o_stream_finish(output) < 0) {
+		mail_storage_set_critical(storage, "write(%s) failed: %s",
+			temp_path, o_stream_get_error(output));
+		ret = -1;
+	}
+	o_stream_unref(&output);
+
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER && ret == 0) {
+		if (fsync(out_fd) < 0) {
+			mail_storage_set_critical(storage,
+				"fsync(%s) failed: %m", temp_path);
+			ret = -1;
+		}
+	}
+	if (close(out_fd) < 0) {
+		mail_storage_set_critical(storage,
+			"close(%s) failed: %m", temp_path);
+		ret = -1;
+	}
+	if (ret < 0) {
+		i_unlink(temp_path);
+		return -1;
+	}
+	/* preserve the original atime/mtime. this isn't necessary for Dovecot,
+	   but could be useful for external reasons. */
+	ut.actime = st.st_atime;
+	ut.modtime = st.st_mtime;
+	if (utime(temp_path, &ut) < 0) {
+		mail_storage_set_critical(storage,
+			"utime(%s) failed: %m", temp_path);
+	}
+
+	/* the temp file was successfully written. rename it now to the
+	   destination file. the destination shouldn't exist, but if it does
+	   its contents should be the same (except for maybe older metadata) */
+	if (rename(temp_path, dest_path) < 0) {
+		mail_storage_set_critical(storage,
+			"rename(%s, %s) failed: %m", temp_path, dest_path);
+		i_unlink_if_exists(temp_path);
+		return -1;
+	}
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
+		if (fdatasync_path(dest_dir) < 0) {
+			mail_storage_set_critical(storage,
+				"fdatasync(%s) failed: %m", dest_dir);
+			i_unlink(dest_path);
+			return -1;
+		}
+	}
+	if (unlink(file->cur_path) < 0) {
+		dbox_file_set_syscall_error(file, "unlink()");
+		if (errno == EACCES) {
+			/* configuration problem? revert the write */
+			i_unlink(dest_path);
+		}
+		/* who knows what happened to the file. keep both just to be
+		   sure both won't get deleted. */
+		return -1;
+	}
+
+	/* file was successfully moved - reopen it */
+	dbox_file_close(file);
+	if (dbox_file_open(file, &deleted) <= 0) {
+		mail_storage_set_critical(storage,
+			"dbox_file_move(%s): reopening file failed", dest_path);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+sdbox_unlink_attachments(struct sdbox_file *sfile,
+			 const ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+	struct dbox_storage *storage = sfile->file.storage;
+	const struct mail_attachment_extref *extref;
+	const char *path;
+	int ret = 0;
+
+	array_foreach(extrefs, extref) T_BEGIN {
+		path = sdbox_file_attachment_relpath(sfile, extref->path);
+		if (index_attachment_delete(&storage->storage,
+					    storage->attachment_fs, path) < 0)
+			ret = -1;
+	} T_END;
+	return ret;
+}
+
+int sdbox_file_unlink_with_attachments(struct sdbox_file *sfile)
+{
+	ARRAY_TYPE(mail_attachment_extref) extrefs;
+	const char *extrefs_line;
+	pool_t pool;
+	int ret;
+
+	ret = sdbox_file_get_attachments(&sfile->file, &extrefs_line);
+	if (ret < 0)
+		return -1;
+	if (ret == 0) {
+		/* no attachments */
+		return dbox_file_unlink(&sfile->file);
+	}
+
+	pool = pool_alloconly_create("sdbox attachments unlink", 1024);
+	p_array_init(&extrefs, pool, 16);
+	if (!index_attachment_parse_extrefs(extrefs_line, pool, &extrefs)) {
+		i_warning("%s: Ignoring corrupted extref: %s",
+			  sfile->file.cur_path, extrefs_line);
+		array_clear(&extrefs);
+	}
+
+	/* try to delete the file first, so if it fails we don't have
+	   missing attachments */
+	if ((ret = dbox_file_unlink(&sfile->file)) >= 0)
+		(void)sdbox_unlink_attachments(sfile, &extrefs);
+	pool_unref(&pool);
+	return ret;
+}
diff --git a/src/lib-storage/index/dbox-single/sdbox-file.h b/src/lib-storage/index/dbox-single/sdbox-file.h
new file mode 100644
index 0000000..ba5a7f9
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-file.h
@@ -0,0 +1,43 @@
+#ifndef SDBOX_FILE_H
+#define SDBOX_FILE_H
+
+#include "dbox-file.h"
+
+struct sdbox_file {
+	struct dbox_file file;
+	struct sdbox_mailbox *mbox;
+
+	/* 0 while file is being created */
+	uint32_t uid;
+
+	/* list of attachment paths while saving/copying message */
+	pool_t attachment_pool;
+	ARRAY_TYPE(const_string) attachment_paths;
+	bool written_to_disk;
+};
+
+struct dbox_file *sdbox_file_init(struct sdbox_mailbox *mbox, uint32_t uid);
+struct dbox_file *sdbox_file_create(struct sdbox_mailbox *mbox);
+void sdbox_file_free(struct dbox_file *file);
+
+/* Get file's extrefs metadata. */
+int sdbox_file_get_attachments(struct dbox_file *file, const char **extrefs_r);
+/* Returns attachment path for this file, given the source path. The result is
+   always <hash>-<guid>-<mailbox_guid>-<uid>. The source path is expected to
+   contain <hash>-<guid>[-*]. */
+const char *
+sdbox_file_attachment_relpath(struct sdbox_file *file, const char *srcpath);
+
+/* Assign UID for a newly created file (by renaming it) */
+int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid);
+
+int sdbox_file_create_fd(struct dbox_file *file, const char *path,
+			 bool parents);
+/* Move the file to alt path or back. */
+int sdbox_file_move(struct dbox_file *file, bool alt_path);
+/* Unlink file and all of its referenced attachments. */
+int sdbox_file_unlink_with_attachments(struct sdbox_file *sfile);
+/* Unlink file and its attachments when rolling back a saved message. */
+int sdbox_file_unlink_aborted_save(struct sdbox_file *file);
+
+#endif
diff --git a/src/lib-storage/index/dbox-single/sdbox-mail.c b/src/lib-storage/index/dbox-single/sdbox-mail.c
new file mode 100644
index 0000000..3b0352c
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-mail.c
@@ -0,0 +1,182 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "str.h"
+#include "index-mail.h"
+#include "dbox-mail.h"
+#include "sdbox-storage.h"
+#include "sdbox-file.h"
+
+#include <sys/stat.h>
+
+static void sdbox_mail_set_expunged(struct dbox_mail *mail)
+{
+	struct mail *_mail = &mail->imail.mail.mail;
+
+	mail_index_refresh(_mail->box->index);
+	if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
+		mail_set_expunged(_mail);
+		return;
+	}
+
+	mail_set_critical(_mail, "dbox: Unexpectedly lost uid");
+	sdbox_set_mailbox_corrupted(_mail->box);
+}
+
+static int sdbox_mail_file_set(struct dbox_mail *mail)
+{
+	struct mail *_mail = &mail->imail.mail.mail;
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(_mail->box);
+	bool deleted;
+	int ret;
+
+	if (mail->open_file != NULL) {
+		/* already set */
+		return 0;
+	} else if (!_mail->saving) {
+		mail->open_file = sdbox_file_init(mbox, _mail->uid);
+		return 0;
+	} else {
+		/* mail is being saved in this transaction */
+		mail->open_file =
+			sdbox_save_file_get_file(_mail->transaction,
+						 _mail->seq);
+		mail->open_file->refcount++;
+
+		/* it doesn't have input stream yet */
+		ret = dbox_file_open(mail->open_file, &deleted);
+		if (ret <= 0) {
+			mail_set_critical(_mail,
+				"dbox: Unexpectedly lost mail being saved");
+			sdbox_set_mailbox_corrupted(_mail->box);
+			return -1;
+		}
+		return 1;
+	}
+}
+
+static int
+sdbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+		       const char **value_r)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(_mail->box);
+	struct dbox_mail *mail = DBOX_MAIL(_mail);
+	struct stat st;
+
+	switch (field) {
+	case MAIL_FETCH_REFCOUNT:
+		if (sdbox_mail_file_set(mail) < 0)
+			return -1;
+
+		_mail->transaction->stats.fstat_lookup_count++;
+		if (dbox_file_stat(mail->open_file, &st) < 0) {
+			if (errno == ENOENT)
+				mail_set_expunged(_mail);
+			return -1;
+		}
+		*value_r = p_strdup_printf(mail->imail.mail.data_pool, "%lu",
+					   (unsigned long)st.st_nlink);
+		return 0;
+	case MAIL_FETCH_REFCOUNT_ID:
+		if (sdbox_mail_file_set(mail) < 0)
+			return -1;
+
+		_mail->transaction->stats.fstat_lookup_count++;
+		if (dbox_file_stat(mail->open_file, &st) < 0) {
+			if (errno == ENOENT)
+				mail_set_expunged(_mail);
+			return -1;
+		}
+		*value_r = p_strdup_printf(mail->imail.mail.data_pool, "%llu",
+					   (unsigned long long)st.st_ino);
+		return 0;
+	case MAIL_FETCH_UIDL_BACKEND:
+		if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id,
+				offsetof(struct sdbox_index_header, flags),
+				DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) {
+			*value_r = "";
+			return 0;
+		}
+		break;
+	case MAIL_FETCH_POP3_ORDER:
+		if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id,
+				offsetof(struct sdbox_index_header, flags),
+				DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) {
+			*value_r = "";
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+	return dbox_mail_get_special(_mail, field, value_r);
+}
+
+int sdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r,
+		    struct dbox_file **file_r)
+{
+	struct mail *_mail = &mail->imail.mail.mail;
+	bool deleted;
+	int ret;
+
+	if (!mail_stream_access_start(_mail))
+		return -1;
+
+	ret = sdbox_mail_file_set(mail);
+	if (ret < 0)
+		return -1;
+	if (ret == 0) {
+		if (!dbox_file_is_open(mail->open_file))
+			_mail->transaction->stats.open_lookup_count++;
+		if (dbox_file_open(mail->open_file, &deleted) <= 0)
+			return -1;
+		if (deleted) {
+			sdbox_mail_set_expunged(mail);
+			return -1;
+		}
+	}
+
+	*file_r = mail->open_file;
+	*offset_r = 0;
+	return 0;
+}
+
+struct mail_vfuncs sdbox_mail_vfuncs = {
+	dbox_mail_close,
+	index_mail_free,
+	index_mail_set_seq,
+	index_mail_set_uid,
+	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
+	index_mail_precache,
+	index_mail_add_temp_wanted_fields,
+
+	index_mail_get_flags,
+	index_mail_get_keywords,
+	index_mail_get_keyword_indexes,
+	index_mail_get_modseq,
+	index_mail_get_pvt_modseq,
+	index_mail_get_parts,
+	index_mail_get_date,
+	dbox_mail_get_received_date,
+	dbox_mail_get_save_date,
+	dbox_mail_get_virtual_size,
+	dbox_mail_get_physical_size,
+	index_mail_get_first_header,
+	index_mail_get_headers,
+	index_mail_get_header_stream,
+	dbox_mail_get_stream,
+	index_mail_get_binary_stream,
+	sdbox_mail_get_special,
+	index_mail_get_backend_mail,
+	index_mail_update_flags,
+	index_mail_update_keywords,
+	index_mail_update_modseq,
+	index_mail_update_pvt_modseq,
+	NULL,
+	index_mail_expunge,
+	index_mail_set_cache_corrupted,
+	index_mail_opened,
+};
diff --git a/src/lib-storage/index/dbox-single/sdbox-save.c b/src/lib-storage/index/dbox-single/sdbox-save.c
new file mode 100644
index 0000000..03e24ab
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-save.c
@@ -0,0 +1,359 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "fdatasync-path.h"
+#include "hex-binary.h"
+#include "hex-dec.h"
+#include "str.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "ostream.h"
+#include "write-full.h"
+#include "index-mail.h"
+#include "mail-copy.h"
+#include "index-pop3-uidl.h"
+#include "dbox-attachment.h"
+#include "dbox-save.h"
+#include "sdbox-storage.h"
+#include "sdbox-file.h"
+#include "sdbox-sync.h"
+
+
+struct sdbox_save_context {
+	struct dbox_save_context ctx;
+
+	struct sdbox_mailbox *mbox;
+	struct sdbox_sync_context *sync_ctx;
+
+	struct dbox_file *cur_file;
+	struct dbox_file_append_context *append_ctx;
+
+	uint32_t first_saved_seq;
+	ARRAY(struct dbox_file *) files;
+};
+
+#define SDBOX_SAVECTX(s)	container_of(DBOX_SAVECTX(s), struct sdbox_save_context, ctx)
+
+struct dbox_file *
+sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq)
+{
+	struct sdbox_save_context *ctx = SDBOX_SAVECTX(t->save_ctx);
+	struct dbox_file *const *files, *file;
+	unsigned int count;
+
+	i_assert(seq >= ctx->first_saved_seq);
+
+	files = array_get(&ctx->files, &count);
+	i_assert(count > 0);
+	i_assert(seq - ctx->first_saved_seq < count);
+
+	file = files[seq - ctx->first_saved_seq];
+	i_assert(((struct sdbox_file *)file)->written_to_disk);
+	return file;
+}
+
+struct mail_save_context *
+sdbox_save_alloc(struct mailbox_transaction_context *t)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(t->box);
+	struct sdbox_save_context *ctx;
+
+	i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	if (t->save_ctx != NULL) {
+		/* use the existing allocated structure */
+		ctx = SDBOX_SAVECTX(t->save_ctx);
+		ctx->cur_file = NULL;
+		ctx->ctx.failed = FALSE;
+		ctx->ctx.finished = FALSE;
+		ctx->ctx.dbox_output = NULL;
+		return &ctx->ctx.ctx;
+	}
+
+	ctx = i_new(struct sdbox_save_context, 1);
+	ctx->ctx.ctx.transaction = t;
+	ctx->ctx.trans = t->itrans;
+	ctx->mbox = mbox;
+	i_array_init(&ctx->files, 32);
+	t->save_ctx = &ctx->ctx.ctx;
+	return t->save_ctx;
+}
+
+void sdbox_save_add_file(struct mail_save_context *_ctx, struct dbox_file *file)
+{
+	struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx);
+	struct dbox_file *const *files;
+	unsigned int count;
+
+	if (ctx->first_saved_seq == 0)
+		ctx->first_saved_seq = ctx->ctx.seq;
+
+	files = array_get(&ctx->files, &count);
+	if (count > 0) {
+		/* a plugin may leave a previously saved file open.
+		   we'll close it here to avoid eating too many fds. */
+		dbox_file_close(files[count-1]);
+	}
+	array_push_back(&ctx->files, &file);
+}
+
+int sdbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
+{
+	struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx);
+	struct dbox_file *file;
+	int ret;
+
+	file = sdbox_file_create(ctx->mbox);
+	ctx->append_ctx = dbox_file_append_init(file);
+	ret = dbox_file_get_append_stream(ctx->append_ctx,
+					  &ctx->ctx.dbox_output);
+	if (ret <= 0) {
+		i_assert(ret != 0);
+		dbox_file_append_rollback(&ctx->append_ctx);
+		dbox_file_unref(&file);
+		ctx->ctx.failed = TRUE;
+		return -1;
+	}
+	ctx->cur_file = file;
+	dbox_save_begin(&ctx->ctx, input);
+
+	sdbox_save_add_file(_ctx, file);
+	return ctx->ctx.failed ? -1 : 0;
+}
+
+static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx,
+					 struct dbox_file *file)
+{
+	struct sdbox_file *sfile = (struct sdbox_file *)file;
+	const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr;
+	const struct mail_attachment_extref *extrefs;
+	struct dbox_message_header dbox_msg_hdr;
+	uoff_t message_size;
+	guid_128_t guid_128;
+	unsigned int i, count;
+
+	i_assert(file->msg_header_size == sizeof(dbox_msg_hdr));
+
+	message_size = ctx->dbox_output->offset -
+		file->msg_header_size - file->file_header_size;
+
+	dbox_save_write_metadata(&ctx->ctx, ctx->dbox_output,
+				 message_size, NULL, guid_128);
+	dbox_msg_header_fill(&dbox_msg_hdr, message_size);
+	if (o_stream_pwrite(ctx->dbox_output, &dbox_msg_hdr,
+			    sizeof(dbox_msg_hdr),
+			    file->file_header_size) < 0) {
+		dbox_file_set_syscall_error(file, "pwrite()");
+		return -1;
+	}
+	sfile->written_to_disk = TRUE;
+
+	/* remember the attachment paths until commit time */
+	extrefs_arr = index_attachment_save_get_extrefs(&ctx->ctx);
+	if (extrefs_arr != NULL)
+		extrefs = array_get(extrefs_arr, &count);
+	else {
+		extrefs = NULL;
+		count = 0;
+	}
+	if (count > 0) {
+		sfile->attachment_pool =
+			pool_alloconly_create("sdbox attachment paths", 512);
+		p_array_init(&sfile->attachment_paths,
+			     sfile->attachment_pool, count);
+		for (i = 0; i < count; i++) {
+			const char *path = p_strdup(sfile->attachment_pool,
+						    extrefs[i].path);
+			array_push_back(&sfile->attachment_paths, &path);
+		}
+	}
+	return 0;
+}
+
+static int dbox_save_finish_write(struct mail_save_context *_ctx)
+{
+	struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
+	struct dbox_file **files;
+
+	ctx->ctx.finished = TRUE;
+	if (ctx->ctx.dbox_output == NULL)
+		return -1;
+
+	if (_ctx->data.save_date != (time_t)-1) {
+		/* we can't change ctime, but we can add the date to cache */
+		struct index_mail *mail = (struct index_mail *)_ctx->dest_mail;
+		uint32_t t = _ctx->data.save_date;
+
+		index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t));
+	}
+	dbox_save_end(&ctx->ctx);
+
+	files = array_back_modifiable(&ctx->files);
+	if (!ctx->ctx.failed) T_BEGIN {
+		if (dbox_save_mail_write_metadata(&ctx->ctx, *files) < 0)
+			ctx->ctx.failed = TRUE;
+	} T_END;
+
+	if (ctx->ctx.failed) {
+		index_storage_save_abort_last(&ctx->ctx.ctx, ctx->ctx.seq);
+		dbox_file_append_rollback(&ctx->append_ctx);
+		dbox_file_unlink(*files);
+		dbox_file_unref(files);
+		array_pop_back(&ctx->files);
+	} else {
+		dbox_file_append_checkpoint(ctx->append_ctx);
+		if (dbox_file_append_commit(&ctx->append_ctx) < 0)
+			ctx->ctx.failed = TRUE;
+		dbox_file_close(*files);
+	}
+
+	i_stream_unref(&ctx->ctx.input);
+	ctx->ctx.dbox_output = NULL;
+
+	return ctx->ctx.failed ? -1 : 0;
+}
+
+int sdbox_save_finish(struct mail_save_context *ctx)
+{
+	int ret;
+
+	ret = dbox_save_finish_write(ctx);
+	index_save_context_free(ctx);
+	return ret;
+}
+
+void sdbox_save_cancel(struct mail_save_context *_ctx)
+{
+	struct dbox_save_context *ctx = DBOX_SAVECTX(_ctx);
+
+	ctx->failed = TRUE;
+	(void)sdbox_save_finish(_ctx);
+}
+
+static int dbox_save_assign_uids(struct sdbox_save_context *ctx,
+				 const ARRAY_TYPE(seq_range) *uids)
+{
+	struct dbox_file *const *files;
+	struct seq_range_iter iter;
+	unsigned int i, count, n = 0;
+	uint32_t uid;
+	bool ret;
+
+	seq_range_array_iter_init(&iter, uids);
+	files = array_get(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		struct sdbox_file *sfile = (struct sdbox_file *)files[i];
+
+		ret = seq_range_array_iter_nth(&iter, n++, &uid);
+		i_assert(ret);
+		if (sdbox_file_assign_uid(sfile, uid) < 0)
+			return -1;
+		if (ctx->ctx.highest_pop3_uidl_seq == i+1) {
+			index_pop3_uidl_set_max_uid(&ctx->mbox->box,
+				ctx->ctx.trans, uid);
+		}
+	}
+	i_assert(!seq_range_array_iter_nth(&iter, n, &uid));
+	return 0;
+}
+
+static void dbox_save_unref_files(struct sdbox_save_context *ctx)
+{
+	struct dbox_file **files;
+	unsigned int i, count;
+
+	files = array_get_modifiable(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		if (ctx->ctx.failed) {
+			struct sdbox_file *sfile =
+				(struct sdbox_file *)files[i];
+
+			(void)sdbox_file_unlink_aborted_save(sfile);
+		}
+		dbox_file_unref(&files[i]);
+	}
+	array_free(&ctx->files);
+}
+
+int sdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
+{
+	struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx);
+	struct mailbox_transaction_context *_t = _ctx->transaction;
+	const struct mail_index_header *hdr;
+
+	i_assert(ctx->ctx.finished);
+
+	if (array_count(&ctx->files) == 0) {
+		/* the mail must be freed in the commit_pre() */
+		return 0;
+	}
+
+	if (sdbox_sync_begin(ctx->mbox, SDBOX_SYNC_FLAG_FORCE |
+			     SDBOX_SYNC_FLAG_FSYNC, &ctx->sync_ctx) < 0) {
+		sdbox_transaction_save_rollback(_ctx);
+		return -1;
+	}
+
+	/* update dbox header flags */
+	dbox_save_update_header_flags(&ctx->ctx, ctx->sync_ctx->sync_view,
+		ctx->mbox->hdr_ext_id, offsetof(struct sdbox_index_header, flags));
+
+	/* assign UIDs for new messages */
+	hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
+	mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
+				      &_t->changes->saved_uids);
+	if (dbox_save_assign_uids(ctx, &_t->changes->saved_uids) < 0) {
+		sdbox_transaction_save_rollback(_ctx);
+		return -1;
+	}
+
+	_t->changes->uid_validity = hdr->uid_validity;
+	return 0;
+}
+
+void sdbox_transaction_save_commit_post(struct mail_save_context *_ctx,
+					struct mail_index_transaction_commit_result *result)
+{
+	struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx);
+	struct mail_storage *storage = _ctx->transaction->box->storage;
+
+	_ctx->transaction = NULL; /* transaction is already freed */
+
+	if (array_count(&ctx->files) == 0) {
+		sdbox_transaction_save_rollback(_ctx);
+		return;
+	}
+
+	mail_index_sync_set_commit_result(ctx->sync_ctx->index_sync_ctx,
+					  result);
+
+	if (sdbox_sync_finish(&ctx->sync_ctx, TRUE) < 0)
+		ctx->ctx.failed = TRUE;
+
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
+		const char *box_path = mailbox_get_path(&ctx->mbox->box);
+
+		if (fdatasync_path(box_path) < 0) {
+			mail_set_critical(_ctx->dest_mail,
+				"fdatasync_path(%s) failed: %m", box_path);
+		}
+	}
+	i_assert(ctx->ctx.finished);
+	dbox_save_unref_files(ctx);
+	i_free(ctx);
+}
+
+void sdbox_transaction_save_rollback(struct mail_save_context *_ctx)
+{
+	struct sdbox_save_context *ctx = SDBOX_SAVECTX(_ctx);
+
+	ctx->ctx.failed = TRUE;
+	if (!ctx->ctx.finished)
+		sdbox_save_cancel(_ctx);
+	dbox_save_unref_files(ctx);
+
+	if (ctx->sync_ctx != NULL)
+		(void)sdbox_sync_finish(&ctx->sync_ctx, FALSE);
+	i_free(ctx);
+}
diff --git a/src/lib-storage/index/dbox-single/sdbox-storage.c b/src/lib-storage/index/dbox-single/sdbox-storage.c
new file mode 100644
index 0000000..5a6fcad
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-storage.c
@@ -0,0 +1,534 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "fs-api.h"
+#include "master-service.h"
+#include "mail-index-modseq.h"
+#include "mail-search-build.h"
+#include "mailbox-list-private.h"
+#include "index-pop3-uidl.h"
+#include "dbox-mail.h"
+#include "dbox-save.h"
+#include "sdbox-file.h"
+#include "sdbox-sync.h"
+#include "sdbox-storage.h"
+
+extern struct mail_storage dbox_storage, sdbox_storage;
+extern struct mailbox sdbox_mailbox;
+extern struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs;
+
+static struct event_category event_category_sdbox = {
+	.name = "sdbox",
+	.parent = &event_category_storage,
+};
+
+static struct mail_storage *sdbox_storage_alloc(void)
+{
+	struct sdbox_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("sdbox storage", 512+256);
+	storage = p_new(pool, struct sdbox_storage, 1);
+	storage->storage.v = sdbox_dbox_storage_vfuncs;
+	storage->storage.storage = sdbox_storage;
+	storage->storage.storage.pool = pool;
+	return &storage->storage.storage;
+}
+
+static int sdbox_storage_create(struct mail_storage *_storage,
+				struct mail_namespace *ns,
+				const char **error_r)
+{
+	struct dbox_storage *storage = DBOX_STORAGE(_storage);
+	enum fs_properties props;
+
+	if (dbox_storage_create(_storage, ns, error_r) < 0)
+		return -1;
+
+	if (storage->attachment_fs != NULL) {
+		props = fs_get_properties(storage->attachment_fs);
+		if ((props & FS_PROPERTY_RENAME) == 0) {
+			*error_r = "mail_attachment_fs: "
+				"Backend doesn't support renaming";
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static const char *
+sdbox_storage_find_root_dir(const struct mail_namespace *ns)
+{
+	bool debug = ns->mail_set->mail_debug;
+	const char *home, *path;
+
+	if (ns->owner != NULL &&
+	    mail_user_get_home(ns->owner, &home) > 0) {
+		path = t_strconcat(home, "/sdbox", NULL);
+		if (access(path, R_OK|W_OK|X_OK) == 0) {
+			if (debug)
+				i_debug("sdbox: root exists (%s)", path);
+			return path;
+		} 
+		if (debug)
+			i_debug("sdbox: access(%s, rwx): failed: %m", path);
+	}
+	return NULL;
+}
+
+static bool sdbox_storage_autodetect(const struct mail_namespace *ns,
+				     struct mailbox_list_settings *set)
+{
+	bool debug = ns->mail_set->mail_debug;
+	struct stat st;
+	const char *path, *root_dir;
+
+	if (set->root_dir != NULL)
+		root_dir = set->root_dir;
+	else {
+		root_dir = sdbox_storage_find_root_dir(ns);
+		if (root_dir == NULL) {
+			if (debug)
+				i_debug("sdbox: couldn't find root dir");
+			return FALSE;
+		}
+	}
+
+	/* NOTE: this check works for mdbox as well. we'll rely on the
+	   autodetect ordering to catch mdbox before we get here. */
+	path = t_strconcat(root_dir, "/"DBOX_MAILBOX_DIR_NAME, NULL);
+	if (stat(path, &st) < 0) {
+		if (debug)
+			i_debug("sdbox autodetect: stat(%s) failed: %m", path);
+		return FALSE;
+	}
+
+	if (!S_ISDIR(st.st_mode)) {
+		if (debug)
+			i_debug("sdbox autodetect: %s not a directory", path);
+		return FALSE;
+	}
+
+	set->root_dir = root_dir;
+	dbox_storage_get_list_settings(ns, set);
+	return TRUE;
+}
+
+static struct mailbox *
+sdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
+		    const char *vname, enum mailbox_flags flags)
+{
+	struct sdbox_mailbox *mbox;
+	struct index_mailbox_context *ibox;
+	pool_t pool;
+
+	/* dbox can't work without index files */
+	flags &= ENUM_NEGATE(MAILBOX_FLAG_NO_INDEX_FILES);
+
+	pool = pool_alloconly_create("sdbox mailbox", 1024*3);
+	mbox = p_new(pool, struct sdbox_mailbox, 1);
+	mbox->box = sdbox_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.mail_vfuncs = &sdbox_mail_vfuncs;
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
+
+	ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
+	ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS |
+		MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY;
+
+	mbox->storage = SDBOX_STORAGE(storage);
+	return &mbox->box;
+}
+
+int sdbox_read_header(struct sdbox_mailbox *mbox,
+		      struct sdbox_index_header *hdr, bool log_error,
+		      bool *need_resize_r)
+{
+	struct mail_index_view *view;
+	const void *data;
+	size_t data_size;
+	int ret = 0;
+
+	i_assert(mbox->box.opened);
+
+	view = mail_index_view_open(mbox->box.index);
+	mail_index_get_header_ext(view, mbox->hdr_ext_id,
+				  &data, &data_size);
+	if (data_size < SDBOX_INDEX_HEADER_MIN_SIZE &&
+	    (!mbox->box.creating || data_size != 0)) {
+		if (log_error) {
+			mailbox_set_critical(&mbox->box,
+				"sdbox: Invalid dbox header size");
+		}
+		ret = -1;
+	} else {
+		i_zero(hdr);
+		memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr)));
+		if (guid_128_is_empty(hdr->mailbox_guid))
+			ret = -1;
+		else {
+			/* data is valid. remember it in case mailbox
+			   is being reset */
+			mail_index_set_ext_init_data(mbox->box.index,
+						     mbox->hdr_ext_id,
+						     hdr, sizeof(*hdr));
+		}
+	}
+	mail_index_view_close(&view);
+	*need_resize_r = data_size < sizeof(*hdr);
+	return ret;
+}
+
+static void sdbox_update_header(struct sdbox_mailbox *mbox,
+				struct mail_index_transaction *trans,
+				const struct mailbox_update *update)
+{
+	struct sdbox_index_header hdr, new_hdr;
+	bool need_resize;
+
+	if (sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0) {
+		i_zero(&hdr);
+		need_resize = TRUE;
+	}
+
+	new_hdr = hdr;
+
+	if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) {
+		memcpy(new_hdr.mailbox_guid, update->mailbox_guid,
+		       sizeof(new_hdr.mailbox_guid));
+	} else if (guid_128_is_empty(new_hdr.mailbox_guid)) {
+		guid_128_generate(new_hdr.mailbox_guid);
+	}
+
+	if (need_resize) {
+		mail_index_ext_resize_hdr(trans, mbox->hdr_ext_id,
+					  sizeof(new_hdr));
+	}
+	if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) {
+		mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0,
+					     &new_hdr, sizeof(new_hdr));
+	}
+	memcpy(mbox->mailbox_guid, new_hdr.mailbox_guid,
+	       sizeof(mbox->mailbox_guid));
+}
+
+int sdbox_mailbox_create_indexes(struct mailbox *box,
+				 const struct mailbox_update *update,
+				 struct mail_index_transaction *trans)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
+	struct mail_index_transaction *new_trans = NULL;
+	const struct mail_index_header *hdr;
+	uint32_t uid_validity, uid_next;
+
+	if (trans == NULL) {
+		new_trans = mail_index_transaction_begin(box->view, 0);
+		trans = new_trans;
+	}
+
+	hdr = mail_index_get_header(box->view);
+	if (update != NULL && update->uid_validity != 0)
+		uid_validity = update->uid_validity;
+	else if (hdr->uid_validity != 0)
+		uid_validity = hdr->uid_validity;
+	else {
+		/* set uidvalidity */
+		uid_validity = dbox_get_uidvalidity_next(box->list);
+	}
+
+	if (hdr->uid_validity != uid_validity) {
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+	if (update != NULL && hdr->next_uid < update->min_next_uid) {
+		uid_next = update->min_next_uid;
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, next_uid),
+			&uid_next, sizeof(uid_next), TRUE);
+	}
+	if (update != NULL && update->min_first_recent_uid != 0 &&
+	    hdr->first_recent_uid < update->min_first_recent_uid) {
+		uint32_t first_recent_uid = update->min_first_recent_uid;
+
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, first_recent_uid),
+			&first_recent_uid, sizeof(first_recent_uid), FALSE);
+	}
+	if (update != NULL && update->min_highest_modseq != 0 &&
+	    mail_index_modseq_get_highest(box->view) <
+	    					update->min_highest_modseq) {
+		mail_index_modseq_enable(box->index);
+		mail_index_update_highest_modseq(trans,
+						 update->min_highest_modseq);
+	}
+
+	if (box->inbox_user && box->creating) {
+		/* initialize pop3-uidl header when creating mailbox
+		   (not on mailbox_update()) */
+		index_pop3_uidl_set_max_uid(box, trans, 0);
+	}
+
+	sdbox_update_header(mbox, trans, update);
+	if (new_trans != NULL) {
+		if (mail_index_transaction_commit(&new_trans) < 0) {
+			mailbox_set_index_error(box);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static const char *
+sdbox_get_attachment_path_suffix(struct dbox_file *_file)
+{
+	struct sdbox_file *file = (struct sdbox_file *)_file;
+
+	return t_strdup_printf("-%s-%u",
+			guid_128_to_string(file->mbox->mailbox_guid),
+			file->uid);
+}
+
+void sdbox_set_mailbox_corrupted(struct mailbox *box)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
+	struct sdbox_index_header hdr;
+	bool need_resize;
+
+	if (sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0 ||
+	    hdr.rebuild_count == 0)
+		mbox->corrupted_rebuild_count = 1;
+	else
+		mbox->corrupted_rebuild_count = hdr.rebuild_count;
+}
+
+static void sdbox_set_file_corrupted(struct dbox_file *_file)
+{
+	struct sdbox_file *file = (struct sdbox_file *)_file;
+
+	sdbox_set_mailbox_corrupted(&file->mbox->box);
+}
+
+static int sdbox_mailbox_alloc_index(struct sdbox_mailbox *mbox)
+{
+	struct sdbox_index_header hdr;
+
+	if (index_storage_mailbox_alloc_index(&mbox->box) < 0)
+		return -1;
+
+	mbox->hdr_ext_id =
+		mail_index_ext_register(mbox->box.index, "dbox-hdr",
+					sizeof(struct sdbox_index_header), 0, 0);
+	/* set the initialization data in case the mailbox is created */
+	i_zero(&hdr);
+	guid_128_generate(hdr.mailbox_guid);
+	mail_index_set_ext_init_data(mbox->box.index, mbox->hdr_ext_id,
+				     &hdr, sizeof(hdr));
+	return 0;
+}
+
+static int sdbox_mailbox_open(struct mailbox *box)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
+	struct sdbox_index_header hdr;
+	bool need_resize;
+	time_t path_ctime;
+
+	if (dbox_mailbox_check_existence(box, &path_ctime) < 0)
+		return -1;
+
+	if (sdbox_mailbox_alloc_index(mbox) < 0)
+		return -1;
+
+	if (dbox_mailbox_open(box, path_ctime) < 0)
+		return -1;
+
+	if (box->creating) {
+		/* wait for mailbox creation to initialize the index */
+		return 0;
+	}
+
+	/* get/generate mailbox guid */
+	if (sdbox_read_header(mbox, &hdr, FALSE, &need_resize) < 0) {
+		/* looks like the mailbox is corrupted */
+		(void)sdbox_sync(mbox, SDBOX_SYNC_FLAG_FORCE);
+		if (sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0)
+			i_zero(&hdr);
+	}
+
+	if (guid_128_is_empty(hdr.mailbox_guid)) {
+		/* regenerate it */
+		if (sdbox_mailbox_create_indexes(box, NULL, NULL) < 0 ||
+		    sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0)
+			return -1;
+	}
+	memcpy(mbox->mailbox_guid, hdr.mailbox_guid,
+	       sizeof(mbox->mailbox_guid));
+	return 0;
+}
+
+static void sdbox_mailbox_close(struct mailbox *box)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
+
+	if (mbox->corrupted_rebuild_count != 0)
+		(void)sdbox_sync(mbox, 0);
+	index_storage_mailbox_close(box);
+}
+
+static int
+sdbox_mailbox_create(struct mailbox *box,
+		     const struct mailbox_update *update, bool directory)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
+	struct sdbox_index_header hdr;
+	bool need_resize;
+
+	if (dbox_mailbox_create(box, update, directory) < 0)
+		return -1;
+	if (directory || !guid_128_is_empty(mbox->mailbox_guid))
+		return 0;
+
+	/* another process just created the mailbox. read the mailbox_guid. */
+	if (sdbox_read_header(mbox, &hdr, FALSE, &need_resize) < 0) {
+		mailbox_set_critical(box,
+			"sdbox: Failed to read newly created dbox header");
+		return -1;
+	}
+	memcpy(mbox->mailbox_guid, hdr.mailbox_guid,
+	       sizeof(mbox->mailbox_guid));
+	i_assert(!guid_128_is_empty(mbox->mailbox_guid));
+	return 0;
+}
+
+static int
+sdbox_mailbox_get_metadata(struct mailbox *box,
+			   enum mailbox_metadata_items items,
+			   struct mailbox_metadata *metadata_r)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
+
+	if (index_mailbox_get_metadata(box, items, metadata_r) < 0)
+		return -1;
+	if ((items & MAILBOX_METADATA_GUID) != 0) {
+		memcpy(metadata_r->guid, mbox->mailbox_guid,
+		       sizeof(metadata_r->guid));
+	}
+	return 0;
+}
+
+static int
+dbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update)
+{
+	if (!box->opened) {
+		if (mailbox_open(box) < 0)
+			return -1;
+	}
+	if (sdbox_mailbox_create_indexes(box, update, NULL) < 0)
+		return -1;
+	return index_storage_mailbox_update_common(box, update);
+}
+
+struct mail_storage sdbox_storage = {
+	.name = SDBOX_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_BINARY_DATA,
+	.event_category = &event_category_sdbox,
+
+	.v = {
+                NULL,
+		sdbox_storage_alloc,
+		sdbox_storage_create,
+		dbox_storage_destroy,
+		NULL,
+		dbox_storage_get_list_settings,
+		sdbox_storage_autodetect,
+		sdbox_mailbox_alloc,
+		NULL,
+		mail_storage_list_index_rebuild,
+	}
+};
+
+struct mail_storage dbox_storage = {
+	.name = "dbox", /* alias */
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG,
+	.event_category = &event_category_sdbox,
+
+	.v = {
+		NULL,
+		sdbox_storage_alloc,
+		sdbox_storage_create,
+		dbox_storage_destroy,
+		NULL,
+		dbox_storage_get_list_settings,
+		sdbox_storage_autodetect,
+		sdbox_mailbox_alloc,
+		NULL,
+		mail_storage_list_index_rebuild,
+	}
+};
+
+struct mailbox sdbox_mailbox = {
+	.v = {
+		index_storage_is_readonly,
+		index_storage_mailbox_enable,
+		index_storage_mailbox_exists,
+		sdbox_mailbox_open,
+		sdbox_mailbox_close,
+		index_storage_mailbox_free,
+		sdbox_mailbox_create,
+		dbox_mailbox_update,
+		index_storage_mailbox_delete,
+		index_storage_mailbox_rename,
+		index_storage_get_status,
+		sdbox_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		index_storage_list_index_has_changed,
+		index_storage_list_index_update_sync,
+		sdbox_storage_sync_init,
+		index_mailbox_sync_next,
+		index_mailbox_sync_deinit,
+		NULL,
+		dbox_notify_changes,
+		index_transaction_begin,
+		index_transaction_commit,
+		index_transaction_rollback,
+		NULL,
+		dbox_mail_alloc,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		sdbox_save_alloc,
+		sdbox_save_begin,
+		dbox_save_continue,
+		sdbox_save_finish,
+		sdbox_save_cancel,
+		sdbox_copy,
+		sdbox_transaction_save_commit_pre,
+		sdbox_transaction_save_commit_post,
+		sdbox_transaction_save_rollback,
+		index_storage_is_inconsistent
+	}
+};
+
+struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs = {
+	sdbox_file_free,
+	sdbox_file_create_fd,
+	sdbox_mail_open,
+	sdbox_mailbox_create_indexes,
+	sdbox_get_attachment_path_suffix,
+	sdbox_set_mailbox_corrupted,
+	sdbox_set_file_corrupted
+};
diff --git a/src/lib-storage/index/dbox-single/sdbox-storage.h b/src/lib-storage/index/dbox-single/sdbox-storage.h
new file mode 100644
index 0000000..66d08da
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-storage.h
@@ -0,0 +1,69 @@
+#ifndef SDBOX_STORAGE_H
+#define SDBOX_STORAGE_H
+
+#include "index-storage.h"
+#include "dbox-storage.h"
+
+#define SDBOX_STORAGE_NAME "sdbox"
+#define SDBOX_MAIL_FILE_PREFIX "u."
+#define SDBOX_MAIL_FILE_FORMAT SDBOX_MAIL_FILE_PREFIX"%u"
+
+#define SDBOX_INDEX_HEADER_MIN_SIZE (sizeof(uint32_t))
+struct sdbox_index_header {
+	/* increased every time a full mailbox rebuild is done */
+	uint32_t rebuild_count;
+	guid_128_t mailbox_guid;
+	uint8_t flags; /* enum dbox_index_header_flags */
+	uint8_t unused[3];
+};
+
+struct sdbox_storage {
+	struct dbox_storage storage;
+};
+
+struct sdbox_mailbox {
+	struct mailbox box;
+	struct sdbox_storage *storage;
+
+	uint32_t hdr_ext_id;
+	/* if non-zero, storage should be rebuilt (except if rebuild_count
+	   has changed from this value) */
+	uint32_t corrupted_rebuild_count;
+
+	guid_128_t mailbox_guid;
+};
+
+#define SDBOX_STORAGE(s)	container_of(DBOX_STORAGE(s), struct sdbox_storage, storage)
+#define SDBOX_MAILBOX(s)	container_of(s, struct sdbox_mailbox, box)
+
+extern struct mail_vfuncs sdbox_mail_vfuncs;
+
+int sdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r,
+		    struct dbox_file **file_r);
+
+int sdbox_read_header(struct sdbox_mailbox *mbox,
+		      struct sdbox_index_header *hdr, bool log_error,
+		      bool *need_resize_r);
+int sdbox_mailbox_create_indexes(struct mailbox *box,
+				 const struct mailbox_update *update,
+				 struct mail_index_transaction *trans);
+void sdbox_set_mailbox_corrupted(struct mailbox *box);
+
+struct mail_save_context *
+sdbox_save_alloc(struct mailbox_transaction_context *_t);
+int sdbox_save_begin(struct mail_save_context *ctx, struct istream *input);
+int sdbox_save_finish(struct mail_save_context *ctx);
+void sdbox_save_cancel(struct mail_save_context *ctx);
+
+struct dbox_file *
+sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq);
+void sdbox_save_add_file(struct mail_save_context *ctx, struct dbox_file *file);
+
+int sdbox_transaction_save_commit_pre(struct mail_save_context *ctx);
+void sdbox_transaction_save_commit_post(struct mail_save_context *ctx,
+					struct mail_index_transaction_commit_result *result);
+void sdbox_transaction_save_rollback(struct mail_save_context *ctx);
+
+int sdbox_copy(struct mail_save_context *ctx, struct mail *mail);
+
+#endif
diff --git a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c
new file mode 100644
index 0000000..aa4832f
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c
@@ -0,0 +1,218 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "index-rebuild.h"
+#include "mail-cache.h"
+#include "sdbox-storage.h"
+#include "sdbox-file.h"
+#include "sdbox-sync.h"
+
+#include <dirent.h>
+
+static void sdbox_sync_set_uidvalidity(struct index_rebuild_context *ctx)
+{
+	uint32_t uid_validity;
+
+	/* if uidvalidity is set in the old index, use it */
+	uid_validity = mail_index_get_header(ctx->view)->uid_validity;
+	if (uid_validity == 0)
+		uid_validity = dbox_get_uidvalidity_next(ctx->box->list);
+
+	mail_index_update_header(ctx->trans,
+		offsetof(struct mail_index_header, uid_validity),
+		&uid_validity, sizeof(uid_validity), TRUE);
+}
+
+static int
+sdbox_sync_add_file_index(struct index_rebuild_context *ctx,
+			  struct dbox_file *file, uint32_t uid, bool primary)
+{
+	uint32_t seq;
+	bool deleted;
+	int ret;
+
+	if ((ret = dbox_file_open(file, &deleted)) > 0) {
+		if (deleted)
+			return 0;
+		ret = dbox_file_seek(file, 0);
+	}
+	if (ret == 0) {
+		if ((ret = dbox_file_fix(file, 0)) > 0)
+			ret = dbox_file_seek(file, 0);
+	}
+
+	if (ret <= 0) {
+		if (ret < 0)
+			return -1;
+
+		i_warning("sdbox: Skipping unfixable file: %s", file->cur_path);
+		return 0;
+	}
+
+	if (!dbox_file_is_in_alt(file) && !primary) {
+		/* we were supposed to open the file in alt storage, but it
+		   exists in primary storage as well. skip it to avoid adding
+		   it twice. */
+		return 0;
+	}
+
+	mail_index_append(ctx->trans, uid, &seq);
+	T_BEGIN {
+		index_rebuild_index_metadata(ctx, seq, uid);
+	} T_END;
+	return 0;
+}
+
+static int
+sdbox_sync_add_file(struct index_rebuild_context *ctx,
+		    const char *fname, bool primary)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(ctx->box);
+	struct dbox_file *file;
+	uint32_t uid;
+	int ret;
+
+	if (!str_begins(fname, SDBOX_MAIL_FILE_PREFIX))
+		return 0;
+	fname += strlen(SDBOX_MAIL_FILE_PREFIX);
+
+	if (str_to_uint32(fname, &uid) < 0 || uid == 0) {
+		i_warning("sdbox %s: Ignoring invalid filename %s",
+			  mailbox_get_path(ctx->box), fname);
+		return 0;
+	}
+
+	file = sdbox_file_init(mbox, uid);
+	if (!primary)
+		file->cur_path = file->alt_path;
+	ret = sdbox_sync_add_file_index(ctx, file, uid, primary);
+	dbox_file_unref(&file);
+	return ret;
+}
+
+static int sdbox_sync_index_rebuild_dir(struct index_rebuild_context *ctx,
+					const char *path, bool primary)
+{
+	DIR *dir;
+	struct dirent *d;
+	int ret = 0;
+
+	dir = opendir(path);
+	if (dir == NULL) {
+		if (errno == ENOENT) {
+			if (!primary) {
+				/* alt directory doesn't exist, ignore */
+				return 0;
+			}
+			return index_mailbox_fix_inconsistent_existence(ctx->box, path);
+		}
+		mailbox_set_critical(ctx->box, "opendir(%s) failed: %m", path);
+		return -1;
+	}
+	do {
+		errno = 0;
+		if ((d = readdir(dir)) == NULL)
+			break;
+
+		ret = sdbox_sync_add_file(ctx, d->d_name, primary);
+	} while (ret >= 0);
+	if (errno != 0) {
+		mailbox_set_critical(ctx->box, "readdir(%s) failed: %m", path);
+		ret = -1;
+	}
+
+	if (closedir(dir) < 0) {
+		mailbox_set_critical(ctx->box, "closedir(%s) failed: %m", path);
+		ret = -1;
+	}
+	return ret;
+}
+
+static void sdbox_sync_update_header(struct index_rebuild_context *ctx)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(ctx->box);
+	struct sdbox_index_header hdr;
+	bool need_resize;
+
+	if (sdbox_read_header(mbox, &hdr, FALSE, &need_resize) < 0)
+		i_zero(&hdr);
+	if (guid_128_is_empty(hdr.mailbox_guid))
+		guid_128_generate(hdr.mailbox_guid);
+	if (++hdr.rebuild_count == 0)
+		hdr.rebuild_count = 1;
+	/* mailbox is being reset. this gets written directly there */
+	mail_index_set_ext_init_data(ctx->box->index, mbox->hdr_ext_id,
+				     &hdr, sizeof(hdr));
+}
+
+static int
+sdbox_sync_index_rebuild_singles(struct index_rebuild_context *ctx)
+{
+	const char *path, *alt_path;
+	int ret = 0;
+
+	path = mailbox_get_path(ctx->box);
+	if (mailbox_get_path_to(ctx->box, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX,
+				&alt_path) < 0)
+		return -1;
+
+	sdbox_sync_set_uidvalidity(ctx);
+	if (sdbox_sync_index_rebuild_dir(ctx, path, TRUE) < 0) {
+		mailbox_set_critical(ctx->box, "sdbox: Rebuilding failed");
+		ret = -1;
+	} else if (alt_path != NULL) {
+		if (sdbox_sync_index_rebuild_dir(ctx, alt_path, FALSE) < 0) {
+			mailbox_set_critical(ctx->box,
+				"sdbox: Rebuilding failed on alt path %s",
+				alt_path);
+			ret = -1;
+		}
+	}
+	sdbox_sync_update_header(ctx);
+	return ret;
+}
+
+int sdbox_sync_index_rebuild(struct sdbox_mailbox *mbox, bool force)
+{
+	struct index_rebuild_context *ctx;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	struct sdbox_index_header hdr;
+	bool need_resize;
+	int ret;
+
+	if (!force && sdbox_read_header(mbox, &hdr, FALSE, &need_resize) == 0) {
+		if (hdr.rebuild_count != mbox->corrupted_rebuild_count &&
+		    hdr.rebuild_count != 0) {
+			/* already rebuilt by someone else */
+			return 0;
+		}
+	}
+	i_warning("sdbox %s: Rebuilding index", mailbox_get_path(&mbox->box));
+
+	if (dbox_verify_alt_storage(mbox->box.list) < 0) {
+		mailbox_set_critical(&mbox->box,
+			"sdbox: Alt storage not mounted, "
+			"aborting index rebuild");
+		return -1;
+	}
+
+	view = mail_index_view_open(mbox->box.index);
+	trans = mail_index_transaction_begin(view,
+					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+
+	ctx = index_index_rebuild_init(&mbox->box, view, trans);
+	ret = sdbox_sync_index_rebuild_singles(ctx);
+	index_index_rebuild_deinit(&ctx, dbox_get_uidvalidity_next);
+
+	if (ret < 0)
+		mail_index_transaction_rollback(&trans);
+	else {
+		mail_index_unset_fscked(trans);
+		ret = mail_index_transaction_commit(&trans);
+	}
+	mail_index_view_close(&view);
+	mbox->corrupted_rebuild_count = 0;
+	return ret;
+}
diff --git a/src/lib-storage/index/dbox-single/sdbox-sync.c b/src/lib-storage/index/dbox-single/sdbox-sync.c
new file mode 100644
index 0000000..670b768
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-sync.c
@@ -0,0 +1,326 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dbox-attachment.h"
+#include "sdbox-storage.h"
+#include "sdbox-file.h"
+#include "sdbox-sync.h"
+#include "mailbox-recent-flags.h"
+
+#define SDBOX_REBUILD_COUNT 3
+
+static void
+dbox_sync_file_move_if_needed(struct dbox_file *file,
+			      enum sdbox_sync_entry_type type)
+{
+	struct stat st;
+	bool move_to_alt = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT;
+	bool deleted;
+
+	if (move_to_alt == dbox_file_is_in_alt(file) &&
+	    !move_to_alt) {
+		/* unopened dbox files default to primary dir.
+		   stat the file to update its location. */
+		(void)dbox_file_stat(file, &st);
+
+	}
+	if (move_to_alt != dbox_file_is_in_alt(file)) {
+		/* move the file. if it fails, nothing broke so
+		   don't worry about it. */
+		if (dbox_file_open(file, &deleted) > 0 && !deleted)
+			(void)sdbox_file_move(file, move_to_alt);
+	}
+}
+
+static void sdbox_sync_file(struct sdbox_sync_context *ctx,
+			    uint32_t seq, uint32_t uid,
+			    enum sdbox_sync_entry_type type)
+{
+	struct dbox_file *file;
+	enum modify_type modify_type;
+
+	switch (type) {
+	case SDBOX_SYNC_ENTRY_TYPE_EXPUNGE:
+		if (!mail_index_transaction_is_expunged(ctx->trans, seq)) {
+			mail_index_expunge(ctx->trans, seq);
+			array_push_back(&ctx->expunged_uids, &uid);
+		}
+		break;
+	case SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT:
+	case SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT:
+		/* update flags in the sync transaction, mainly to make
+		   sure that these alt changes get marked as synced
+		   and won't be retried */
+		modify_type = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT ?
+			MODIFY_ADD : MODIFY_REMOVE;
+		mail_index_update_flags(ctx->trans, seq, modify_type,
+					(enum mail_flags)DBOX_INDEX_FLAG_ALT);
+		file = sdbox_file_init(ctx->mbox, uid);
+		dbox_sync_file_move_if_needed(file, type);
+		dbox_file_unref(&file);
+		break;
+	}
+}
+
+static void sdbox_sync_add(struct sdbox_sync_context *ctx,
+			   const struct mail_index_sync_rec *sync_rec)
+{
+	uint32_t uid;
+	enum sdbox_sync_entry_type type;
+	uint32_t seq, seq1, seq2;
+
+	if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
+		/* we're interested */
+		type = SDBOX_SYNC_ENTRY_TYPE_EXPUNGE;
+	} else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) {
+		/* we care only about alt flag changes */
+		if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0)
+			type = SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT;
+		else if ((sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) != 0)
+			type = SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT;
+		else
+			return;
+	} else {
+		/* not interested */
+		return;
+	}
+
+	if (!mail_index_lookup_seq_range(ctx->sync_view,
+					 sync_rec->uid1, sync_rec->uid2,
+					 &seq1, &seq2)) {
+		/* already expunged everything. nothing to do. */
+		return;
+	}
+
+	for (seq = seq1; seq <= seq2; seq++) {
+		mail_index_lookup_uid(ctx->sync_view, seq, &uid);
+		sdbox_sync_file(ctx, seq, uid, type);
+	}
+}
+
+static int sdbox_sync_index(struct sdbox_sync_context *ctx)
+{
+	struct mailbox *box = &ctx->mbox->box;
+	const struct mail_index_header *hdr;
+	struct mail_index_sync_rec sync_rec;
+	uint32_t seq1, seq2;
+
+	hdr = mail_index_get_header(ctx->sync_view);
+	if (hdr->uid_validity == 0) {
+		/* newly created index file */
+		if (hdr->next_uid == 1) {
+			/* could be just a race condition where we opened the
+			   mailbox between mkdir and index creation. fix this
+			   silently. */
+			if (sdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0)
+				return -1;
+			return 1;
+		}
+		mailbox_set_critical(box,
+			"sdbox: Broken index: missing UIDVALIDITY");
+		sdbox_set_mailbox_corrupted(box);
+		return 0;
+	}
+
+	/* mark the newly seen messages as recent */
+	if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
+					hdr->next_uid, &seq1, &seq2))
+		mailbox_recent_flags_set_seqs(box, ctx->sync_view, seq1, seq2);
+
+	while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec))
+		sdbox_sync_add(ctx, &sync_rec);
+	return 1;
+}
+
+static void dbox_sync_file_expunge(struct sdbox_sync_context *ctx,
+				   uint32_t uid)
+{
+	struct mailbox *box = &ctx->mbox->box;
+	struct dbox_file *file;
+	struct sdbox_file *sfile;
+	int ret;
+
+	file = sdbox_file_init(ctx->mbox, uid);
+	sfile = (struct sdbox_file *)file;
+	if (file->storage->attachment_dir != NULL)
+		ret = sdbox_file_unlink_with_attachments(sfile);
+	else
+		ret = dbox_file_unlink(file);
+
+	/* do sync_notify only when the file was unlinked by us */
+	if (ret > 0)
+		mailbox_sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
+	dbox_file_unref(&file);
+}
+
+static void dbox_sync_expunge_files(struct sdbox_sync_context *ctx)
+{
+	uint32_t uid;
+
+	/* NOTE: Index is no longer locked. Multiple processes may be unlinking
+	   the files at the same time. */
+	ctx->mbox->box.tmp_sync_view = ctx->sync_view;
+	array_foreach_elem(&ctx->expunged_uids, uid) T_BEGIN {
+		dbox_sync_file_expunge(ctx, uid);
+	} T_END;
+	mailbox_sync_notify(&ctx->mbox->box, 0, 0);
+	ctx->mbox->box.tmp_sync_view = NULL;
+}
+
+static int
+sdbox_refresh_header(struct sdbox_mailbox *mbox, bool retry, bool log_error)
+{
+	struct mail_index_view *view;
+	struct sdbox_index_header hdr;
+	bool need_resize;
+	int ret;
+
+	view = mail_index_view_open(mbox->box.index);
+	ret = sdbox_read_header(mbox, &hdr, log_error, &need_resize);
+	mail_index_view_close(&view);
+
+	if (ret < 0 && retry) {
+		mail_index_refresh(mbox->box.index);
+		return sdbox_refresh_header(mbox, FALSE, log_error);
+	}
+	return ret;
+}
+
+int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags,
+		     struct sdbox_sync_context **ctx_r)
+{
+	const struct mail_index_header *hdr =
+		mail_index_get_header(mbox->box.view);
+	struct sdbox_sync_context *ctx;
+	enum mail_index_sync_flags sync_flags;
+	unsigned int i;
+	int ret;
+	bool rebuild, force_rebuild;
+
+	force_rebuild = (flags & SDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
+	rebuild = force_rebuild ||
+		(hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0 ||
+		mbox->corrupted_rebuild_count != 0 ||
+		sdbox_refresh_header(mbox, TRUE, FALSE) < 0;
+
+	ctx = i_new(struct sdbox_sync_context, 1);
+	ctx->mbox = mbox;
+	ctx->flags = flags;
+	i_array_init(&ctx->expunged_uids, 32);
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box);
+	if (!rebuild && (flags & SDBOX_SYNC_FLAG_FORCE) == 0)
+		sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
+	if ((flags & SDBOX_SYNC_FLAG_FSYNC) != 0)
+		sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC;
+	/* don't write unnecessary dirty flag updates */
+	sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;
+
+	for (i = 0;; i++) {
+		ret = index_storage_expunged_sync_begin(&mbox->box,
+				&ctx->index_sync_ctx, &ctx->sync_view,
+				&ctx->trans, sync_flags);
+		if (mail_index_reset_fscked(mbox->box.index))
+			sdbox_set_mailbox_corrupted(&mbox->box);
+		if (ret <= 0) {
+			array_free(&ctx->expunged_uids);
+			i_free(ctx);
+			*ctx_r = NULL;
+			return ret;
+		}
+
+		if (rebuild)
+			ret = 0;
+		else {
+			if ((ret = sdbox_sync_index(ctx)) > 0)
+				break;
+		}
+
+		/* failure. keep the index locked while we're doing a
+		   rebuild. */
+		if (ret == 0) {
+			if (i >= SDBOX_REBUILD_COUNT) {
+				mailbox_set_critical(&ctx->mbox->box,
+					"sdbox: Index keeps breaking");
+				ret = -1;
+			} else {
+				/* do a full resync and try again. */
+				rebuild = FALSE;
+				ret = sdbox_sync_index_rebuild(mbox,
+							       force_rebuild);
+			}
+		}
+		mail_index_sync_rollback(&ctx->index_sync_ctx);
+		if (ret < 0) {
+			index_storage_expunging_deinit(&ctx->mbox->box);
+			array_free(&ctx->expunged_uids);
+			i_free(ctx);
+			return -1;
+		}
+	}
+
+	*ctx_r = ctx;
+	return 0;
+}
+
+int sdbox_sync_finish(struct sdbox_sync_context **_ctx, bool success)
+{
+	struct sdbox_sync_context *ctx = *_ctx;
+	struct mail_storage *storage = &ctx->mbox->storage->storage.storage;
+	int ret = success ? 0 : -1;
+
+	*_ctx = NULL;
+
+	if (success) {
+		mail_index_view_ref(ctx->sync_view);
+
+		if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
+			mailbox_set_index_error(&ctx->mbox->box);
+			ret = -1;
+		} else {
+			dbox_sync_expunge_files(ctx);
+			mail_index_view_close(&ctx->sync_view);
+		}
+	} else {
+		mail_index_sync_rollback(&ctx->index_sync_ctx);
+	}
+
+        if (storage->rebuild_list_index)
+		ret = mail_storage_list_index_rebuild_and_set_uncorrupted(storage);
+
+	index_storage_expunging_deinit(&ctx->mbox->box);
+	array_free(&ctx->expunged_uids);
+	i_free(ctx);
+	return ret;
+}
+
+int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags)
+{
+	struct sdbox_sync_context *sync_ctx;
+
+	if (sdbox_sync_begin(mbox, flags, &sync_ctx) < 0)
+		return -1;
+
+	if (sync_ctx == NULL)
+		return 0;
+	return sdbox_sync_finish(&sync_ctx, TRUE);
+}
+
+struct mailbox_sync_context *
+sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct sdbox_mailbox *mbox = SDBOX_MAILBOX(box);
+	enum sdbox_sync_flags sdbox_sync_flags = 0;
+	int ret = 0;
+
+	if (mail_index_reset_fscked(box->index))
+		sdbox_set_mailbox_corrupted(box);
+	if (index_mailbox_want_full_sync(&mbox->box, flags) ||
+	    mbox->corrupted_rebuild_count != 0) {
+		if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)
+			sdbox_sync_flags |= SDBOX_SYNC_FLAG_FORCE_REBUILD;
+		ret = sdbox_sync(mbox, sdbox_sync_flags);
+	}
+
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
diff --git a/src/lib-storage/index/dbox-single/sdbox-sync.h b/src/lib-storage/index/dbox-single/sdbox-sync.h
new file mode 100644
index 0000000..70306c9
--- /dev/null
+++ b/src/lib-storage/index/dbox-single/sdbox-sync.h
@@ -0,0 +1,38 @@
+#ifndef SDBOX_SYNC_H
+#define SDBOX_SYNC_H
+
+struct mailbox;
+struct sdbox_mailbox;
+
+enum sdbox_sync_flags {
+	SDBOX_SYNC_FLAG_FORCE		= 0x01,
+	SDBOX_SYNC_FLAG_FSYNC		= 0x02,
+	SDBOX_SYNC_FLAG_FORCE_REBUILD	= 0x04
+};
+
+enum sdbox_sync_entry_type {
+	SDBOX_SYNC_ENTRY_TYPE_EXPUNGE,
+	SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT,
+	SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT
+};
+
+struct sdbox_sync_context {
+	struct sdbox_mailbox *mbox;
+        struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *trans;
+	enum sdbox_sync_flags flags;
+	ARRAY_TYPE(uint32_t) expunged_uids;
+};
+
+int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags,
+		     struct sdbox_sync_context **ctx_r);
+int sdbox_sync_finish(struct sdbox_sync_context **ctx, bool success);
+int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags);
+
+int sdbox_sync_index_rebuild(struct sdbox_mailbox *mbox, bool force);
+
+struct mailbox_sync_context *
+sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+
+#endif
diff --git a/src/lib-storage/index/imapc/Makefile.am b/src/lib-storage/index/imapc/Makefile.am
new file mode 100644
index 0000000..72ee102
--- /dev/null
+++ b/src/lib-storage/index/imapc/Makefile.am
@@ -0,0 +1,36 @@
+noinst_LTLIBRARIES = libstorage_imapc.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-imap-client \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/list \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/lib-ssl-iostream
+
+libstorage_imapc_la_SOURCES = \
+	imapc-list.c \
+	imapc-mail.c \
+	imapc-mail-fetch.c \
+	imapc-mailbox.c \
+	imapc-save.c \
+	imapc-search.c \
+	imapc-settings.c \
+	imapc-sync.c \
+	imapc-storage.c
+
+headers = \
+	imapc-list.h \
+	imapc-mail.h \
+	imapc-search.h \
+	imapc-settings.h \
+	imapc-storage.h \
+	imapc-sync.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/imapc/Makefile.in b/src/lib-storage/index/imapc/Makefile.in
new file mode 100644
index 0000000..d36e1ce
--- /dev/null
+++ b/src/lib-storage/index/imapc/Makefile.in
@@ -0,0 +1,861 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/imapc
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_imapc_la_LIBADD =
+am_libstorage_imapc_la_OBJECTS = imapc-list.lo imapc-mail.lo \
+	imapc-mail-fetch.lo imapc-mailbox.lo imapc-save.lo \
+	imapc-search.lo imapc-settings.lo imapc-sync.lo \
+	imapc-storage.lo
+libstorage_imapc_la_OBJECTS = $(am_libstorage_imapc_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/imapc-list.Plo \
+	./$(DEPDIR)/imapc-mail-fetch.Plo ./$(DEPDIR)/imapc-mail.Plo \
+	./$(DEPDIR)/imapc-mailbox.Plo ./$(DEPDIR)/imapc-save.Plo \
+	./$(DEPDIR)/imapc-search.Plo ./$(DEPDIR)/imapc-settings.Plo \
+	./$(DEPDIR)/imapc-storage.Plo ./$(DEPDIR)/imapc-sync.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_imapc_la_SOURCES)
+DIST_SOURCES = $(libstorage_imapc_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_imapc.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-imap-client \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/list \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/lib-ssl-iostream
+
+libstorage_imapc_la_SOURCES = \
+	imapc-list.c \
+	imapc-mail.c \
+	imapc-mail-fetch.c \
+	imapc-mailbox.c \
+	imapc-save.c \
+	imapc-search.c \
+	imapc-settings.c \
+	imapc-sync.c \
+	imapc-storage.c
+
+headers = \
+	imapc-list.h \
+	imapc-mail.h \
+	imapc-search.h \
+	imapc-settings.h \
+	imapc-storage.h \
+	imapc-sync.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/imapc/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/imapc/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_imapc.la: $(libstorage_imapc_la_OBJECTS) $(libstorage_imapc_la_DEPENDENCIES) $(EXTRA_libstorage_imapc_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_imapc_la_OBJECTS) $(libstorage_imapc_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-list.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-mail-fetch.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-mailbox.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-save.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-search.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-settings.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/imapc-sync.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/imapc-list.Plo
+	-rm -f ./$(DEPDIR)/imapc-mail-fetch.Plo
+	-rm -f ./$(DEPDIR)/imapc-mail.Plo
+	-rm -f ./$(DEPDIR)/imapc-mailbox.Plo
+	-rm -f ./$(DEPDIR)/imapc-save.Plo
+	-rm -f ./$(DEPDIR)/imapc-search.Plo
+	-rm -f ./$(DEPDIR)/imapc-settings.Plo
+	-rm -f ./$(DEPDIR)/imapc-storage.Plo
+	-rm -f ./$(DEPDIR)/imapc-sync.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/imapc-list.Plo
+	-rm -f ./$(DEPDIR)/imapc-mail-fetch.Plo
+	-rm -f ./$(DEPDIR)/imapc-mail.Plo
+	-rm -f ./$(DEPDIR)/imapc-mailbox.Plo
+	-rm -f ./$(DEPDIR)/imapc-save.Plo
+	-rm -f ./$(DEPDIR)/imapc-search.Plo
+	-rm -f ./$(DEPDIR)/imapc-settings.Plo
+	-rm -f ./$(DEPDIR)/imapc-storage.Plo
+	-rm -f ./$(DEPDIR)/imapc-sync.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/imapc/imapc-list.c b/src/lib-storage/index/imapc/imapc-list.c
new file mode 100644
index 0000000..a9e03ec
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-list.c
@@ -0,0 +1,1011 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+/*
+   There are various different mailbox names here. Here's an example assuming
+    - imapc_list_prefix = "prefix"
+    - remote imapc server separator = '/'
+    - mailbox_list separator = '^' (actually this is currently always the same
+      as remote separator, but this clarifies the example)
+    - namespace separator = ':'
+    - fs_list separator = '.'
+    - mailbox_list storage_name_escape_char = '+'
+    - mailbox_list vname_escape_char = '~'
+    - fs_list storage_name_escape_char = '%'
+
+   remote_name = "prefix/~foo/bar^baz+_%_."
+   storage_name = "prefix^~foo^bar+5ebaz+2b_%_."
+    - separator is changed from / to ^
+    - conflicting ^ separator in remote_name is escaped as +5e
+    - storage_name_escape character + is escaped as +2b
+   vname = "~7efoo:bar.baz+_%_."
+    - imapc_list_prefix is dropped
+    - vname_escape_character ~ is escaped into ~7e
+    - separator is changed from ^ to :
+    - storage_name_escape_characters are unescaped
+   fs_name = "prefix.~foo.bar^baz+_%25_%2e"
+    - this is generated from remote_name
+    - separator is changed from / to .
+    - storage_name_escape_character=% and fs_list separator . are escaped
+*/
+
+#include "lib.h"
+#include "ioloop.h"
+#include "str.h"
+#include "imap-arg.h"
+#include "imap-match.h"
+#include "imap-utf7.h"
+#include "mailbox-tree.h"
+#include "mailbox-list-subscriptions.h"
+#include "imapc-storage.h"
+#include "imapc-list.h"
+
+struct imapc_mailbox_list_iterate_context {
+	struct mailbox_list_iterate_context ctx;
+	struct mailbox_tree_context *tree;
+	struct mailbox_node *ns_root;
+
+	struct mailbox_tree_iterate_context *iter;
+	struct mailbox_info info;
+	string_t *special_use;
+};
+
+static struct {
+	const char *str;
+	enum mailbox_info_flags flag;
+} imap_list_flags[] = {
+	{ "\\NoSelect", MAILBOX_NOSELECT },
+	{ "\\NonExistent", MAILBOX_NONEXISTENT },
+	{ "\\NoInferiors", MAILBOX_NOINFERIORS },
+	{ "\\Subscribed", MAILBOX_SUBSCRIBED },
+	{ "\\All", MAILBOX_SPECIALUSE_ALL },
+	{ "\\Archive", MAILBOX_SPECIALUSE_ARCHIVE },
+	{ "\\Drafts", MAILBOX_SPECIALUSE_DRAFTS },
+	{ "\\Flagged", MAILBOX_SPECIALUSE_FLAGGED },
+	{ "\\Junk", MAILBOX_SPECIALUSE_JUNK },
+	{ "\\Sent", MAILBOX_SPECIALUSE_SENT },
+	{ "\\Trash", MAILBOX_SPECIALUSE_TRASH },
+	{ "\\Important", MAILBOX_SPECIALUSE_IMPORTANT }
+};
+
+extern struct mailbox_list imapc_mailbox_list;
+
+static void imapc_list_send_hierarchy_sep_lookup(struct imapc_mailbox_list *list);
+static void imapc_untagged_list(const struct imapc_untagged_reply *reply,
+				struct imapc_storage_client *client);
+static void imapc_untagged_lsub(const struct imapc_untagged_reply *reply,
+				struct imapc_storage_client *client);
+
+static struct mailbox_list *imapc_list_alloc(void)
+{
+	struct imapc_mailbox_list *list;
+	pool_t pool;
+
+	pool = pool_alloconly_create("imapc mailbox list", 1024);
+	list = p_new(pool, struct imapc_mailbox_list, 1);
+	list->list = imapc_mailbox_list;
+	list->list.pool = pool;
+	/* separator is set lazily */
+	list->mailboxes = mailbox_tree_init('\0');
+	mailbox_tree_set_parents_nonexistent(list->mailboxes);
+	return &list->list;
+}
+
+static int imapc_list_init(struct mailbox_list *_list, const char **error_r)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+
+	list->set = mail_user_set_get_driver_settings(_list->ns->user->set_info,
+						      _list->ns->user_set,
+						      IMAPC_STORAGE_NAME);
+	if (imapc_storage_client_create(_list->ns, list->set, _list->mail_set,
+					&list->client, error_r) < 0)
+		return -1;
+	list->client->_list = list;
+
+	imapc_storage_client_register_untagged(list->client, "LIST",
+					       imapc_untagged_list);
+	imapc_storage_client_register_untagged(list->client, "LSUB",
+					       imapc_untagged_lsub);
+	imapc_list_send_hierarchy_sep_lookup(list);
+	return 0;
+}
+
+static void imapc_list_deinit(struct mailbox_list *_list)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+
+	/* make sure all pending commands are aborted before anything is
+	   deinitialized */
+	if (list->client != NULL) {
+		list->client->destroying = TRUE;
+		imapc_client_logout(list->client->client);
+		imapc_storage_client_unref(&list->client);
+	}
+	if (list->index_list != NULL)
+		mailbox_list_destroy(&list->index_list);
+	mailbox_tree_deinit(&list->mailboxes);
+	if (list->tmp_subscriptions != NULL)
+		mailbox_tree_deinit(&list->tmp_subscriptions);
+	pool_unref(&list->list.pool);
+}
+
+static void
+imapc_list_copy_error_from_reply(struct imapc_mailbox_list *list,
+				 enum mail_error default_error,
+				 const struct imapc_command_reply *reply)
+{
+	enum mail_error error;
+
+	if (imapc_resp_text_code_parse(reply->resp_text_key, &error)) {
+		mailbox_list_set_error(&list->list, error,
+				       reply->text_without_resp);
+	} else {
+		mailbox_list_set_error(&list->list, default_error,
+				       reply->text_without_resp);
+	}
+}
+
+static void imapc_list_simple_callback(const struct imapc_command_reply *reply,
+				       void *context)
+{
+	struct imapc_simple_context *ctx = context;
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		ctx->ret = 0;
+	else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		imapc_list_copy_error_from_reply(ctx->client->_list,
+						 MAIL_ERROR_PARAMS, reply);
+		ctx->ret = -1;
+	} else if (imapc_storage_client_handle_auth_failure(ctx->client)) {
+		ctx->ret = -1;
+	} else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		mailbox_list_set_internal_error(&ctx->client->_list->list);
+		ctx->ret = -1;
+	} else {
+		mailbox_list_set_critical(&ctx->client->_list->list,
+			"imapc: Command failed: %s", reply->text_full);
+		ctx->ret = -1;
+	}
+	imapc_client_stop(ctx->client->client);
+}
+
+static bool
+imap_list_flag_parse(const char *str, enum mailbox_info_flags *flag_r)
+{
+	unsigned int i;
+
+	for (i = 0; i < N_ELEMENTS(imap_list_flags); i++) {
+		if (strcasecmp(str, imap_list_flags[i].str) == 0) {
+			*flag_r = imap_list_flags[i].flag;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static const char *
+imapc_list_remote_to_storage_name(struct imapc_mailbox_list *list,
+				  const char *remote_name)
+{
+	/* typically mailbox_list_escape_name() is used to escape vname into
+	   a list name. but we want to convert remote IMAP name to a list name,
+	   so we need to use the remote IMAP separator. */
+	return mailbox_list_escape_name_params(remote_name, "",
+		list->root_sep,
+		mailbox_list_get_hierarchy_sep(&list->list),
+		list->list.set.storage_name_escape_char, "");
+}
+
+static const char *
+imapc_list_remote_to_vname(struct imapc_mailbox_list *list,
+			   const char *remote_name)
+{
+	return mailbox_list_get_vname(&list->list,
+		imapc_list_remote_to_storage_name(list, remote_name));
+}
+
+const char *
+imapc_list_storage_to_remote_name(struct imapc_mailbox_list *list,
+				  const char *storage_name)
+{
+	return mailbox_list_unescape_name_params(storage_name, "", list->root_sep,
+				mailbox_list_get_hierarchy_sep(&list->list),
+				list->list.set.storage_name_escape_char);
+}
+
+static struct mailbox_node *
+imapc_list_update_tree(struct imapc_mailbox_list *list,
+		       struct mailbox_tree_context *tree,
+		       const struct imap_arg *args)
+{
+	struct mailbox_node *node;
+	const struct imap_arg *flags;
+	const char *remote_name, *flag;
+	enum mailbox_info_flags info_flag, info_flags = 0;
+	bool created;
+
+	if (!imap_arg_get_list(&args[0], &flags) ||
+	    args[1].type == IMAP_ARG_EOL ||
+	    !imap_arg_get_astring(&args[2], &remote_name))
+		return NULL;
+
+	while (imap_arg_get_atom(flags, &flag)) {
+		if (imap_list_flag_parse(flag, &info_flag))
+			info_flags |= info_flag;
+		flags++;
+	}
+
+	T_BEGIN {
+		const char *vname =
+			imapc_list_remote_to_vname(list, remote_name);
+
+		if ((info_flags & MAILBOX_NONEXISTENT) != 0)
+			node = mailbox_tree_lookup(tree, vname);
+		else
+			node = mailbox_tree_get(tree, vname, &created);
+	} T_END;
+	if (node != NULL)
+		node->flags = info_flags;
+	return node;
+}
+
+static void imapc_untagged_list(const struct imapc_untagged_reply *reply,
+				struct imapc_storage_client *client)
+{
+	struct imapc_mailbox_list *list = client->_list;
+	const struct imap_arg *args = reply->args;
+	const char *sep, *remote_name;
+
+	if (list->root_sep == '\0') {
+		/* we haven't asked for the separator yet.
+		   lets see if this is the reply for its request. */
+		if (args[0].type == IMAP_ARG_EOL ||
+		    !imap_arg_get_nstring(&args[1], &sep) ||
+		    !imap_arg_get_astring(&args[2], &remote_name))
+			return;
+
+		/* we can't handle NIL separator yet */
+		list->root_sep = sep == NULL ? '/' : sep[0];
+		mailbox_tree_set_separator(list->mailboxes, list->root_sep);
+	} else {
+		(void)imapc_list_update_tree(list, list->mailboxes, args);
+	}
+}
+
+static void imapc_untagged_lsub(const struct imapc_untagged_reply *reply,
+				struct imapc_storage_client *client)
+{
+	struct imapc_mailbox_list *list = client->_list;
+	const struct imap_arg *args = reply->args;
+	struct mailbox_node *node;
+
+	if (list->root_sep == '\0') {
+		/* we haven't asked for the separator yet */
+		return;
+	}
+	node = imapc_list_update_tree(list, list->tmp_subscriptions != NULL ?
+				      list->tmp_subscriptions :
+				      list->list.subscriptions, args);
+	if (node != NULL) {
+		if ((node->flags & MAILBOX_NOSELECT) == 0)
+			node->flags |= MAILBOX_SUBSCRIBED;
+		else {
+			/* LSUB \Noselect means that the mailbox isn't
+			   subscribed, but it has children that are */
+			node->flags &= ENUM_NEGATE(MAILBOX_NOSELECT);
+		}
+	}
+}
+
+static void imapc_list_sep_verify(struct imapc_mailbox_list *list)
+{
+	const char *imapc_list_prefix = list->set->imapc_list_prefix;
+
+	if (list->root_sep == '\0') {
+		mailbox_list_set_critical(&list->list,
+			"imapc: LIST didn't return hierarchy separator");
+	} else if (imapc_list_prefix[0] != '\0' &&
+		   imapc_list_prefix[strlen(imapc_list_prefix)-1] == list->root_sep) {
+		mailbox_list_set_critical(&list->list,
+			"imapc_list_prefix must not end with hierarchy separator");
+	}
+}
+
+static void imapc_storage_sep_callback(const struct imapc_command_reply *reply,
+				       void *context)
+{
+	struct imapc_mailbox_list *list = context;
+
+	list->root_sep_pending = FALSE;
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		imapc_list_sep_verify(list);
+	else if (reply->state == IMAPC_COMMAND_STATE_NO)
+		imapc_list_copy_error_from_reply(list, MAIL_ERROR_PARAMS, reply);
+	else if (imapc_storage_client_handle_auth_failure(list->client))
+		;
+	else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED)
+		mailbox_list_set_internal_error(&list->list);
+	else if (!list->list.ns->user->deinitializing) {
+		mailbox_list_set_critical(&list->list,
+			"imapc: Command failed: %s", reply->text_full);
+	}
+	imapc_client_stop(list->client->client);
+}
+
+static void imapc_list_send_hierarchy_sep_lookup(struct imapc_mailbox_list *list)
+{
+	struct imapc_command *cmd;
+
+	if (list->root_sep_pending)
+		return;
+	list->root_sep_pending = TRUE;
+
+	cmd = imapc_client_cmd(list->client->client,
+			       imapc_storage_sep_callback, list);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_send(cmd, "LIST \"\" \"\"");
+}
+
+int imapc_list_try_get_root_sep(struct imapc_mailbox_list *list, char *sep_r)
+{
+	if (list->root_sep == '\0') {
+		if (imapc_storage_client_handle_auth_failure(list->client))
+			return -1;
+		imapc_list_send_hierarchy_sep_lookup(list);
+		while (list->root_sep_pending)
+			imapc_client_run(list->client->client);
+		if (list->root_sep == '\0')
+			return -1;
+	}
+	*sep_r = list->root_sep;
+	return 0;
+}
+
+static char imapc_list_get_hierarchy_sep(struct mailbox_list *_list)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	char sep;
+
+	if (imapc_list_try_get_root_sep(list, &sep) < 0) {
+		/* we can't really return a failure here. just return a common
+		   separator and fail all the future list operations. */
+		return '/';
+	}
+	return sep;
+}
+
+static const char *
+imapc_list_get_storage_name(struct mailbox_list *_list, const char *vname)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	const char *prefix = list->set->imapc_list_prefix;
+	const char *storage_name;
+
+	storage_name = mailbox_list_default_get_storage_name(_list, vname);
+	if (*prefix != '\0' && strcasecmp(storage_name, "INBOX") != 0) {
+		storage_name = storage_name[0] == '\0' ? prefix :
+			t_strdup_printf("%s%c%s", prefix,
+			mailbox_list_get_hierarchy_sep(_list),
+			storage_name);
+	}
+	return storage_name;
+}
+
+static const char *
+imapc_list_get_vname(struct mailbox_list *_list, const char *storage_name)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	const char *prefix = list->set->imapc_list_prefix;
+	size_t prefix_len;
+
+	if (*storage_name == '\0') {
+		/* ACL plugin does these lookups */
+	} else if (*prefix != '\0' && strcasecmp(storage_name, "INBOX") != 0) {
+		prefix_len = strlen(prefix);
+		i_assert(str_begins(storage_name, prefix));
+		storage_name += prefix_len;
+		if (storage_name[0] == '\0') {
+			/* we're looking up the prefix itself */
+		} else {
+			i_assert(storage_name[0] ==
+				 mailbox_list_get_hierarchy_sep(_list));
+			storage_name++;
+		}
+	}
+	return mailbox_list_default_get_vname(_list, storage_name);
+}
+
+static struct mailbox_list *imapc_list_get_fs(struct imapc_mailbox_list *list)
+{
+	struct mailbox_list_settings list_set;
+	const char *error, *dir;
+
+	dir = list->list.set.index_dir;
+	if (dir == NULL)
+		dir = list->list.set.root_dir;
+
+	if (dir == NULL || dir[0] == '\0') {
+		/* indexes disabled */
+	} else if (list->index_list == NULL && !list->index_list_failed) {
+		mailbox_list_settings_init_defaults(&list_set);
+		list_set.layout = MAILBOX_LIST_NAME_MAILDIRPLUSPLUS;
+		list_set.root_dir = dir;
+		list_set.index_pvt_dir = p_strdup_empty(list->list.pool, list->list.set.index_pvt_dir);
+		/* Filesystem needs to be able to store any kind of a mailbox
+		   name. */
+		list_set.storage_name_escape_char =
+			IMAPC_LIST_FS_NAME_ESCAPE_CHAR;
+
+		if (mailbox_list_create(list_set.layout, list->list.ns,
+					&list_set, MAILBOX_LIST_FLAG_SECONDARY,
+					&list->index_list, &error) < 0) {
+			i_error("imapc: Couldn't create %s mailbox list: %s",
+				list_set.layout, error);
+			list->index_list_failed = TRUE;
+		}
+	}
+	return list->index_list;
+}
+
+static const char *
+imapc_list_storage_to_fs_name(struct imapc_mailbox_list *list,
+			      const char *storage_name)
+{
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+	const char *remote_name;
+
+	if (storage_name == NULL)
+		return NULL;
+
+	remote_name = imapc_list_storage_to_remote_name(list, storage_name);
+	return mailbox_list_escape_name_params(remote_name, "",
+			list->root_sep,
+			mailbox_list_get_hierarchy_sep(fs_list),
+			fs_list->set.storage_name_escape_char, "");
+}
+
+static const char *
+imapc_list_fs_to_storage_name(struct imapc_mailbox_list *list,
+			      const char *fs_name)
+{
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+	const char *remote_name;
+
+	if (fs_name == NULL)
+		return NULL;
+
+	remote_name = mailbox_list_unescape_name_params(fs_name, "",
+			list->root_sep,
+			mailbox_list_get_hierarchy_sep(fs_list),
+			fs_list->set.storage_name_escape_char);
+	return imapc_list_remote_to_storage_name(list, remote_name);
+}
+
+static int
+imapc_list_get_path(struct mailbox_list *_list, const char *name,
+		    enum mailbox_list_path_type type, const char **path_r)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+	const char *fs_name;
+
+	if (fs_list != NULL) {
+		fs_name = imapc_list_storage_to_fs_name(list, name);
+		return mailbox_list_get_path(fs_list, fs_name, type, path_r);
+	} else {
+		*path_r = NULL;
+		return 0;
+	}
+}
+
+static const char *
+imapc_list_get_temp_prefix(struct mailbox_list *_list, bool global)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+
+	if (fs_list != NULL) {
+		return global ?
+			mailbox_list_get_global_temp_prefix(fs_list) :
+			mailbox_list_get_temp_prefix(fs_list);
+	} else {
+		i_panic("imapc: Can't return a temp prefix for '%s'",
+			_list->ns->prefix);
+	}
+}
+
+static const char *
+imapc_list_join_refpattern(struct mailbox_list *list ATTR_UNUSED,
+			   const char *ref, const char *pattern)
+{
+	return t_strconcat(ref, pattern, NULL);
+}
+
+static struct imapc_command *
+imapc_list_simple_context_init(struct imapc_simple_context *ctx,
+			       struct imapc_mailbox_list *list)
+{
+	imapc_simple_context_init(ctx, list->client);
+	return imapc_client_cmd(list->client->client,
+				imapc_list_simple_callback, ctx);
+}
+
+static void imapc_list_delete_unused_indexes(struct imapc_mailbox_list *list)
+{
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+	struct mailbox_list_iterate_context *iter;
+	const struct mailbox_info *info;
+	const char *fs_name, *storage_name, *vname;
+
+	if (fs_list == NULL)
+		return;
+
+	iter = mailbox_list_iter_init(fs_list, "*",
+				      MAILBOX_LIST_ITER_RAW_LIST |
+				      MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+				      MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
+	while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+		fs_name = mailbox_list_get_storage_name(fs_list, info->vname);
+		storage_name = imapc_list_fs_to_storage_name(list, fs_name);
+		vname = mailbox_list_get_vname(&list->list, storage_name);
+
+		/* list->mailboxes contains proper vnames. fs_vname  */
+		if (mailbox_tree_lookup(list->mailboxes, vname) == NULL)
+			(void)fs_list->v.delete_mailbox(fs_list, fs_name);
+	} T_END;
+	(void)mailbox_list_iter_deinit(&iter);
+}
+
+static int imapc_list_refresh(struct imapc_mailbox_list *list)
+{
+	struct imapc_command *cmd;
+	struct imapc_simple_context ctx;
+	struct mailbox_node *node;
+	const char *pattern;
+	char sep;
+
+	if (imapc_list_try_get_root_sep(list, &sep) < 0)
+		return -1;
+	if (list->refreshed_mailboxes)
+		return 0;
+
+	if (*list->set->imapc_list_prefix == '\0')
+		pattern = "*";
+	else {
+		/* list "prefix*" instead of "prefix.*". this may return a bit
+		   more than we want, but we're also interested in the flags
+		   of the prefix itself. */
+		pattern = t_strdup_printf("%s*", list->set->imapc_list_prefix);
+	}
+
+	cmd = imapc_list_simple_context_init(&ctx, list);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_sendf(cmd, "LIST \"\" %s", pattern);
+	mailbox_tree_deinit(&list->mailboxes);
+	list->mailboxes = mailbox_tree_init(mail_namespace_get_sep(list->list.ns));
+	mailbox_tree_set_parents_nonexistent(list->mailboxes);
+	imapc_simple_run(&ctx, &cmd);
+
+	if ((list->list.ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
+		/* INBOX always exists in IMAP server. since this namespace is
+		   marked with inbox=yes, show the INBOX even if
+		   imapc_list_prefix doesn't match it */
+		bool created;
+		node = mailbox_tree_get(list->mailboxes, "INBOX", &created);
+		if (*list->set->imapc_list_prefix != '\0') {
+			/* this listing didn't include the INBOX itself, but
+			   might have included its children. make sure there
+			   aren't any extra flags in it (especially
+			   \NonExistent) */
+			node->flags &= MAILBOX_CHILDREN;
+		}
+	}
+
+	if (ctx.ret == 0) {
+		list->refreshed_mailboxes = TRUE;
+		list->refreshed_mailboxes_recently = TRUE;
+		list->last_refreshed_mailboxes = ioloop_time;
+		imapc_list_delete_unused_indexes(list);
+	}
+	return ctx.ret;
+}
+
+static void
+imapc_list_build_match_tree(struct imapc_mailbox_list_iterate_context *ctx)
+{
+	struct imapc_mailbox_list *list =
+		(struct imapc_mailbox_list *)ctx->ctx.list;
+	struct mailbox_list_iter_update_context update_ctx;
+	struct mailbox_tree_iterate_context *iter;
+	struct mailbox_node *node;
+	const char *vname;
+
+	i_zero(&update_ctx);
+	update_ctx.iter_ctx = &ctx->ctx;
+	update_ctx.tree_ctx = ctx->tree;
+	update_ctx.glob = ctx->ctx.glob;
+	update_ctx.match_parents = TRUE;
+
+	iter = mailbox_tree_iterate_init(list->mailboxes, NULL, 0);
+	while ((node = mailbox_tree_iterate_next(iter, &vname)) != NULL) {
+		update_ctx.leaf_flags = node->flags;
+		mailbox_list_iter_update(&update_ctx, vname);
+	}
+	mailbox_tree_iterate_deinit(&iter);
+}
+
+static struct mailbox_list_iterate_context *
+imapc_list_iter_init(struct mailbox_list *_list, const char *const *patterns,
+		     enum mailbox_list_iter_flags flags)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	struct mailbox_list_iterate_context *_ctx;
+	struct imapc_mailbox_list_iterate_context *ctx;
+	pool_t pool;
+	const char *ns_root_name;
+	char ns_sep;
+	int ret = 0;
+
+	if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0 ||
+	    (flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) == 0)
+		ret = imapc_list_refresh(list);
+
+	list->iter_count++;
+
+	if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0) {
+		/* we're listing only subscriptions. just use the cached
+		   subscriptions list. */
+		_ctx = mailbox_list_subscriptions_iter_init(_list, patterns,
+							    flags);
+		if (ret < 0)
+			_ctx->failed = TRUE;
+		return _ctx;
+	}
+
+	/* if we've already failed, make sure we don't call
+	   mailbox_list_get_hierarchy_sep(), since it clears the error */
+	ns_sep = ret < 0 ? '/' : mail_namespace_get_sep(_list->ns);
+
+	pool = pool_alloconly_create("mailbox list imapc iter", 1024);
+	ctx = p_new(pool, struct imapc_mailbox_list_iterate_context, 1);
+	ctx->ctx.pool = pool;
+	ctx->ctx.list = _list;
+	ctx->ctx.flags = flags;
+	ctx->ctx.glob = imap_match_init_multiple(pool, patterns, FALSE, ns_sep);
+	array_create(&ctx->ctx.module_contexts, pool, sizeof(void *), 5);
+
+	ctx->info.ns = _list->ns;
+
+	ctx->tree = mailbox_tree_init(ns_sep);
+	mailbox_tree_set_parents_nonexistent(ctx->tree);
+	if (ret == 0)
+		imapc_list_build_match_tree(ctx);
+
+	if (list->list.ns->prefix_len > 0) {
+		ns_root_name = t_strndup(_list->ns->prefix,
+					 _list->ns->prefix_len - 1);
+		ctx->ns_root = mailbox_tree_lookup(ctx->tree, ns_root_name);
+	}
+
+	ctx->iter = mailbox_tree_iterate_init(ctx->tree, NULL, 0);
+	if (ret < 0)
+		ctx->ctx.failed = TRUE;
+	return &ctx->ctx;
+}
+
+static void
+imapc_list_write_special_use(struct imapc_mailbox_list_iterate_context *ctx,
+			     struct mailbox_node *node)
+{
+	unsigned int i;
+
+	if (ctx->special_use == NULL)
+		ctx->special_use = str_new(ctx->ctx.pool, 64);
+	str_truncate(ctx->special_use, 0);
+
+	for (i = 0; i < N_ELEMENTS(imap_list_flags); i++) {
+		if ((node->flags & imap_list_flags[i].flag) != 0 &&
+		    (node->flags & MAILBOX_SPECIALUSE_MASK) != 0) {
+			str_append(ctx->special_use, imap_list_flags[i].str);
+			str_append_c(ctx->special_use, ' ');
+		}
+	}
+
+	if (str_len(ctx->special_use) > 0) {
+		str_truncate(ctx->special_use, str_len(ctx->special_use) - 1);
+		ctx->info.special_use = str_c(ctx->special_use);
+	} else {
+		ctx->info.special_use = NULL;
+	}
+}
+
+static bool
+imapc_list_is_ns_root(struct imapc_mailbox_list_iterate_context *ctx,
+		      struct mailbox_node *node)
+{
+	struct mailbox_node *root_node = ctx->ns_root;
+
+	while (root_node != NULL) {
+		if (node == root_node)
+			return TRUE;
+		root_node = root_node->parent;
+	}
+	return FALSE;
+}
+
+static const struct mailbox_info *
+imapc_list_iter_next(struct mailbox_list_iterate_context *_ctx)
+{
+	struct imapc_mailbox_list_iterate_context *ctx =
+		(struct imapc_mailbox_list_iterate_context *)_ctx;
+	struct imapc_mailbox_list *list =
+		(struct imapc_mailbox_list *)_ctx->list;
+	struct mailbox_node *node;
+	const char *vname;
+
+	if (_ctx->failed)
+		return NULL;
+
+	if ((_ctx->flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0)
+		return mailbox_list_subscriptions_iter_next(_ctx);
+
+	do {
+		node = mailbox_tree_iterate_next(ctx->iter, &vname);
+		if (node == NULL)
+			return mailbox_list_iter_default_next(_ctx);
+	} while ((node->flags & MAILBOX_MATCHED) == 0 ||
+		 imapc_list_is_ns_root(ctx, node));
+
+	if (ctx->info.ns->prefix_len > 0 &&
+	    strcasecmp(vname, "INBOX") != 0 &&
+	    strncmp(vname, ctx->info.ns->prefix, ctx->info.ns->prefix_len-1) == 0 &&
+	    vname[ctx->info.ns->prefix_len] == '\0' &&
+	    list->set->imapc_list_prefix[0] == '\0') {
+		/* don't return "" name */
+		return imapc_list_iter_next(_ctx);
+	}
+
+	ctx->info.vname = vname;
+	ctx->info.flags = node->flags;
+	if ((_ctx->list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
+		/* we're iterating the INBOX namespace. pass through the
+		   SPECIAL-USE flags if they exist. */
+		imapc_list_write_special_use(ctx, node);
+	} else {
+		ctx->info.special_use = NULL;
+	}
+	return &ctx->info;
+}
+
+static int imapc_list_iter_deinit(struct mailbox_list_iterate_context *_ctx)
+{
+	struct imapc_mailbox_list_iterate_context *ctx =
+		(struct imapc_mailbox_list_iterate_context *)_ctx;
+	struct imapc_mailbox_list *list =
+		(struct imapc_mailbox_list *)_ctx->list;
+	int ret = _ctx->failed ? -1 : 0;
+
+	i_assert(list->iter_count > 0);
+
+	if (--list->iter_count == 0) {
+		list->refreshed_mailboxes = FALSE;
+		list->refreshed_subscriptions = FALSE;
+	}
+
+	if ((_ctx->flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0)
+		return mailbox_list_subscriptions_iter_deinit(_ctx);
+
+	mailbox_tree_iterate_deinit(&ctx->iter);
+	mailbox_tree_deinit(&ctx->tree);
+	pool_unref(&_ctx->pool);
+	return ret;
+}
+
+static int
+imapc_list_subscriptions_refresh(struct mailbox_list *_src_list,
+				 struct mailbox_list *dest_list)
+{
+	struct imapc_mailbox_list *src_list =
+		(struct imapc_mailbox_list *)_src_list;
+	struct imapc_simple_context ctx;
+	struct imapc_command *cmd;
+	const char *pattern;
+	char list_sep, dest_sep = mail_namespace_get_sep(dest_list->ns);
+
+	i_assert(src_list->tmp_subscriptions == NULL);
+
+	if (imapc_list_try_get_root_sep(src_list, &list_sep) < 0)
+		return -1;
+
+	if (src_list->refreshed_subscriptions) {
+		if (dest_list->subscriptions == NULL)
+			dest_list->subscriptions = mailbox_tree_init(dest_sep);
+		return 0;
+	}
+
+	src_list->tmp_subscriptions =
+		mailbox_tree_init(mail_namespace_get_sep(_src_list->ns));
+
+	cmd = imapc_list_simple_context_init(&ctx, src_list);
+	if (*src_list->set->imapc_list_prefix == '\0')
+		pattern = "*";
+	else
+		pattern = t_strdup_printf("%s*", src_list->set->imapc_list_prefix);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_sendf(cmd, "LSUB \"\" %s", pattern);
+	imapc_simple_run(&ctx, &cmd);
+
+	if (ctx.ret < 0)
+		return -1;
+
+	/* replace subscriptions tree in destination */
+	if (dest_list->subscriptions != NULL)
+		mailbox_tree_deinit(&dest_list->subscriptions);
+	dest_list->subscriptions = src_list->tmp_subscriptions;
+	src_list->tmp_subscriptions = NULL;
+	mailbox_tree_set_separator(dest_list->subscriptions, dest_sep);
+
+	src_list->refreshed_subscriptions = TRUE;
+	return 0;
+}
+
+static int imapc_list_set_subscribed(struct mailbox_list *_list,
+				     const char *name, bool set)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	struct imapc_command *cmd;
+	struct imapc_simple_context ctx;
+
+	cmd = imapc_list_simple_context_init(&ctx, list);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_sendf(cmd, set ? "SUBSCRIBE %s" : "UNSUBSCRIBE %s",
+			    imapc_list_storage_to_remote_name(list, name));
+	imapc_simple_run(&ctx, &cmd);
+	return ctx.ret;
+}
+
+static int
+imapc_list_delete_mailbox(struct mailbox_list *_list, const char *name)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+	enum imapc_capability capa;
+	struct imapc_command *cmd;
+	struct imapc_simple_context ctx;
+
+	if (imapc_storage_client_handle_auth_failure(list->client))
+		return -1;
+	if (imapc_client_get_capabilities(list->client->client, &capa) < 0)
+		return -1;
+
+	cmd = imapc_list_simple_context_init(&ctx, list);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	if (!imapc_command_connection_is_selected(cmd))
+		imapc_command_abort(&cmd);
+	else {
+		imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT);
+		if ((capa & IMAPC_CAPABILITY_UNSELECT) != 0)
+			imapc_command_sendf(cmd, "UNSELECT");
+		else
+			imapc_command_sendf(cmd, "SELECT \"~~~\"");
+		imapc_simple_run(&ctx, &cmd);
+	}
+
+	cmd = imapc_list_simple_context_init(&ctx, list);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_sendf(cmd, "DELETE %s", imapc_list_storage_to_remote_name(list, name));
+	imapc_simple_run(&ctx, &cmd);
+
+	if (fs_list != NULL && ctx.ret == 0) {
+		const char *fs_name = imapc_list_storage_to_fs_name(list, name);
+		(void)fs_list->v.delete_mailbox(fs_list, fs_name);
+	}
+	return ctx.ret;
+}
+
+static int
+imapc_list_delete_dir(struct mailbox_list *_list, const char *name)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+
+	if (fs_list != NULL) {
+		const char *fs_name = imapc_list_storage_to_fs_name(list, name);
+		(void)mailbox_list_delete_dir(fs_list, fs_name);
+	}
+	return 0;
+}
+
+static int
+imapc_list_delete_symlink(struct mailbox_list *list,
+			  const char *name ATTR_UNUSED)
+{
+	mailbox_list_set_error(list, MAIL_ERROR_NOTPOSSIBLE, "Not supported");
+	return -1;
+}
+
+static int
+imapc_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname,
+			  struct mailbox_list *newlist, const char *newname)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)oldlist;
+	struct mailbox_list *fs_list = imapc_list_get_fs(list);
+	struct imapc_command *cmd;
+	struct imapc_simple_context ctx;
+
+	if (oldlist != newlist) {
+		mailbox_list_set_error(oldlist, MAIL_ERROR_NOTPOSSIBLE,
+			"Can't rename mailboxes across storages.");
+		return -1;
+	}
+
+	cmd = imapc_list_simple_context_init(&ctx, list);
+	imapc_command_sendf(cmd, "RENAME %s %s",
+			    imapc_list_storage_to_remote_name(list, oldname),
+			    imapc_list_storage_to_remote_name(list, newname));
+	imapc_simple_run(&ctx, &cmd);
+	if (ctx.ret == 0 && fs_list != NULL && oldlist == newlist) {
+		const char *old_fs_name =
+			imapc_list_storage_to_fs_name(list, oldname);
+		const char *new_fs_name =
+			imapc_list_storage_to_fs_name(list, newname);
+		(void)fs_list->v.rename_mailbox(fs_list, old_fs_name,
+						fs_list, new_fs_name);
+	}
+	return ctx.ret;
+}
+
+int imapc_list_get_mailbox_flags(struct mailbox_list *_list, const char *name,
+				 enum mailbox_info_flags *flags_r)
+{
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list;
+	struct mailbox_node *node;
+	const char *vname;
+
+	vname = mailbox_list_get_vname(_list, name);
+	if (!list->refreshed_mailboxes_recently) {
+		if (imapc_list_refresh(list) < 0)
+			return -1;
+		i_assert(list->refreshed_mailboxes_recently);
+	}
+
+	if (list->mailboxes == NULL) {
+		/* imapc list isn't used, but e.g. LAYOUT=none */
+		*flags_r = 0;
+		return 0;
+	}
+	node = mailbox_tree_lookup(list->mailboxes, vname);
+	if (node == NULL)
+		*flags_r = MAILBOX_NONEXISTENT;
+	else
+		*flags_r = node->flags;
+	return 0;
+}
+
+struct mailbox_list imapc_mailbox_list = {
+	.name = MAILBOX_LIST_NAME_IMAPC,
+	.props = MAILBOX_LIST_PROP_NO_ROOT | MAILBOX_LIST_PROP_AUTOCREATE_DIRS |
+		 MAILBOX_LIST_PROP_NO_LIST_INDEX,
+	.mailbox_name_max_length = MAILBOX_LIST_NAME_MAX_LENGTH,
+
+	.v = {
+		.alloc = imapc_list_alloc,
+		.init = imapc_list_init,
+		.deinit = imapc_list_deinit,
+		.get_hierarchy_sep = imapc_list_get_hierarchy_sep,
+		.get_vname = imapc_list_get_vname,
+		.get_storage_name = imapc_list_get_storage_name,
+		.get_path = imapc_list_get_path,
+		.get_temp_prefix = imapc_list_get_temp_prefix,
+		.join_refpattern = imapc_list_join_refpattern,
+		.iter_init = imapc_list_iter_init,
+		.iter_next = imapc_list_iter_next,
+		.iter_deinit = imapc_list_iter_deinit,
+		.subscriptions_refresh = imapc_list_subscriptions_refresh,
+		.set_subscribed = imapc_list_set_subscribed,
+		.delete_mailbox = imapc_list_delete_mailbox,
+		.delete_dir = imapc_list_delete_dir,
+		.delete_symlink = imapc_list_delete_symlink,
+		.rename_mailbox = imapc_list_rename_mailbox,
+	}
+};
diff --git a/src/lib-storage/index/imapc/imapc-list.h b/src/lib-storage/index/imapc/imapc-list.h
new file mode 100644
index 0000000..11df6b0
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-list.h
@@ -0,0 +1,41 @@
+#ifndef IMAPC_LIST_H
+#define IMAPC_LIST_H
+
+struct imap_arg;
+
+#include "mailbox-list-private.h"
+
+#define MAILBOX_LIST_NAME_IMAPC "imapc"
+
+struct imapc_mailbox_list {
+	struct mailbox_list list;
+	const struct imapc_settings *set;
+	struct imapc_storage_client *client;
+	struct mailbox_list *index_list;
+
+	/* mailboxes are stored as vnames */
+	struct mailbox_tree_context *mailboxes, *tmp_subscriptions;
+	char root_sep;
+	time_t last_refreshed_mailboxes;
+
+	unsigned int iter_count;
+
+	/* mailboxes/subscriptions are fully refreshed only during
+	   mailbox list iteration. */
+	bool refreshed_subscriptions:1;
+	bool refreshed_mailboxes:1;
+	/* mailbox list's "recently refreshed" state is reset by syncing a
+	   mailbox. mainly we use this to cache mailboxes' existence to avoid
+	   issuing a LIST command every time. */
+	bool refreshed_mailboxes_recently:1;
+	bool index_list_failed:1;
+	bool root_sep_pending:1;
+};
+
+int imapc_list_get_mailbox_flags(struct mailbox_list *list, const char *name,
+				 enum mailbox_info_flags *flags_r);
+int imapc_list_try_get_root_sep(struct imapc_mailbox_list *list, char *sep_r);
+const char *imapc_list_storage_to_remote_name(struct imapc_mailbox_list *list,
+					      const char *storage_name);
+
+#endif
diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
new file mode 100644
index 0000000..1b0eee7
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -0,0 +1,911 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "istream-concat.h"
+#include "istream-header-filter.h"
+#include "message-header-parser.h"
+#include "imap-arg.h"
+#include "imap-util.h"
+#include "imap-date.h"
+#include "imap-quote.h"
+#include "imap-bodystructure.h"
+#include "imap-resp-code.h"
+#include "imapc-mail.h"
+#include "imapc-storage.h"
+
+static void imapc_mail_set_failure(struct imapc_mail *mail,
+				   const struct imapc_command_reply *reply)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
+
+	mail->last_fetch_reply = p_strdup(mail->imail.mail.pool, reply->text_full);
+
+	switch (reply->state) {
+	case IMAPC_COMMAND_STATE_OK:
+		break;
+	case IMAPC_COMMAND_STATE_NO:
+		if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS)) {
+			/* fetch-fix-broken-mails feature disabled -
+			   fail any mails with missing replies */
+			break;
+		}
+		if (reply->resp_text_key != NULL &&
+		    (strcasecmp(reply->resp_text_key, IMAP_RESP_CODE_SERVERBUG) == 0 ||
+		     strcasecmp(reply->resp_text_key, IMAP_RESP_CODE_LIMIT) == 0)) {
+			/* this is a temporary error, retrying should work.
+			   Yahoo sends * BYE +
+			   NO [LIMIT] UID FETCH Rate limit hit. */
+		} else {
+			/* hopefully this is a permanent failure */
+			mail->fetch_ignore_if_missing = TRUE;
+		}
+		break;
+	case IMAPC_COMMAND_STATE_BAD:
+	case IMAPC_COMMAND_STATE_DISCONNECTED:
+	case IMAPC_COMMAND_STATE_AUTH_FAILED:
+		mail->fetch_failed = TRUE;
+		break;
+	}
+}
+
+static void
+imapc_mail_fetch_callback(const struct imapc_command_reply *reply,
+			  void *context)
+{
+	struct imapc_fetch_request *request = context;
+	struct imapc_fetch_request *const *requests;
+	struct imapc_mail *mail;
+	struct imapc_mailbox *mbox = NULL;
+	unsigned int i, count;
+
+	array_foreach_elem(&request->mails, mail) {
+		i_assert(mail->fetch_count > 0);
+		imapc_mail_set_failure(mail, reply);
+		if (--mail->fetch_count == 0)
+			mail->fetching_fields = 0;
+		mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
+	}
+	i_assert(mbox != NULL);
+
+	requests = array_get(&mbox->fetch_requests, &count);
+	for (i = 0; i < count; i++) {
+		if (requests[i] == request) {
+			array_delete(&mbox->fetch_requests, i, 1);
+			break;
+		}
+	}
+	i_assert(i < count);
+
+	array_free(&request->mails);
+	i_free(request);
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		;
+	else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS,
+					    reply);
+	} else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		/* The disconnection message was already logged */
+		mail_storage_set_internal_error(&mbox->storage->storage);
+	} else {
+		mailbox_set_critical(&mbox->box,
+			"imapc: Mail FETCH failed: %s", reply->text_full);
+	}
+	imapc_client_stop(mbox->storage->client->client);
+}
+
+static bool
+headers_have_subset(const char *const *superset, const char *const *subset)
+{
+	unsigned int i;
+
+	if (superset == NULL)
+		return FALSE;
+	if (subset != NULL) {
+		for (i = 0; subset[i] != NULL; i++) {
+			if (!str_array_icase_find(superset, subset[i]))
+				return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+static const char *const *
+headers_merge(pool_t pool, const char *const *h1, const char *const *h2)
+{
+	ARRAY_TYPE(const_string) headers;
+	const char *value;
+	unsigned int i;
+
+	p_array_init(&headers, pool, 16);
+	if (h1 != NULL) {
+		for (i = 0; h1[i] != NULL; i++) {
+			value = p_strdup(pool, h1[i]);
+			array_push_back(&headers, &value);
+		}
+	}
+	if (h2 != NULL) {
+		for (i = 0; h2[i] != NULL; i++) {
+			if (h1 == NULL || !str_array_icase_find(h1, h2[i])) {
+				value = p_strdup(pool, h2[i]);
+				array_push_back(&headers, &value);
+			}
+		}
+	}
+	array_append_zero(&headers);
+	return array_front(&headers);
+}
+
+static bool
+imapc_mail_try_merge_fetch(struct imapc_mailbox *mbox, string_t *str)
+{
+	const char *s1 = str_c(str);
+	const char *s2 = str_c(mbox->pending_fetch_cmd);
+	const char *p1, *p2;
+
+	i_assert(str_begins(s1, "UID FETCH "));
+	i_assert(str_begins(s2, "UID FETCH "));
+
+	/* skip over UID range */
+	p1 = strchr(s1+10, ' ');
+	p2 = strchr(s2+10, ' ');
+
+	if (null_strcmp(p1, p2) != 0)
+		return FALSE;
+	/* append the new UID to the pending FETCH UID range */
+	str_truncate(str, p1-s1);
+	str_insert(mbox->pending_fetch_cmd, p2-s2, ",");
+	str_insert(mbox->pending_fetch_cmd, p2-s2+1, str_c(str) + 10);
+	return TRUE;
+}
+
+static void
+imapc_mail_delayed_send_or_merge(struct imapc_mail *mail, string_t *str)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
+
+	if (mbox->pending_fetch_request != NULL &&
+	    !imapc_mail_try_merge_fetch(mbox, str)) {
+		/* send the previous FETCH and create a new one */
+		imapc_mail_fetch_flush(mbox);
+	}
+	if (mbox->pending_fetch_request == NULL) {
+		mbox->pending_fetch_request =
+			i_new(struct imapc_fetch_request, 1);
+		i_array_init(&mbox->pending_fetch_request->mails, 4);
+		i_assert(mbox->pending_fetch_cmd->used == 0);
+		str_append_str(mbox->pending_fetch_cmd, str);
+	}
+	array_push_back(&mbox->pending_fetch_request->mails, &mail);
+
+	if (mbox->to_pending_fetch_send == NULL &&
+	    array_count(&mbox->pending_fetch_request->mails) >
+	    			mbox->box.storage->set->mail_prefetch_count) {
+		/* we're now prefetching the maximum number of mails. this
+		   most likely means that we need to flush out the command now
+		   before sending anything else. delay it a little bit though
+		   in case the sending code doesn't actually use
+		   mail_prefetch_count and wants to fetch more.
+
+		   note that we don't want to add this timeout too early,
+		   because we want to optimize the maximum number of messages
+		   placed into a single FETCH. even without timeout the command
+		   gets flushed by imapc_mail_fetch() call. */
+		mbox->to_pending_fetch_send =
+			timeout_add_short(0, imapc_mail_fetch_flush, mbox);
+	}
+}
+
+static int
+imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields,
+		      const char *const *headers)
+{
+	struct imapc_mail *mail = IMAPC_MAIL(_mail);
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct mail_index_view *view;
+	string_t *str;
+	uint32_t seq;
+	unsigned int i;
+
+	i_assert(headers == NULL ||
+		 IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS));
+
+	if (!mbox->selected) {
+		mail_storage_set_error(_mail->box->storage,
+				MAIL_ERROR_NOTPOSSIBLE, "Can't fetch mails before selecting mailbox");
+		return -1;
+	}
+
+	if (!mail_stream_access_start(_mail))
+		return -1;
+
+	/* drop any fields that we may already be fetching currently */
+	fields &= ENUM_NEGATE(mail->fetching_fields);
+	if (headers_have_subset(mail->fetching_headers, headers))
+		headers = NULL;
+	if (fields == 0 && headers == NULL)
+		return mail->fetch_sent ? 0 : 1;
+
+	if (!_mail->saving) {
+		/* if we already know that the mail is expunged,
+		   don't try to FETCH it */
+		view = mbox->delayed_sync_view != NULL ?
+			mbox->delayed_sync_view : mbox->box.view;
+		if (!mail_index_lookup_seq(view, _mail->uid, &seq) ||
+		    mail_index_is_expunged(view, seq)) {
+			mail_set_expunged(_mail);
+			return -1;
+		}
+	} else if (mbox->client_box == NULL) {
+		/* opened as save-only. we'll need to fetch the mail,
+		   so actually SELECT/EXAMINE the mailbox */
+		i_assert(mbox->box.opened);
+
+		if (imapc_mailbox_select(mbox) < 0)
+			return -1;
+	}
+
+	if ((fields & MAIL_FETCH_STREAM_BODY) != 0)
+		fields |= MAIL_FETCH_STREAM_HEADER;
+
+	str = t_str_new(64);
+	str_printfa(str, "UID FETCH %u (", _mail->uid);
+	if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0)
+		str_append(str, "INTERNALDATE ");
+	if ((fields & MAIL_FETCH_SAVE_DATE) != 0) {
+		i_assert(HAS_ALL_BITS(mbox->capabilities,
+				      IMAPC_CAPABILITY_SAVEDATE));
+		str_append(str, "SAVEDATE ");
+	}
+	if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0)
+		str_append(str, "RFC822.SIZE ");
+	if ((fields & MAIL_FETCH_GUID) != 0) {
+		str_append(str, mbox->guid_fetch_field_name);
+		str_append_c(str, ' ');
+	}
+	if ((fields & MAIL_FETCH_IMAP_BODY) != 0)
+		str_append(str, "BODY ");
+	if ((fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0)
+		str_append(str, "BODYSTRUCTURE ");
+
+	if ((fields & MAIL_FETCH_STREAM_BODY) != 0) {
+		if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_ZIMBRA_WORKAROUNDS))
+			str_append(str, "BODY.PEEK[] ");
+		else {
+			/* BODY.PEEK[] can return different headers than
+			   BODY.PEEK[HEADER] (e.g. invalid 8bit chars replaced
+			   with '?' in HEADER) - this violates IMAP protocol
+			   and messes up dsync since it sometimes fetches the
+			   full body and sometimes only the headers. */
+			str_append(str, "BODY.PEEK[HEADER] BODY.PEEK[TEXT] ");
+		}
+		fields |= MAIL_FETCH_STREAM_HEADER;
+	} else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0)
+		str_append(str, "BODY.PEEK[HEADER] ");
+	else if (headers != NULL) {
+		mail->fetching_headers =
+			headers_merge(mail->imail.mail.data_pool, headers,
+				      mail->fetching_headers);
+		str_append(str, "BODY.PEEK[HEADER.FIELDS (");
+		for (i = 0; mail->fetching_headers[i] != NULL; i++) {
+			if (i > 0)
+				str_append_c(str, ' ');
+			imap_append_astring(str, mail->fetching_headers[i]);
+		}
+		str_append(str, ")] ");
+		mail->header_list_fetched = FALSE;
+	}
+	str_truncate(str, str_len(str)-1);
+	str_append_c(str, ')');
+
+	mail->fetching_fields |= fields;
+	mail->fetch_count++;
+	mail->fetch_sent = FALSE;
+	mail->fetch_failed = FALSE;
+
+	imapc_mail_delayed_send_or_merge(mail, str);
+	return 1;
+}
+
+static void imapc_mail_cache_get(struct imapc_mail *mail,
+				 struct imapc_mail_cache *cache)
+{
+	if (mail->body_fetched)
+		return;
+
+	if (cache->fd != -1) {
+		mail->fd = cache->fd;
+		mail->imail.data.stream = i_stream_create_fd(mail->fd, 0);
+		cache->fd = -1;
+	} else if (cache->buf != NULL) {
+		mail->body = cache->buf;
+		mail->imail.data.stream =
+			i_stream_create_from_data(mail->body->data,
+						  mail->body->used);
+		cache->buf = NULL;
+	} else {
+		return;
+	}
+	mail->header_fetched = TRUE;
+	mail->body_fetched = TRUE;
+	/* The stream was already accessed and now it's cached.
+	   It still needs to be set accessed to avoid assert-crash. */
+	mail->imail.mail.mail.mail_stream_accessed = TRUE;
+	imapc_mail_init_stream(mail);
+}
+
+static enum mail_fetch_field
+imapc_mail_get_wanted_fetch_fields(struct imapc_mail *mail)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
+	struct index_mail_data *data = &mail->imail.data;
+	enum mail_fetch_field fields = 0;
+
+	if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 &&
+	    data->received_date == (time_t)-1)
+		fields |= MAIL_FETCH_RECEIVED_DATE;
+	if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0 &&
+	    data->save_date == (time_t)-1) {
+		if (HAS_ALL_BITS(mbox->capabilities, IMAPC_CAPABILITY_SAVEDATE))
+			fields |= MAIL_FETCH_SAVE_DATE;
+		else
+			fields |= MAIL_FETCH_RECEIVED_DATE;
+	}
+	if ((data->wanted_fields & (MAIL_FETCH_PHYSICAL_SIZE |
+				    MAIL_FETCH_VIRTUAL_SIZE)) != 0 &&
+	    data->physical_size == UOFF_T_MAX &&
+	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE))
+		fields |= MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE;
+	if ((data->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0 &&
+	    data->body == NULL &&
+	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE))
+		fields |= MAIL_FETCH_IMAP_BODY;
+	if ((data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0 &&
+	    data->bodystructure == NULL &&
+	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE))
+		fields |= MAIL_FETCH_IMAP_BODYSTRUCTURE;
+	if ((data->wanted_fields & MAIL_FETCH_GUID) != 0 &&
+	    data->guid == NULL && mbox->guid_fetch_field_name != NULL)
+		fields |= MAIL_FETCH_GUID;
+
+	if (data->stream == NULL && data->access_part != 0) {
+		if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0)
+			fields |= MAIL_FETCH_STREAM_BODY;
+		fields |= MAIL_FETCH_STREAM_HEADER;
+	}
+	return fields;
+}
+
+void imapc_mail_try_init_stream_from_cache(struct imapc_mail *mail)
+{
+	struct mail *_mail = &mail->imail.mail.mail;
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+
+	if (mbox->prev_mail_cache.uid == _mail->uid)
+		imapc_mail_cache_get(mail, &mbox->prev_mail_cache);
+}
+
+bool imapc_mail_prefetch(struct mail *_mail)
+{
+	struct imapc_mail *mail = IMAPC_MAIL(_mail);
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct index_mail_data *data = &mail->imail.data;
+	enum mail_fetch_field fields;
+	const char *const *headers = NULL;
+
+	/* try to get as much from cache as possible */
+	imapc_mail_update_access_parts(&mail->imail);
+	/* If mail is already cached we can avoid re-FETCHing the mail.
+	   However, don't initialize the stream if we don't actually want to
+	   access the mail. */
+	if (mail->imail.data.access_part != 0)
+		imapc_mail_try_init_stream_from_cache(mail);
+
+	fields = imapc_mail_get_wanted_fetch_fields(mail);
+	if (data->wanted_headers != NULL && data->stream == NULL &&
+	    (fields & MAIL_FETCH_STREAM_HEADER) == 0 &&
+	    !imapc_mail_has_headers_in_cache(&mail->imail, data->wanted_headers)) {
+		/* fetch specific headers */
+		if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS))
+			headers = data->wanted_headers->name;
+		else
+			fields |= MAIL_FETCH_STREAM_HEADER;
+	}
+	if (fields != 0 || headers != NULL) T_BEGIN {
+		if (imapc_mail_send_fetch(_mail, fields, headers) > 0)
+			mail->imail.data.prefetch_sent = TRUE;
+	} T_END;
+	return !mail->imail.data.prefetch_sent;
+}
+
+static bool
+imapc_mail_have_fields(struct imapc_mail *imail, enum mail_fetch_field fields)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(imail->imail.mail.mail.box);
+
+	if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) {
+		if (imail->imail.data.received_date == (time_t)-1)
+			return FALSE;
+		fields &= ENUM_NEGATE(MAIL_FETCH_RECEIVED_DATE);
+	}
+	if ((fields & MAIL_FETCH_SAVE_DATE) != 0) {
+		i_assert(HAS_ALL_BITS(mbox->capabilities,
+				      IMAPC_CAPABILITY_SAVEDATE));
+		if (imail->imail.data.save_date == (time_t)-1)
+			return FALSE;
+		fields &= ENUM_NEGATE(MAIL_FETCH_SAVE_DATE);
+	}
+	if ((fields & (MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE)) != 0) {
+		if (imail->imail.data.physical_size == UOFF_T_MAX)
+			return FALSE;
+		fields &= ENUM_NEGATE(MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE);
+	}
+	if ((fields & MAIL_FETCH_GUID) != 0) {
+		if (imail->imail.data.guid == NULL)
+			return FALSE;
+		fields &= ENUM_NEGATE(MAIL_FETCH_GUID);
+	}
+	if ((fields & MAIL_FETCH_IMAP_BODY) != 0) {
+		if (imail->imail.data.body == NULL)
+			return FALSE;
+		fields &= ENUM_NEGATE(MAIL_FETCH_IMAP_BODY);
+	}
+	if ((fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0) {
+		if (imail->imail.data.bodystructure == NULL)
+			return FALSE;
+		fields &= ENUM_NEGATE(MAIL_FETCH_IMAP_BODYSTRUCTURE);
+	}
+	if ((fields & (MAIL_FETCH_STREAM_HEADER |
+		       MAIL_FETCH_STREAM_BODY)) != 0) {
+		if (imail->imail.data.stream == NULL)
+			return FALSE;
+		fields &= ENUM_NEGATE(MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY);
+	}
+	i_assert(fields == 0);
+	return TRUE;
+}
+
+int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields,
+		     const char *const *headers)
+{
+	struct imapc_mail *imail = IMAPC_MAIL(_mail);
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	int ret;
+
+	if ((fields & MAIL_FETCH_GUID) != 0 &&
+	    mbox->guid_fetch_field_name == NULL) {
+		mail_storage_set_error(_mail->box->storage,
+			MAIL_ERROR_NOTPOSSIBLE,
+			"Message GUID not available in this server");
+		return -1;
+	}
+	if (_mail->saving) {
+		mail_storage_set_error(_mail->box->storage,
+			MAIL_ERROR_NOTPOSSIBLE,
+			"Attempting to issue FETCH for a mail not yet committed");
+		return -1;
+	}
+
+	fields |= imapc_mail_get_wanted_fetch_fields(imail);
+	T_BEGIN {
+		ret = imapc_mail_send_fetch(_mail, fields, headers);
+	} T_END;
+	if (ret < 0)
+		return -1;
+
+	/* we'll continue waiting until we've got all the fields we wanted,
+	   or until all FETCH replies have been received (i.e. some FETCHes
+	   failed) */
+	if (ret > 0)
+		imapc_mail_fetch_flush(mbox);
+	while (imail->fetch_count > 0 &&
+	       (!imapc_mail_have_fields(imail, fields) ||
+		!imail->header_list_fetched)) {
+		imapc_mailbox_run_nofetch(mbox);
+	}
+	if (imail->fetch_failed) {
+		mail_storage_set_internal_error(&mbox->storage->storage);
+		return -1;
+	}
+	return 0;
+}
+
+void imapc_mail_fetch_flush(struct imapc_mailbox *mbox)
+{
+	struct imapc_command *cmd;
+	struct imapc_mail *mail;
+
+	if (mbox->pending_fetch_request == NULL) {
+		i_assert(mbox->to_pending_fetch_send == NULL);
+		return;
+	}
+
+	array_foreach_elem(&mbox->pending_fetch_request->mails, mail)
+		mail->fetch_sent = TRUE;
+
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_mail_fetch_callback,
+				       mbox->pending_fetch_request);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	array_push_back(&mbox->fetch_requests, &mbox->pending_fetch_request);
+
+	imapc_command_send(cmd, str_c(mbox->pending_fetch_cmd));
+
+	mbox->pending_fetch_request = NULL;
+	timeout_remove(&mbox->to_pending_fetch_send);
+	str_truncate(mbox->pending_fetch_cmd, 0);
+}
+
+static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply,
+				 const struct imap_arg *arg, int *fd_r)
+{
+	const struct imap_arg *list;
+	unsigned int i, count;
+
+	for (i = 0; i < reply->file_args_count; i++) {
+		const struct imapc_arg_file *farg = &reply->file_args[i];
+
+		if (farg->parent_arg == arg->parent &&
+		    imap_arg_get_list_full(arg->parent, &list, &count) &&
+		    farg->list_idx < count && &list[farg->list_idx] == arg) {
+			*fd_r = farg->fd;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static void imapc_stream_filter(struct istream **input)
+{
+	static const char *imapc_hide_headers[] = {
+		/* Added by MS Exchange 2010 when \Flagged flag is set.
+		   This violates IMAP guarantee of messages being immutable. */
+		"X-Message-Flag"
+	};
+	struct istream *filter_input;
+
+	filter_input = i_stream_create_header_filter(*input,
+		HEADER_FILTER_EXCLUDE,
+		imapc_hide_headers, N_ELEMENTS(imapc_hide_headers),
+		*null_header_filter_callback, NULL);
+	i_stream_unref(input);
+	*input = filter_input;
+}
+
+void imapc_mail_init_stream(struct imapc_mail *mail)
+{
+	struct index_mail *imail = &mail->imail;
+	struct mail *_mail = &imail->mail.mail;
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct istream *input;
+	uoff_t size;
+	int ret;
+
+	i_stream_set_name(imail->data.stream,
+			  t_strdup_printf("imapc mail uid=%u", _mail->uid));
+	index_mail_set_read_buffer_size(_mail, imail->data.stream);
+
+	if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
+		/* enable filtering only when we're not passing through
+		   RFC822.SIZE. otherwise we'll get size mismatches. */
+		imapc_stream_filter(&imail->data.stream);
+	}
+	if (imail->mail.v.istream_opened != NULL) {
+		if (imail->mail.v.istream_opened(_mail,
+						 &imail->data.stream) < 0) {
+			index_mail_close_streams(imail);
+			return;
+		}
+	}
+	ret = i_stream_get_size(imail->data.stream, TRUE, &size);
+	if (ret < 0) {
+		index_mail_close_streams(imail);
+		return;
+	}
+	i_assert(ret != 0);
+	/* Once message body is fetched, we can be sure of what its size is.
+	   If we had already received RFC822.SIZE, overwrite it here in case
+	   it's wrong. Also in more special cases the RFC822.SIZE may be
+	   smaller than the fetched message header. In this case change the
+	   size as well, otherwise reading via istream-mail will fail. */
+	if (mail->body_fetched || imail->data.physical_size < size) {
+		if (mail->body_fetched) {
+			imail->data.inexact_total_sizes = FALSE;
+			/* Don't trust any existing virtual_size. Also don't
+			   set it to size, because there's no guarantees about
+			   the content having proper CRLF newlines, especially
+			   not if istream_opened() has changed the stream. */
+			imail->data.virtual_size = UOFF_T_MAX;
+		}
+		imail->data.physical_size = size;
+	}
+
+	imail->data.stream_has_only_header = !mail->body_fetched;
+	if (index_mail_init_stream(imail, NULL, NULL, &input) < 0)
+		index_mail_close_streams(imail);
+}
+
+static void
+imapc_fetch_stream(struct imapc_mail *mail,
+		   const struct imapc_untagged_reply *reply,
+		   const struct imap_arg *arg,
+		   bool have_header, bool have_body)
+{
+	struct index_mail *imail = &mail->imail;
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(imail->mail.mail.box);
+	struct istream *hdr_stream = NULL;
+	const char *value;
+	int fd;
+
+	if (imail->data.stream != NULL) {
+		i_assert(mail->header_fetched);
+		if (mail->body_fetched || !have_body)
+			return;
+		if (have_header) {
+			/* replace the existing stream */
+		} else if (mail->fd == -1) {
+			/* append this body stream to the existing
+			   header stream */
+			hdr_stream = imail->data.stream;
+			i_stream_ref(hdr_stream);
+		} else {
+			/* append this body stream to the existing
+			   header stream. we'll need to recreate the stream
+			   with autoclosed fd. */
+			if (lseek(mail->fd, 0, SEEK_SET) < 0)
+				i_error("lseek(imapc) failed: %m");
+			hdr_stream = i_stream_create_fd_autoclose(&mail->fd, 0);
+		}
+		index_mail_close_streams(imail);
+		i_close_fd(&mail->fd);
+	} else {
+		if (!have_header) {
+			/* BODY.PEEK[TEXT] received - we can't currently handle
+			   this before receiving BODY.PEEK[HEADER] reply */
+			return;
+		}
+	}
+
+	if (arg->type == IMAP_ARG_LITERAL_SIZE) {
+		if (!imapc_find_lfile_arg(reply, arg, &fd)) {
+			i_stream_unref(&hdr_stream);
+			return;
+		}
+		if ((fd = dup(fd)) == -1) {
+			i_error("dup() failed: %m");
+			i_stream_unref(&hdr_stream);
+			return;
+		}
+		mail->fd = fd;
+		imail->data.stream = i_stream_create_fd(fd, 0);
+	} else {
+		if (!imap_arg_get_nstring(arg, &value))
+			value = NULL;
+		if (value == NULL ||
+		    (value[0] == '\0' &&
+		     IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_EMPTY_IS_EXPUNGED))) {
+			mail_set_expunged(&imail->mail.mail);
+			i_stream_unref(&hdr_stream);
+			return;
+		}
+		if (mail->body == NULL) {
+			mail->body = buffer_create_dynamic(default_pool,
+							   arg->str_len + 1);
+		} else if (!have_header && hdr_stream != NULL) {
+			/* header is already in the buffer - add body now
+			   without destroying the existing header data */
+			i_stream_unref(&hdr_stream);
+		} else {
+			buffer_set_used_size(mail->body, 0);
+		}
+		buffer_append(mail->body, value, arg->str_len);
+		imail->data.stream = i_stream_create_from_data(mail->body->data,
+							       mail->body->used);
+	}
+	if (have_header)
+		mail->header_fetched = TRUE;
+	mail->body_fetched = have_body;
+
+	if (hdr_stream != NULL) {
+		struct istream *inputs[3];
+
+		inputs[0] = hdr_stream;
+		inputs[1] = imail->data.stream;
+		inputs[2] = NULL;
+		imail->data.stream = i_stream_create_concat(inputs);
+		i_stream_unref(&inputs[0]);
+		i_stream_unref(&inputs[1]);
+	}
+
+	imapc_mail_init_stream(mail);
+}
+
+static void
+imapc_fetch_header_stream(struct imapc_mail *mail,
+			  const struct imapc_untagged_reply *reply,
+			  const struct imap_arg *args)
+{
+	const enum message_header_parser_flags hdr_parser_flags =
+		MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP |
+		MESSAGE_HEADER_PARSER_FLAG_DROP_CR;
+	const struct imap_arg *hdr_list;
+	struct mailbox_header_lookup_ctx *headers_ctx;
+	struct message_header_parser_ctx *parser;
+	struct message_header_line *hdr;
+	struct istream *input;
+	ARRAY_TYPE(const_string) hdr_arr;
+	const char *value;
+	int ret, fd;
+
+	if (!imap_arg_get_list(args, &hdr_list))
+		return;
+	if (!imap_arg_atom_equals(args+1, "]"))
+		return;
+	args += 2;
+
+	/* see if this is reply to the latest headers list request
+	   (parse it even if it's not) */
+	t_array_init(&hdr_arr, 16);
+	while (imap_arg_get_astring(hdr_list, &value)) {
+		array_push_back(&hdr_arr, &value);
+		hdr_list++;
+	}
+	if (hdr_list->type != IMAP_ARG_EOL)
+		return;
+	array_append_zero(&hdr_arr);
+
+	if (headers_have_subset(array_front(&hdr_arr), mail->fetching_headers))
+		mail->header_list_fetched = TRUE;
+
+	if (args->type == IMAP_ARG_LITERAL_SIZE) {
+		if (!imapc_find_lfile_arg(reply, args, &fd))
+			return;
+		input = i_stream_create_fd(fd, 0);
+	} else {
+		if (!imap_arg_get_nstring(args, &value))
+			return;
+		if (value == NULL) {
+			mail_set_expunged(&mail->imail.mail.mail);
+			return;
+		}
+		input = i_stream_create_from_data(value, args->str_len);
+	}
+
+	headers_ctx = mailbox_header_lookup_init(mail->imail.mail.mail.box,
+						 array_front(&hdr_arr));
+	index_mail_parse_header_init(&mail->imail, headers_ctx);
+
+	parser = message_parse_header_init(input, NULL, hdr_parser_flags);
+	while ((ret = message_parse_header_next(parser, &hdr)) > 0)
+		index_mail_parse_header(NULL, hdr, &mail->imail);
+	i_assert(ret != 0);
+	index_mail_parse_header(NULL, NULL, &mail->imail);
+	message_parse_header_deinit(&parser);
+
+	mailbox_header_lookup_unref(&headers_ctx);
+	i_stream_destroy(&input);
+}
+
+static const char *
+imapc_args_to_bodystructure(struct imapc_mail *mail,
+			    const struct imap_arg *list_arg, bool extended)
+{
+	const struct imap_arg *args;
+	struct message_part *parts = NULL;
+	const char *ret, *error;
+	pool_t pool;
+
+	if (!imap_arg_get_list(list_arg, &args)) {
+		mail_set_critical(&mail->imail.mail.mail,
+			"imapc: Server sent invalid BODYSTRUCTURE parameters");
+		return NULL;
+	}
+
+	pool = pool_alloconly_create("imap bodystructure", 1024);
+	if (imap_bodystructure_parse_args(args, pool, &parts, &error) < 0) {
+		mail_set_critical(&mail->imail.mail.mail,
+			"imapc: Server sent invalid BODYSTRUCTURE: %s", error);
+		ret = NULL;
+	} else {
+		string_t *str = t_str_new(128);
+		if (imap_bodystructure_write(parts, str, extended, &error) < 0) {
+			/* All the input to imap_bodystructure_write() came
+			   from imap_bodystructure_parse_args(). We should never
+			   get here. Instead, if something is wrong the
+			   parsing should have returned an error already. */
+			str_truncate(str, 0);
+			imap_write_args(str, args);
+			i_panic("Failed to write parsed BODYSTRUCTURE: %s "
+				"(original string: '%s')", error, str_c(str));
+		}
+		ret = p_strdup(mail->imail.mail.data_pool, str_c(str));
+	}
+	pool_unref(&pool);
+	return ret;
+}
+
+void imapc_mail_fetch_update(struct imapc_mail *mail,
+			     const struct imapc_untagged_reply *reply,
+			     const struct imap_arg *args)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->imail.mail.mail.box);
+	const char *key, *value;
+	unsigned int i;
+	uoff_t size;
+	time_t t;
+	int tz;
+	bool match = FALSE;
+
+	for (i = 0; args[i].type != IMAP_ARG_EOL; i += 2) {
+		if (!imap_arg_get_atom(&args[i], &key) ||
+		    args[i+1].type == IMAP_ARG_EOL)
+			break;
+
+		if (strcasecmp(key, "BODY[]") == 0) {
+			imapc_fetch_stream(mail, reply, &args[i+1], TRUE, TRUE);
+			match = TRUE;
+		} else if (strcasecmp(key, "BODY[HEADER]") == 0) {
+			imapc_fetch_stream(mail, reply, &args[i+1], TRUE, FALSE);
+			match = TRUE;
+		} else if (strcasecmp(key, "BODY[TEXT]") == 0) {
+			imapc_fetch_stream(mail, reply, &args[i+1], FALSE, TRUE);
+			match = TRUE;
+		} else if (strcasecmp(key, "BODY[HEADER.FIELDS") == 0) {
+			imapc_fetch_header_stream(mail, reply, &args[i+1]);
+			match = TRUE;
+		} else if (strcasecmp(key, "INTERNALDATE") == 0) {
+			if (imap_arg_get_astring(&args[i+1], &value) &&
+			    imap_parse_datetime(value, &t, &tz)) {
+				mail->imail.data.received_date = t;
+				if (HAS_NO_BITS(mbox->capabilities,
+						 IMAPC_CAPABILITY_SAVEDATE))
+					mail->imail.data.save_date = t;
+			}
+			match = TRUE;
+		} else if (strcasecmp(key, "SAVEDATE") == 0) {
+			if (imap_arg_get_astring(&args[i+1], &value)) {
+				if (strcasecmp(value, "NIL") == 0)
+					mail->imail.data.save_date = 0;
+				else if (imap_parse_datetime(value, &t, &tz))
+					mail->imail.data.save_date = t;
+			}
+			match = TRUE;
+		} else if (strcasecmp(key, "BODY") == 0) {
+			if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) {
+				mail->imail.data.body =
+					imapc_args_to_bodystructure(mail, &args[i+1], FALSE);
+			}
+			match = TRUE;
+		} else if (strcasecmp(key, "BODYSTRUCTURE") == 0) {
+			if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) {
+				mail->imail.data.bodystructure =
+					imapc_args_to_bodystructure(mail, &args[i+1], TRUE);
+			}
+			match = TRUE;
+		} else if (strcasecmp(key, "RFC822.SIZE") == 0) {
+			if (imap_arg_get_atom(&args[i+1], &value) &&
+			    str_to_uoff(value, &size) == 0 &&
+			    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
+				mail->imail.data.physical_size = size;
+				mail->imail.data.virtual_size = size;
+				mail->imail.data.inexact_total_sizes = TRUE;
+			}
+			match = TRUE;
+		} else if (strcasecmp(key, "X-GM-MSGID") == 0 ||
+			   strcasecmp(key, "X-GUID") == 0) {
+			if (imap_arg_get_astring(&args[i+1], &value)) {
+				mail->imail.data.guid =
+					p_strdup(mail->imail.mail.pool, value);
+			}
+			match = TRUE;
+		}
+	}
+	if (!match) {
+		/* this is only a FETCH FLAGS update for the wanted mail */
+	} else {
+		imapc_client_stop(mbox->storage->client->client);
+	}
+}
diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
new file mode 100644
index 0000000..1ecf03e
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -0,0 +1,675 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "sha1.h"
+#include "istream.h"
+#include "message-part-data.h"
+#include "imap-envelope.h"
+#include "imapc-msgmap.h"
+#include "imapc-mail.h"
+#include "imapc-storage.h"
+
+static bool imapc_mail_get_cached_guid(struct mail *_mail);
+
+struct mail *
+imapc_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers)
+{
+	struct imapc_mail *mail;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mail", 2048);
+	mail = p_new(pool, struct imapc_mail, 1);
+	mail->fd = -1;
+
+	index_mail_init(&mail->imail, t, wanted_fields, wanted_headers, pool, NULL);
+	return &mail->imail.mail.mail;
+}
+
+static bool imapc_mail_is_expunged(struct mail *_mail)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct imapc_msgmap *msgmap;
+	uint32_t lseq, rseq;
+
+	if (!mbox->initial_sync_done) {
+		/* unknown at this point */
+		return FALSE;
+	}
+
+	if (mbox->sync_view != NULL) {
+		/* check if another session has already expunged it */
+		if (!mail_index_lookup_seq(mbox->sync_view, _mail->uid, &lseq))
+			return TRUE;
+	}
+
+	/* check if we've received EXPUNGE for it */
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	if (!imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq))
+		return TRUE;
+
+	/* we may be running against a server that hasn't bothered sending
+	   us an EXPUNGE. see if NOOP sends it. */
+	imapc_mailbox_noop(mbox);
+	if (!mbox->initial_sync_done) {
+		/* NOOP caused a reconnection and desync */
+		return FALSE;
+	}
+
+	return !imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq);
+}
+
+static int imapc_mail_failed(struct mail *mail, const char *field)
+{
+	struct imapc_mail *imail = IMAPC_MAIL(mail);
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(mail->box);
+	bool fix_broken_mail = FALSE;
+
+	if (mail->expunged || imapc_mail_is_expunged(mail)) {
+		mail_set_expunged(mail);
+	} else if (!imapc_client_mailbox_is_opened(mbox->client_box)) {
+		/* we've already logged a disconnection error */
+		mail_storage_set_internal_error(mail->box->storage);
+	} else {
+		/* By default we'll assume that this is a critical failure,
+		   because we don't want to lose any data. We can be here
+		   either because it's a temporary failure on the server or
+		   it's a permanent failure. Unfortunately we can't know
+		   which case it is, so permanent failures need to be worked
+		   around by setting imapc_features=fetch-fix-broken-mails.
+
+		   One reason for permanent failures was that earlier Exchange
+		   versions failed to return any data for messages in Calendars
+		   mailbox. This seems to be fixed in newer versions.
+		   */
+		fix_broken_mail = imail->fetch_ignore_if_missing;
+		mail_set_critical(mail,
+			"imapc: Remote server didn't send %s%s (FETCH replied: %s)",
+			field, fix_broken_mail ? " - treating it as empty" : "",
+			imail->last_fetch_reply);
+	}
+	return fix_broken_mail ? 0 : -1;
+}
+
+static uint64_t imapc_mail_get_modseq(struct mail *_mail)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct imapc_msgmap *msgmap;
+	const uint64_t *modseqs;
+	unsigned int count;
+	uint32_t rseq;
+
+	if (!imapc_mailbox_has_modseqs(mbox))
+		return index_mail_get_modseq(_mail);
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	if (imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) {
+		modseqs = array_get(&mbox->rseq_modseqs, &count);
+		if (rseq <= count)
+			return modseqs[rseq-1];
+	}
+	return 1; /* unknown modseq */
+}
+
+static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	if (index_mail_get_received_date(_mail, date_r) == 0)
+		return 0;
+
+	if (data->received_date == (time_t)-1) {
+		if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE, NULL) < 0)
+			return -1;
+		if (data->received_date == (time_t)-1) {
+			if (imapc_mail_failed(_mail, "INTERNALDATE") < 0)
+				return -1;
+			/* assume that the server never returns INTERNALDATE
+			   for this mail (see BODY[] failure handling) */
+			data->received_date = 0;
+		}
+	}
+	*date_r = data->received_date;
+	return 0;
+}
+
+static int imapc_mail_get_save_date(struct mail *_mail, time_t *date_r)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	if (data->save_date != 0 && index_mail_get_save_date(_mail, date_r) > 0)
+		return 1;
+
+	if (HAS_NO_BITS(mbox->capabilities, IMAPC_CAPABILITY_SAVEDATE)) {
+		data->save_date = 0;
+	} else if (data->save_date == (time_t)-1) {
+		if (imapc_mail_fetch(_mail, MAIL_FETCH_SAVE_DATE, NULL) < 0)
+			return -1;
+		if (data->save_date == (time_t)-1 &&
+		    imapc_mail_failed(_mail, "SAVEDATE") < 0)
+			return -1;
+	}
+	if (data->save_date == (time_t)-1 || data->save_date == 0) {
+		if (imapc_mail_get_received_date(_mail, date_r) < 0)
+			return -1;
+		return 0;
+	}
+	*date_r = data->save_date;
+	return 1;
+}
+
+static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct istream *input;
+	uoff_t old_offset;
+	int ret;
+
+	if (data->physical_size == UOFF_T_MAX)
+		(void)index_mail_get_physical_size(_mail, size_r);
+	if (data->physical_size != UOFF_T_MAX) {
+		*size_r = data->physical_size;
+		return 0;
+	}
+
+	if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) &&
+	    data->stream == NULL) {
+		/* Trust RFC822.SIZE to be correct enough to present to the
+		   IMAP client. However, it can be wrong in some implementation
+		   so try not to trust it too much. */
+		if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0)
+			return -1;
+		if (data->physical_size == UOFF_T_MAX) {
+			if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0)
+				return -1;
+			/* assume that the server never returns RFC822.SIZE
+			   for this mail (see BODY[] failure handling) */
+			data->physical_size = 0;
+		}
+		*size_r = data->physical_size;
+		return 0;
+	}
+
+	old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
+	if (mail_get_stream(_mail, NULL, NULL, &input) < 0)
+		return -1;
+	i_assert(data->stream != NULL);
+	i_stream_seek(data->stream, old_offset);
+
+	ret = i_stream_get_size(data->stream, TRUE,
+				&data->physical_size);
+	if (ret <= 0) {
+		i_assert(ret != 0);
+		mail_set_critical(_mail, "imapc: stat(%s) failed: %m",
+				  i_stream_get_name(data->stream));
+		return -1;
+	}
+	*size_r = data->physical_size;
+	return 0;
+}
+
+static int imapc_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	if (imapc_mail_get_physical_size(_mail, size_r) < 0)
+		return -1;
+	data->virtual_size = data->physical_size;
+	return 0;
+}
+
+static int
+imapc_mail_get_header_stream(struct mail *_mail,
+			     struct mailbox_header_lookup_ctx *headers,
+			     struct istream **stream_r)
+{
+	struct imapc_mail *mail = IMAPC_MAIL(_mail);
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	enum mail_lookup_abort old_abort = _mail->lookup_abort;
+	int ret;
+
+	if (mail->imail.data.access_part != 0 ||
+	    !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) {
+		/* we're going to be reading the header/body anyway */
+		return index_mail_get_header_stream(_mail, headers, stream_r);
+	}
+
+	/* see if the wanted headers are already in cache */
+	_mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
+	ret = index_mail_get_header_stream(_mail, headers, stream_r);
+	_mail->lookup_abort = old_abort;
+	if (ret == 0)
+		return 0;
+
+	/* fetch only the wanted headers */
+	if (imapc_mail_fetch(_mail, 0, headers->name) < 0)
+		return -1;
+	/* the headers should cached now. */
+	return index_mail_get_header_stream(_mail, headers, stream_r);
+}
+
+static int
+imapc_mail_get_headers(struct mail *_mail, const char *field,
+		       bool decode_to_utf8, const char *const **value_r)
+{
+	struct mailbox_header_lookup_ctx *headers;
+	const char *header_names[2];
+	const unsigned char *data;
+	size_t size;
+	struct istream *input;
+	int ret;
+
+	header_names[0] = field;
+	header_names[1] = NULL;
+	headers = mailbox_header_lookup_init(_mail->box, header_names);
+	ret = mail_get_header_stream(_mail, headers, &input);
+	mailbox_header_lookup_unref(&headers);
+	if (ret < 0)
+		return -1;
+
+	while (i_stream_read_more(input, &data, &size) > 0)
+		i_stream_skip(input, size);
+	/* the header should cached now. */
+	return index_mail_get_headers(_mail, field, decode_to_utf8, value_r);
+}
+
+static int
+imapc_mail_get_first_header(struct mail *_mail, const char *field,
+			    bool decode_to_utf8, const char **value_r)
+{
+	const char *const *values;
+	int ret;
+
+	ret = imapc_mail_get_headers(_mail, field, decode_to_utf8, &values);
+	if (ret <= 0)
+		return ret;
+	*value_r = values[0];
+	return 1;
+}
+
+static int
+imapc_mail_get_stream(struct mail *_mail, bool get_body,
+		      struct message_size *hdr_size,
+		      struct message_size *body_size, struct istream **stream_r)
+{
+	struct imapc_mail *mail = IMAPC_MAIL(_mail);
+	struct index_mail_data *data = &mail->imail.data;
+	enum mail_fetch_field fetch_field;
+
+	if (get_body && !mail->body_fetched &&
+	    mail->imail.data.stream != NULL) {
+		/* we've fetched the header, but we need the body now too */
+		index_mail_close_streams(&mail->imail);
+		/* don't re-use any cached header sizes. we may be
+		   intentionally downloading the full body because the header
+		   wasn't returned correctly (e.g. pop3-migration does this) */
+		data->hdr_size_set = FALSE;
+	}
+
+	/* See if we can get it from cache. If the wanted_fields/headers are
+	   set properly, this is usually already done by prefetching. */
+	imapc_mail_try_init_stream_from_cache(mail);
+
+	if (data->stream == NULL) {
+		if (!data->initialized) {
+			/* coming here from mail_set_seq() */
+			mail_set_aborted(_mail);
+			return -1;
+		}
+		if (_mail->expunged) {
+			/* We already detected that the mail is expunged.
+			   Don't spend time trying to FETCH it again. */
+			mail_set_expunged(_mail);
+			return -1;
+		}
+		fetch_field = get_body ||
+			(data->access_part & READ_BODY) != 0 ?
+			MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER;
+		if (imapc_mail_fetch(_mail, fetch_field, NULL) < 0)
+			return -1;
+
+		if (data->stream == NULL) {
+			if (imapc_mail_failed(_mail, "BODY[]") < 0)
+				return -1;
+			i_assert(data->stream == NULL);
+
+			/* return the broken email as empty */
+			mail->body_fetched = TRUE;
+			data->stream = i_stream_create_from_data(NULL, 0);
+			imapc_mail_init_stream(mail);
+		}
+	}
+
+	return index_mail_init_stream(&mail->imail, hdr_size, body_size,
+				      stream_r);
+}
+
+bool imapc_mail_has_headers_in_cache(struct index_mail *mail,
+				     struct mailbox_header_lookup_ctx *headers)
+{
+	struct mail *_mail = &mail->mail.mail;
+	unsigned int i;
+
+	for (i = 0; i < headers->count; i++) {
+		if (mail_cache_field_exists(_mail->transaction->cache_view,
+					    _mail->seq, headers->idx[i]) <= 0)
+			return FALSE;
+	}
+	return TRUE;
+}
+
+void imapc_mail_update_access_parts(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct index_mail_data *data = &mail->data;
+	struct mailbox_header_lookup_ctx *header_ctx;
+	const char *str;
+	time_t date;
+	uoff_t size;
+
+	if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0)
+		(void)index_mail_get_received_date(_mail, &date);
+	if ((data->wanted_fields & MAIL_FETCH_SAVE_DATE) != 0) {
+		if (index_mail_get_save_date(_mail, &date) < 0 &&
+		    HAS_NO_BITS(mbox->capabilities,
+				IMAPC_CAPABILITY_SAVEDATE)) {
+			(void)index_mail_get_received_date(_mail, &date);
+			data->save_date = data->received_date;
+		}
+	}
+	if ((data->wanted_fields & (MAIL_FETCH_PHYSICAL_SIZE |
+				    MAIL_FETCH_VIRTUAL_SIZE)) != 0) {
+		if (index_mail_get_physical_size(_mail, &size) < 0 &&
+		    !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE))
+			data->access_part |= READ_HDR | READ_BODY;
+	}
+	if ((data->wanted_fields & MAIL_FETCH_GUID) != 0)
+		(void)imapc_mail_get_cached_guid(_mail);
+	if ((data->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0)
+		(void)index_mail_get_cached_body(mail, &str);
+	if ((data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0)
+		(void)index_mail_get_cached_bodystructure(mail, &str);
+
+	if (data->access_part == 0 && data->wanted_headers != NULL &&
+	    !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) {
+		/* see if all wanted headers exist in cache */
+		if (!imapc_mail_has_headers_in_cache(mail, data->wanted_headers))
+			data->access_part |= PARSE_HDR;
+	}
+	if (data->access_part == 0 &&
+	    (data->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0 &&
+	    !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) {
+		/* the common code already checked this partially,
+		   but we need a guaranteed correct answer */
+		header_ctx = mailbox_header_lookup_init(_mail->box,
+							message_part_envelope_headers);
+		if (!imapc_mail_has_headers_in_cache(mail, header_ctx))
+			data->access_part |= PARSE_HDR;
+		mailbox_header_lookup_unref(&header_ctx);
+	}
+}
+
+static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
+{
+	struct imapc_mail *imail = IMAPC_MAIL(_mail);
+	struct index_mail *mail = &imail->imail;
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+
+	index_mail_set_seq(_mail, seq, saving);
+	if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
+		/* RFC822.SIZE may be read from vsize record or cache. It may
+		   not be exactly correct. */
+		mail->data.inexact_total_sizes = TRUE;
+	}
+
+	/* searching code handles prefetching internally,
+	   elsewhere we want to do it immediately */
+	if (!mail->mail.search_mail && !_mail->saving)
+		(void)imapc_mail_prefetch(_mail);
+}
+
+static void
+imapc_mail_add_temp_wanted_fields(struct mail *_mail,
+				  enum mail_fetch_field fields,
+				  struct mailbox_header_lookup_ctx *headers)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	index_mail_add_temp_wanted_fields(_mail, fields, headers);
+	if (_mail->seq != 0)
+		imapc_mail_update_access_parts(mail);
+}
+
+static void imapc_mail_close(struct mail *_mail)
+{
+	struct imapc_mail *mail = IMAPC_MAIL(_mail);
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct imapc_mail_cache *cache = &mbox->prev_mail_cache;
+
+	if (mail->fetch_count > 0) {
+		imapc_mail_fetch_flush(mbox);
+		while (mail->fetch_count > 0)
+			imapc_mailbox_run_nofetch(mbox);
+	}
+
+	index_mail_close(_mail);
+
+	mail->fetching_headers = NULL;
+	if (mail->body_fetched) {
+		imapc_mail_cache_free(cache);
+		cache->uid = _mail->uid;
+		if (mail->fd != -1) {
+			cache->fd = mail->fd;
+			mail->fd = -1;
+		} else {
+			cache->buf = mail->body;
+			mail->body = NULL;
+		}
+	}
+	i_close_fd(&mail->fd);
+	buffer_free(&mail->body);
+	mail->header_fetched = FALSE;
+	mail->body_fetched = FALSE;
+
+	i_assert(mail->fetch_count == 0);
+}
+
+static int imapc_mail_get_hdr_hash(struct index_mail *imail)
+{
+	struct istream *input;
+	const unsigned char *data;
+	size_t size;
+	uoff_t old_offset;
+	struct sha1_ctxt sha1_ctx;
+	unsigned char sha1_output[SHA1_RESULTLEN];
+	const char *sha1_str;
+
+	sha1_init(&sha1_ctx);
+	old_offset = imail->data.stream == NULL ? 0 :
+		imail->data.stream->v_offset;
+	if (mail_get_hdr_stream(&imail->mail.mail, NULL, &input) < 0)
+		return -1;
+	i_assert(imail->data.stream != NULL);
+	while (i_stream_read_more(input, &data, &size) > 0) {
+		sha1_loop(&sha1_ctx, data, size);
+		i_stream_skip(input, size);
+	}
+	i_stream_seek(imail->data.stream, old_offset);
+	sha1_result(&sha1_ctx, sha1_output);
+
+	sha1_str = binary_to_hex(sha1_output, sizeof(sha1_output));
+	imail->data.guid = p_strdup(imail->mail.data_pool, sha1_str);
+	return 0;
+}
+
+static bool imapc_mail_get_cached_guid(struct mail *_mail)
+{
+	struct index_mail *imail = INDEX_MAIL(_mail);
+	const enum index_cache_field cache_idx =
+		imail->ibox->cache_fields[MAIL_CACHE_GUID].idx;
+	string_t *str;
+
+	if (imail->data.guid != NULL) {
+		if (mail_cache_field_can_add(_mail->transaction->cache_trans,
+					     _mail->seq, cache_idx)) {
+			/* GUID was prefetched - add to cache */
+			index_mail_cache_add_idx(imail, cache_idx,
+				imail->data.guid, strlen(imail->data.guid));
+		}
+		return TRUE;
+	}
+
+	str = str_new(imail->mail.data_pool, 64);
+	if (mail_cache_lookup_field(_mail->transaction->cache_view,
+				    str, imail->mail.mail.seq, cache_idx) > 0) {
+		imail->data.guid = str_c(str);
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static int imapc_mail_get_guid(struct mail *_mail, const char **value_r)
+{
+	struct index_mail *imail = INDEX_MAIL(_mail);
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	const enum index_cache_field cache_idx =
+		imail->ibox->cache_fields[MAIL_CACHE_GUID].idx;
+
+	if (imapc_mail_get_cached_guid(_mail)) {
+		*value_r = imail->data.guid;
+		return 0;
+	}
+
+	/* GUID not in cache, fetch it */
+	if (mbox->guid_fetch_field_name != NULL) {
+		if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID, NULL) < 0)
+			return -1;
+		if (imail->data.guid == NULL) {
+			(void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name);
+			return -1;
+		}
+	} else {
+		/* use hash of message headers as the GUID */
+		if (imapc_mail_get_hdr_hash(imail) < 0)
+			return -1;
+	}
+
+	index_mail_cache_add_idx(imail, cache_idx,
+				 imail->data.guid, strlen(imail->data.guid));
+	*value_r = imail->data.guid;
+	return 0;
+}
+
+static int
+imapc_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+		       const char **value_r)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(_mail->box);
+	struct index_mail *imail = INDEX_MAIL(_mail);
+	uint64_t num;
+
+	switch (field) {
+	case MAIL_FETCH_GUID:
+		if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GUID_FORCED) &&
+		    mbox->guid_fetch_field_name == NULL) {
+			/* GUIDs not supported by server */
+			break;
+		}
+		*value_r = "";
+		return imapc_mail_get_guid(_mail, value_r);
+	case MAIL_FETCH_UIDL_BACKEND:
+		if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION))
+			break;
+		if (imapc_mail_get_guid(_mail, value_r) < 0)
+			return -1;
+		if (str_to_uint64(*value_r, &num) < 0) {
+			mail_set_critical(_mail,
+				"X-GM-MSGID not 64bit integer as expected for POP3 UIDL generation: %s", *value_r);
+			return -1;
+		}
+
+		*value_r = p_strdup_printf(imail->mail.data_pool,
+					   "GmailId%"PRIx64, num);
+		return 0;
+	case MAIL_FETCH_IMAP_BODY:
+		if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE))
+			break;
+
+		if (index_mail_get_cached_body(imail, value_r))
+			return 0;
+		if (imapc_mail_fetch(_mail, field, NULL) < 0)
+			return -1;
+		if (imail->data.body == NULL) {
+			(void)imapc_mail_failed(_mail, "BODY");
+			return -1;
+		}
+		*value_r = imail->data.body;
+		return 0;
+	case MAIL_FETCH_IMAP_BODYSTRUCTURE:
+		if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_BODYSTRUCTURE))
+			break;
+
+		if (index_mail_get_cached_bodystructure(imail, value_r))
+			return 0;
+		if (imapc_mail_fetch(_mail, field, NULL) < 0)
+			return -1;
+		if (imail->data.bodystructure == NULL) {
+			(void)imapc_mail_failed(_mail, "BODYSTRUCTURE");
+			return -1;
+		}
+		*value_r = imail->data.bodystructure;
+		return 0;
+	default:
+		break;
+	}
+
+	return index_mail_get_special(_mail, field, value_r);
+}
+
+struct mail_vfuncs imapc_mail_vfuncs = {
+	imapc_mail_close,
+	index_mail_free,
+	imapc_mail_set_seq,
+	index_mail_set_uid,
+	index_mail_set_uid_cache_updates,
+	imapc_mail_prefetch,
+	index_mail_precache,
+	imapc_mail_add_temp_wanted_fields,
+
+	index_mail_get_flags,
+	index_mail_get_keywords,
+	index_mail_get_keyword_indexes,
+	imapc_mail_get_modseq,
+	index_mail_get_pvt_modseq,
+	index_mail_get_parts,
+	index_mail_get_date,
+	imapc_mail_get_received_date,
+	imapc_mail_get_save_date,
+	imapc_mail_get_virtual_size,
+	imapc_mail_get_physical_size,
+	imapc_mail_get_first_header,
+	imapc_mail_get_headers,
+	imapc_mail_get_header_stream,
+	imapc_mail_get_stream,
+	index_mail_get_binary_stream,
+	imapc_mail_get_special,
+	index_mail_get_backend_mail,
+	index_mail_update_flags,
+	index_mail_update_keywords,
+	index_mail_update_modseq,
+	index_mail_update_pvt_modseq,
+	NULL,
+	index_mail_expunge,
+	index_mail_set_cache_corrupted,
+	index_mail_opened,
+};
diff --git a/src/lib-storage/index/imapc/imapc-mail.h b/src/lib-storage/index/imapc/imapc-mail.h
new file mode 100644
index 0000000..52dfe5e
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-mail.h
@@ -0,0 +1,51 @@
+#ifndef IMAPC_MAIL_H
+#define IMAPC_MAIL_H
+
+#include "index-mail.h"
+
+struct imap_arg;
+struct imapc_untagged_reply;
+struct imapc_mailbox;
+
+struct imapc_mail {
+	struct index_mail imail;
+
+	enum mail_fetch_field fetching_fields;
+	const char *const *fetching_headers;
+	unsigned int fetch_count;
+	bool fetch_sent;
+	const char *last_fetch_reply;
+
+	int fd;
+	buffer_t *body;
+	bool header_fetched;
+	bool body_fetched;
+	bool header_list_fetched;
+	bool fetch_ignore_if_missing;
+	bool fetch_failed;
+};
+
+#define IMAPC_MAIL(s)	container_of(s, struct imapc_mail, imail.mail.mail)
+
+extern struct mail_vfuncs imapc_mail_vfuncs;
+
+struct mail *
+imapc_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers);
+int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields,
+		     const char *const *headers);
+void imapc_mail_try_init_stream_from_cache(struct imapc_mail *mail);
+bool imapc_mail_prefetch(struct mail *mail);
+void imapc_mail_fetch_flush(struct imapc_mailbox *mbox);
+void imapc_mail_init_stream(struct imapc_mail *mail);
+bool imapc_mail_has_headers_in_cache(struct index_mail *mail,
+				     struct mailbox_header_lookup_ctx *headers);
+
+void imapc_mail_fetch_update(struct imapc_mail *mail,
+			     const struct imapc_untagged_reply *reply,
+			     const struct imap_arg *args);
+void imapc_mail_update_access_parts(struct index_mail *mail);
+void imapc_mail_command_flush(struct imapc_mailbox *mbox);
+
+#endif
diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
new file mode 100644
index 0000000..ff915d9
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -0,0 +1,994 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "str.h"
+#include "mail-index-modseq.h"
+#include "imap-arg.h"
+#include "imap-seqset.h"
+#include "imap-util.h"
+#include "imapc-mail.h"
+#include "imapc-msgmap.h"
+#include "imapc-list.h"
+#include "imapc-search.h"
+#include "imapc-sync.h"
+#include "imapc-storage.h"
+
+#define NOTIFY_DELAY_MSECS 500
+
+void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox,
+				 const char *reason, ...)
+{
+	const char *errmsg;
+	va_list va;
+
+	va_start(va, reason);
+	errmsg = t_strdup_printf("Mailbox '%s' state corrupted: %s",
+		mbox->box.name, t_strdup_vprintf(reason, va));
+	va_end(va);
+
+	mail_storage_set_internal_error(&mbox->storage->storage);
+
+	if (!mbox->initial_sync_done) {
+		/* we failed during initial sync. need to rebuild indexes if
+		   we want to get this fixed */
+		mail_index_mark_corrupted(mbox->box.index);
+	} else {
+		/* maybe the remote server is buggy and has become confused.
+		   try reconnecting. */
+	}
+	imapc_client_mailbox_reconnect(mbox->client_box, errmsg);
+}
+
+struct mail_index_view *
+imapc_mailbox_get_sync_view(struct imapc_mailbox *mbox)
+{
+	if (mbox->sync_view == NULL)
+		mbox->sync_view = mail_index_view_open(mbox->box.index);
+	return mbox->sync_view;
+}
+
+static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox)
+{
+	if (mbox->delayed_sync_trans != NULL)
+		return;
+
+	i_assert(mbox->delayed_sync_cache_view == NULL);
+	i_assert(mbox->delayed_sync_cache_trans == NULL);
+
+	mbox->delayed_sync_trans =
+		mail_index_transaction_begin(imapc_mailbox_get_sync_view(mbox),
+					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+	mbox->delayed_sync_view =
+		mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);
+
+	mbox->delayed_sync_cache_view =
+		mail_cache_view_open(mbox->box.cache, mbox->delayed_sync_view);
+	mbox->delayed_sync_cache_trans =
+		mail_cache_get_transaction(mbox->delayed_sync_cache_view,
+					   mbox->delayed_sync_trans);
+}
+
+static int imapc_mailbox_commit_delayed_expunges(struct imapc_mailbox *mbox)
+{
+	struct mail_index_view *view = imapc_mailbox_get_sync_view(mbox);
+	struct mail_index_transaction *trans;
+	struct seq_range_iter iter;
+	unsigned int n;
+	uint32_t lseq, uid;
+	int ret;
+
+	trans = mail_index_transaction_begin(view,
+			MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+
+	seq_range_array_iter_init(&iter, &mbox->delayed_expunged_uids); n = 0;
+	while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+		if (mail_index_lookup_seq(view, uid, &lseq))
+			mail_index_expunge(trans, lseq);
+	}
+	array_clear(&mbox->delayed_expunged_uids);
+	ret = mail_index_transaction_commit(&trans);
+	if (ret < 0)
+		mailbox_set_index_error(&mbox->box);
+	return ret;
+}
+
+int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox,
+				       bool force, bool *changes_r)
+{
+	int ret = 0;
+
+	*changes_r = FALSE;
+
+	if (mbox->delayed_sync_view != NULL)
+		mail_index_view_close(&mbox->delayed_sync_view);
+	if (mbox->delayed_sync_trans == NULL)
+		;
+	else if (!mbox->selected && !force) {
+		/* ignore any changes done during SELECT */
+		mail_index_transaction_rollback(&mbox->delayed_sync_trans);
+	} else {
+		if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) {
+			mailbox_set_index_error(&mbox->box);
+			ret = -1;
+		}
+		*changes_r = TRUE;
+	}
+	mbox->delayed_sync_cache_trans = NULL;
+	if (mbox->delayed_sync_cache_view != NULL)
+		mail_cache_view_close(&mbox->delayed_sync_cache_view);
+
+	if (array_count(&mbox->delayed_expunged_uids) > 0) {
+		/* delayed expunges - commit them now in a separate
+		   transaction. Reopen mbox->sync_view to see changes
+		   committed in delayed_sync_trans. */
+		if (mbox->sync_view != NULL)
+			mail_index_view_close(&mbox->sync_view);
+		if (imapc_mailbox_commit_delayed_expunges(mbox) < 0)
+			ret = -1;
+	}
+
+	if (mbox->sync_view != NULL)
+		mail_index_view_close(&mbox->sync_view);
+	i_assert(mbox->delayed_sync_trans == NULL);
+	i_assert(mbox->delayed_sync_view == NULL);
+	i_assert(mbox->delayed_sync_cache_trans == NULL);
+	return ret;
+}
+
+static void imapc_mailbox_idle_timeout(struct imapc_mailbox *mbox)
+{
+	timeout_remove(&mbox->to_idle_delay);
+	if (mbox->box.notify_callback != NULL)
+		mbox->box.notify_callback(&mbox->box, mbox->box.notify_context);
+}
+
+static void imapc_mailbox_idle_notify(struct imapc_mailbox *mbox)
+{
+	struct ioloop *old_ioloop = current_ioloop;
+
+	if (mbox->box.notify_callback != NULL &&
+	    mbox->to_idle_delay == NULL) {
+		io_loop_set_current(mbox->storage->root_ioloop);
+		mbox->to_idle_delay =
+			timeout_add_short(NOTIFY_DELAY_MSECS,
+					  imapc_mailbox_idle_timeout, mbox);
+		io_loop_set_current(old_ioloop);
+	}
+}
+
+static void
+imapc_mailbox_index_expunge(struct imapc_mailbox *mbox, uint32_t uid)
+{
+	uint32_t lseq;
+
+	if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq))
+		mail_index_expunge(mbox->delayed_sync_trans, lseq);
+	else if (mail_index_lookup_seq(mbox->delayed_sync_view, uid, &lseq)) {
+		/* this message exists only in this transaction. lib-index
+		   can't currently handle expunging anything except the last
+		   appended message in a transaction, and fixing it would be
+		   quite a lot of trouble. so instead we'll just delay doing
+		   this expunge until after the current transaction has been
+		   committed. */
+		seq_range_array_add(&mbox->delayed_expunged_uids, uid);
+	} else {
+		/* already expunged by another session */
+	}
+}
+
+static void
+imapc_mailbox_fetch_state_finish(struct imapc_mailbox *mbox)
+{
+	uint32_t lseq, uid, msg_count;
+
+	if (mbox->sync_next_lseq == 0) {
+		/* FETCH n:*, not 1:* */
+		i_assert(mbox->state_fetched_success ||
+			 (mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0);
+		return;
+	}
+
+	/* if we haven't seen FETCH reply for some messages at the end of
+	   mailbox they've been externally expunged. */
+	msg_count = mail_index_view_get_messages_count(mbox->delayed_sync_view);
+	for (lseq = mbox->sync_next_lseq; lseq <= msg_count; lseq++) {
+		mail_index_lookup_uid(mbox->delayed_sync_view, lseq, &uid);
+		if (uid >= mbox->sync_uid_next) {
+			/* another process already added new messages to index
+			   that our IMAP connection hasn't seen yet */
+			break;
+		}
+		imapc_mailbox_index_expunge(mbox, uid);
+	}
+
+	mbox->sync_next_lseq = 0;
+	mbox->sync_next_rseq = 0;
+	mbox->state_fetched_success = TRUE;
+}
+
+static void
+imapc_mailbox_fetch_state_callback(const struct imapc_command_reply *reply,
+				   void *context)
+{
+	struct imapc_mailbox *mbox = context;
+
+	mbox->state_fetching_uid1 = FALSE;
+	mbox->delayed_untagged_exists = FALSE;
+	imapc_client_stop(mbox->storage->client->client);
+
+	switch (reply->state) {
+	case IMAPC_COMMAND_STATE_OK:
+		imapc_mailbox_fetch_state_finish(mbox);
+		break;
+	case IMAPC_COMMAND_STATE_NO:
+		imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS, reply);
+		break;
+	case IMAPC_COMMAND_STATE_DISCONNECTED:
+		mail_storage_set_internal_error(mbox->box.storage);
+
+		break;
+	default:
+		mail_storage_set_critical(mbox->box.storage,
+			"imapc: state FETCH failed: %s", reply->text_full);
+		break;
+	}
+}
+
+void imap_mailbox_select_finish(struct imapc_mailbox *mbox)
+{
+	if (mbox->exists_count == 0) {
+		/* no mails. expunge everything. */
+		mbox->sync_next_lseq = 1;
+		imapc_mailbox_init_delayed_trans(mbox);
+		imapc_mailbox_fetch_state_finish(mbox);
+	}
+	mbox->selected = TRUE;
+}
+
+bool
+imapc_mailbox_fetch_state(struct imapc_mailbox *mbox, uint32_t first_uid)
+{
+	struct imapc_command *cmd;
+
+	if (mbox->exists_count == 0) {
+		/* empty mailbox - no point in fetching anything.
+		   just make sure everything is expunged in local index.
+		   Delay calling imapc_mailbox_fetch_state_finish() until
+		   SELECT finishes, so we see the updated UIDNEXT. */
+		return FALSE;
+	}
+	if (mbox->state_fetching_uid1) {
+		/* retrying after reconnection - don't send duplicate */
+		return FALSE;
+	}
+
+	string_t *str = t_str_new(64);
+	str_printfa(str, "UID FETCH %u:* (FLAGS", first_uid);
+	if (imapc_mailbox_has_modseqs(mbox)) {
+		str_append(str, " MODSEQ");
+		mail_index_modseq_enable(mbox->box.index);
+	}
+	if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
+		enum mailbox_info_flags flags;
+
+		if (!mail_index_is_in_memory(mbox->box.index)) {
+			/* these can be efficiently fetched among flags and
+			   stored into cache */
+			str_append(str, " X-GM-MSGID");
+		}
+		/* do this only for the \All mailbox */
+		if (imapc_list_get_mailbox_flags(mbox->box.list,
+						 mbox->box.name, &flags) == 0 &&
+		    (flags & MAILBOX_SPECIALUSE_ALL) != 0)
+			str_append(str, " X-GM-LABELS");
+
+	}
+	str_append_c(str, ')');
+
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+		imapc_mailbox_fetch_state_callback, mbox);
+	if (first_uid == 1) {
+		mbox->sync_next_lseq = 1;
+		mbox->sync_next_rseq = 1;
+		mbox->state_fetched_success = FALSE;
+		/* only the FETCH 1:* is retriable - others will be retried
+		   by the 1:* after the reconnection */
+		imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	}
+	mbox->state_fetching_uid1 = first_uid == 1;
+	imapc_command_send(cmd, str_c(str));
+	return TRUE;
+}
+
+static void
+imapc_untagged_exists(const struct imapc_untagged_reply *reply,
+		      struct imapc_mailbox *mbox)
+{
+	struct mail_index_view *view;
+	uint32_t exists_count = reply->num;
+
+	if (mbox == NULL)
+		return;
+	if (mbox->exists_received &&
+	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES)) {
+		/* ignore all except the first EXISTS reply (returned by
+		   SELECT) */
+		return;
+	}
+
+	mbox->exists_count = exists_count;
+	mbox->exists_received = TRUE;
+
+	view = mbox->delayed_sync_view;
+	if (view == NULL)
+		view = imapc_mailbox_get_sync_view(mbox);
+
+	if (mbox->selecting) {
+		/* We don't know the latest flags, refresh them. */
+		(void)imapc_mailbox_fetch_state(mbox, 1);
+	} else if (mbox->sync_fetch_first_uid != 1) {
+		const struct mail_index_header *hdr;
+		hdr = mail_index_get_header(view);
+		mbox->sync_fetch_first_uid = hdr->next_uid;
+		mbox->delayed_untagged_exists = TRUE;
+	}
+	imapc_mailbox_idle_notify(mbox);
+}
+
+static bool keywords_are_equal(struct mail_keywords *kw,
+			       const ARRAY_TYPE(keyword_indexes) *kw_arr)
+{
+	const unsigned int *kw_idx;
+	unsigned int i, j, count;
+
+	kw_idx = array_get(kw_arr, &count);
+	if (count != kw->count)
+		return FALSE;
+
+	/* there are normally only a few keywords, so O(n^2) is fine */
+	for (i = 0; i < count; i++) {
+		for (j = 0; j < count; j++) {
+			if (kw->idx[i] == kw_idx[j])
+				break;
+		}
+		if (j == count)
+			return FALSE;
+	}
+	return TRUE;
+}
+
+static int
+imapc_mailbox_msgmap_update(struct imapc_mailbox *mbox,
+			    uint32_t rseq, uint32_t fetch_uid,
+			    uint32_t *lseq_r, uint32_t *uid_r,
+			    bool *new_message_r)
+{
+	struct imapc_msgmap *msgmap;
+	uint32_t uid, msg_count, rseq2;
+
+	*lseq_r = 0;
+	*uid_r = uid = fetch_uid;
+	*new_message_r = FALSE;
+
+	if (rseq > mbox->exists_count) {
+		/* Receiving a FETCH for a message that EXISTS hasn't
+		   announced yet. MS Exchange has a bug where our UID FETCH
+		   request sometimes sends replies where sequences are above
+		   EXISTS value, but their UIDs are for existing messages.
+		   We'll just ignore these replies. */
+		return 0;
+	}
+	if (rseq < mbox->prev_skipped_rseq &&
+	    fetch_uid > mbox->prev_skipped_uid) {
+		/* This was the initial attempt at catching the above
+		   MS Exchange bug, but the above one appears to catch all
+		   these cases. But keep it here just in case. */
+		imapc_mailbox_set_corrupted(mbox,
+			"FETCH sequence/UID order is mixed "
+			"(seq=%u,%u vs uid=%u,%u)",
+			mbox->prev_skipped_rseq, rseq,
+			mbox->prev_skipped_uid, fetch_uid);
+		return -1;
+	}
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	msg_count = imapc_msgmap_count(msgmap);
+	if (fetch_uid != 0 && mbox->state_fetched_success &&
+	    (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS) ||
+	     IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES))) {
+		/* if we know the UID, use own own generated rseq instead of
+		   the potentially broken rseq that the server sent.
+		   Skip this during the initial FETCH 1:* (UID ..) handling,
+		   or we can't detect duplicate UIDs and will instead
+		   assert-crash later on. */
+		uint32_t fixed_rseq;
+
+		if (imapc_msgmap_uid_to_rseq(msgmap, fetch_uid, &fixed_rseq))
+			rseq = fixed_rseq;
+		else if (fetch_uid >= imapc_msgmap_uidnext(msgmap) &&
+			 rseq <= msg_count) {
+			/* The current rseq is wrong. Lets hope that the
+			   correct rseq is the next new one. This happens
+			   especially with no-msn-updates when mails have been
+			   expunged and new mails arrive in the same session. */
+			rseq = msg_count+1;
+		}
+	}
+
+	if (rseq <= msg_count) {
+		uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
+		if (uid != fetch_uid && fetch_uid != 0) {
+			imapc_mailbox_set_corrupted(mbox,
+				"FETCH UID mismatch (%u != %u)",
+				fetch_uid, uid);
+			return -1;
+		}
+		*uid_r = uid;
+	} else if (fetch_uid == 0 || rseq != msg_count+1) {
+		/* probably a flag update for a message we haven't yet
+		   received our initial UID FETCH for. we should get
+		   another one. */
+		if (fetch_uid == 0)
+			return 0;
+
+		if (imapc_msgmap_uid_to_rseq(msgmap, fetch_uid, &rseq2)) {
+			imapc_mailbox_set_corrupted(mbox,
+				"FETCH returned wrong sequence for UID %u "
+				"(%u != %u)", fetch_uid, rseq, rseq2);
+			return -1;
+		}
+		mbox->prev_skipped_rseq = rseq;
+		mbox->prev_skipped_uid = fetch_uid;
+		/* Check if this uid must be added later when syncing. */
+		*new_message_r = TRUE;
+	} else if (fetch_uid < imapc_msgmap_uidnext(msgmap)) {
+		imapc_mailbox_set_corrupted(mbox,
+			"Expunged message reappeared in session "
+			"(uid=%u < next_uid=%u)",
+			fetch_uid, imapc_msgmap_uidnext(msgmap));
+		return -1;
+	} else {
+		/* newly seen message */
+		imapc_msgmap_append(msgmap, rseq, uid);
+		if (uid < mbox->min_append_uid ||
+		    uid < mail_index_get_header(mbox->delayed_sync_view)->next_uid) {
+			/* message is already added to index */
+		} else if (mbox->state_fetching_uid1) {
+			/* Initial fetching, allow messages to be appened to
+			   index directly */
+			mail_index_append(mbox->delayed_sync_trans,
+					  uid, lseq_r);
+			mbox->min_append_uid = uid + 1;
+		} else {
+			/* message is not yet added to index, in order to
+			   prevent log synchronization errors add this
+			   message later, when the mailbox is synced. */
+			*new_message_r = TRUE;
+		}
+	}
+	return 0;
+}
+
+bool imapc_mailbox_name_equals(struct imapc_mailbox *mbox,
+			       const char *remote_name)
+{
+	const char *imapc_remote_name =
+		imapc_mailbox_get_remote_name(mbox);
+
+	if (strcmp(imapc_remote_name, remote_name) == 0) {
+		/* match */
+		return TRUE;
+	} else if (strcasecmp(mbox->box.name, "INBOX") == 0 &&
+		   strcasecmp(remote_name, "INBOX") == 0) {
+		/* case-insensitive INBOX */
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static struct imapc_untagged_fetch_ctx *
+imapc_untagged_fetch_ctx_create(void)
+{
+	pool_t pool = pool_alloconly_create("imapc untagged fetch ctx", 128);
+	struct imapc_untagged_fetch_ctx *ctx =
+		p_new(pool, struct imapc_untagged_fetch_ctx, 1);
+	ctx->pool = pool;
+	return ctx;
+}
+
+void imapc_untagged_fetch_ctx_free(struct imapc_untagged_fetch_ctx **_ctx)
+{
+	struct imapc_untagged_fetch_ctx *ctx = *_ctx;
+
+	*_ctx = NULL;
+	i_assert(ctx != NULL);
+
+	pool_unref(&ctx->pool);
+}
+
+void imapc_untagged_fetch_update_flags(struct imapc_mailbox *mbox,
+				       struct imapc_untagged_fetch_ctx *ctx,
+				       struct mail_index_view *view,
+				       uint32_t lseq)
+{
+	ARRAY_TYPE(keyword_indexes) old_kws;
+	struct mail_keywords *kw;
+	const struct mail_index_record *rec = NULL;
+	const char *atom;
+
+	if (!ctx->have_flags)
+		return;
+
+	rec = mail_index_lookup(view, lseq);
+	if (rec->flags != ctx->flags) {
+		mail_index_update_flags(mbox->delayed_sync_trans, lseq,
+					MODIFY_REPLACE, ctx->flags);
+	}
+
+	t_array_init(&old_kws, 8);
+	mail_index_lookup_keywords(view, lseq, &old_kws);
+
+	if (ctx->have_gmail_labels) {
+		/* add keyword for mails that have GMail labels.
+		   this can be used for "All Mail" mailbox migrations
+		   with dsync */
+		atom = "$GMailHaveLabels";
+		array_push_back(&ctx->keywords, &atom);
+	}
+
+	array_append_zero(&ctx->keywords);
+	kw = mail_index_keywords_create(mbox->box.index,
+					array_front(&ctx->keywords));
+	if (!keywords_are_equal(kw, &old_kws)) {
+		mail_index_update_keywords(mbox->delayed_sync_trans,
+					   lseq, MODIFY_REPLACE, kw);
+	}
+	mail_index_keywords_unref(&kw);
+}
+
+static bool imapc_untagged_fetch_handle(struct imapc_mailbox *mbox,
+					struct imapc_untagged_fetch_ctx *ctx,
+					uint32_t rseq)
+{
+	uint32_t lseq;
+	bool new_message;
+
+	imapc_mailbox_init_delayed_trans(mbox);
+	if (imapc_mailbox_msgmap_update(mbox, rseq, ctx->fetch_uid,
+					&lseq, &ctx->uid,
+					&new_message) < 0 || ctx->uid == 0)
+		return FALSE;
+
+	if ((ctx->flags & MAIL_RECENT) == 0 && mbox->highest_nonrecent_uid < ctx->uid) {
+		/* remember for STATUS_FIRST_RECENT_UID */
+		mbox->highest_nonrecent_uid = ctx->uid;
+	}
+	/* FIXME: we should ideally also pass these through so they show up
+	   to clients. */
+	ctx->flags &= ENUM_NEGATE(MAIL_RECENT);
+
+	if (lseq == 0) {
+		if (!mail_index_lookup_seq(mbox->delayed_sync_view,
+					   ctx->uid, &lseq)) {
+			/* already expunged by another session */
+			if (rseq == mbox->sync_next_rseq)
+				mbox->sync_next_rseq++;
+			return new_message;
+		}
+	}
+
+	if (rseq == mbox->sync_next_rseq) {
+		/* we're doing the initial full sync of mails. expunge any
+		   mails that no longer exist. */
+		while (mbox->sync_next_lseq < lseq) {
+			mail_index_lookup_uid(mbox->delayed_sync_view,
+					      mbox->sync_next_lseq, &ctx->uid);
+			imapc_mailbox_index_expunge(mbox, ctx->uid);
+			mbox->sync_next_lseq++;
+		}
+		i_assert(lseq == mbox->sync_next_lseq);
+		mbox->sync_next_rseq++;
+		mbox->sync_next_lseq++;
+	}
+
+	if (!new_message) {
+		/* Only update flags immediately for existing messages */
+		imapc_untagged_fetch_update_flags(mbox, ctx,
+						  mbox->delayed_sync_view, lseq);
+	}
+
+	if (ctx->modseq != 0) {
+		if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < ctx->modseq)
+			mail_index_update_modseq(mbox->delayed_sync_trans, lseq, ctx->modseq);
+		array_idx_set(&mbox->rseq_modseqs, rseq-1, &ctx->modseq);
+	}
+	if (ctx->guid != NULL) {
+		struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
+		const enum index_cache_field guid_cache_idx =
+			ibox->cache_fields[MAIL_CACHE_GUID].idx;
+
+		if (mail_cache_field_can_add(mbox->delayed_sync_cache_trans,
+					     lseq, guid_cache_idx)) {
+			mail_cache_add(mbox->delayed_sync_cache_trans, lseq,
+				       guid_cache_idx, ctx->guid, strlen(ctx->guid));
+		}
+	}
+	return new_message;
+}
+
+static bool imapc_untagged_fetch_parse(struct imapc_mailbox *mbox,
+				       struct imapc_untagged_fetch_ctx *ctx,
+				       const struct imap_arg *list)
+{
+	const struct imap_arg *flags_list, *modseq_list;
+	const char *atom, *patom;
+	unsigned int i, j;
+
+	ctx->fetch_uid = 0; ctx->flags = 0;
+	for (i = 0; list[i].type != IMAP_ARG_EOL; i += 2) {
+		if (!imap_arg_get_atom(&list[i], &atom) ||
+		    list[i+1].type == IMAP_ARG_EOL)
+			return FALSE;
+
+		if (strcasecmp(atom, "UID") == 0) {
+			if (!imap_arg_get_atom(&list[i+1], &atom) ||
+			    str_to_uint32(atom, &ctx->fetch_uid) < 0)
+				return FALSE;
+		} else if (strcasecmp(atom, "FLAGS") == 0) {
+			if (!imap_arg_get_list(&list[i+1], &flags_list))
+				return FALSE;
+
+			p_array_init(&ctx->keywords, ctx->pool, 8);
+			ctx->have_flags = TRUE;
+			for (j = 0; flags_list[j].type != IMAP_ARG_EOL; j++) {
+				if (!imap_arg_get_atom(&flags_list[j], &atom))
+					return FALSE;
+				if (atom[0] == '\\')
+					ctx->flags |= imap_parse_system_flag(atom);
+				else {
+					patom = p_strdup(ctx->pool, atom);
+					/* keyword */
+					array_push_back(&ctx->keywords, &patom);
+				}
+			}
+		} else if (strcasecmp(atom, "MODSEQ") == 0 &&
+			   imapc_mailbox_has_modseqs(mbox)) {
+			/* (modseq-number) */
+			if (!imap_arg_get_list(&list[i+1], &modseq_list))
+				return FALSE;
+			if (!imap_arg_get_atom(&modseq_list[0], &atom) ||
+			    str_to_uint64(atom, &ctx->modseq) < 0 ||
+			    modseq_list[1].type != IMAP_ARG_EOL)
+				return FALSE;
+		} else if (strcasecmp(atom, "X-GM-MSGID") == 0 &&
+			   !mbox->initial_sync_done) {
+			if (imap_arg_get_atom(&list[i+1], &atom))
+				ctx->guid = atom;
+		} else if (strcasecmp(atom, "X-GM-LABELS") == 0 &&
+			   IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
+			if (!imap_arg_get_list(&list[i+1], &flags_list))
+				return FALSE;
+			for (j = 0; flags_list[j].type != IMAP_ARG_EOL; j++) {
+				if (!imap_arg_get_astring(&flags_list[j], &atom))
+					return FALSE;
+				if (strcasecmp(atom, "\\Muted") != 0)
+					ctx->have_gmail_labels = TRUE;
+			}
+		}
+	}
+	if (ctx->fetch_uid == 0 &&
+	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES)) {
+		/* UID missing and we're not tracking MSNs */
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
+				 struct imapc_mailbox *mbox)
+{
+	const struct imap_arg *list;
+	struct imapc_fetch_request *fetch_request;
+	struct imapc_mail *mail;
+	bool new_message = FALSE;
+
+	if (mbox == NULL || reply->num == 0 || !imap_arg_get_list(reply->args, &list))
+		return;
+
+	struct imapc_untagged_fetch_ctx *ctx =
+		imapc_untagged_fetch_ctx_create();
+	if (!imapc_untagged_fetch_parse(mbox, ctx, list)) {
+		imapc_untagged_fetch_ctx_free(&ctx);
+		return;
+	}
+
+	new_message = imapc_untagged_fetch_handle(mbox, ctx, reply->num);
+
+	/* if this is a reply to some FETCH request, update the mail's fields */
+	array_foreach_elem(&mbox->fetch_requests, fetch_request) {
+		array_foreach_elem(&fetch_request->mails, mail) {
+			if (mail->imail.mail.mail.uid == ctx->uid)
+				imapc_mail_fetch_update(mail, reply, list);
+		}
+	}
+
+	if (!new_message) {
+		/* Handling this context is finished if the mail was not new
+		   to the local index. It has not been added to
+		   mbox->untagged_fetch_contexts so no need to delete it from
+		   the array. The context itself can be freed here. */
+		imapc_untagged_fetch_ctx_free(&ctx);
+	} else {
+		/* If this is a new message store this context to be handled
+		   when syncing */
+		array_push_back(&mbox->untagged_fetch_contexts, &ctx);
+	}
+	imapc_mailbox_idle_notify(mbox);
+}
+
+static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
+				   struct imapc_mailbox *mbox)
+{
+	struct imapc_msgmap *msgmap;
+	uint32_t uid, rseq = reply->num;
+
+	if (mbox == NULL || rseq == 0 ||
+	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES))
+		return;
+
+	mbox->prev_skipped_rseq = 0;
+	mbox->prev_skipped_uid = 0;
+
+	if (mbox->exists_count == 0) {
+		imapc_mailbox_set_corrupted(mbox,
+			"EXPUNGE received for empty mailbox");
+		return;
+	}
+	mbox->exists_count--;
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	if (rseq > imapc_msgmap_count(msgmap)) {
+		/* we haven't even seen this message yet */
+		return;
+	}
+	uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
+	imapc_msgmap_expunge(msgmap, rseq);
+	if (array_is_created(&mbox->rseq_modseqs))
+		array_delete(&mbox->rseq_modseqs, rseq-1, 1);
+
+	imapc_mailbox_init_delayed_trans(mbox);
+	imapc_mailbox_index_expunge(mbox, uid);
+	imapc_mailbox_idle_notify(mbox);
+}
+
+static void
+imapc_untagged_esearch_gmail_pop3(const struct imap_arg *args,
+				  struct imapc_mailbox *mbox)
+{
+	struct imapc_msgmap *msgmap;
+	const char *atom;
+	struct seq_range_iter iter;
+	ARRAY_TYPE(seq_range) rseqs;
+	unsigned int n;
+	uint32_t rseq, lseq, uid;
+	ARRAY_TYPE(keyword_indexes) keywords;
+	struct mail_keywords *kw;
+	unsigned int pop3_deleted_kw_idx;
+
+	i_free_and_null(mbox->sync_gmail_pop3_search_tag);
+
+	/* It should contain ALL <seqset> or nonexistent if nothing matched */
+	if (args[0].type == IMAP_ARG_EOL)
+		return;
+	t_array_init(&rseqs, 64);
+	if (!imap_arg_atom_equals(&args[0], "ALL") ||
+	    !imap_arg_get_atom(&args[1], &atom) ||
+	    imap_seq_set_nostar_parse(atom, &rseqs) < 0) {
+		i_error("Invalid gmail-pop3 ESEARCH reply");
+		return;
+	}
+
+	mail_index_keyword_lookup_or_create(mbox->box.index,
+		mbox->storage->set->pop3_deleted_flag, &pop3_deleted_kw_idx);
+
+	t_array_init(&keywords, 1);
+	array_push_back(&keywords, &pop3_deleted_kw_idx);
+	kw = mail_index_keywords_create_from_indexes(mbox->box.index, &keywords);
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	seq_range_array_iter_init(&iter, &rseqs); n = 0;
+	while (seq_range_array_iter_nth(&iter, n++, &rseq)) {
+		if (rseq > imapc_msgmap_count(msgmap)) {
+			/* we haven't even seen this message yet */
+			break;
+		}
+		uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
+		if (!mail_index_lookup_seq(mbox->delayed_sync_view,
+					   uid, &lseq))
+			continue;
+
+		/* add the pop3_deleted_flag */
+		mail_index_update_keywords(mbox->delayed_sync_trans,
+					   lseq, MODIFY_ADD, kw);
+	}
+	mail_index_keywords_unref(&kw);
+}
+
+static void imapc_untagged_search(const struct imapc_untagged_reply *reply,
+				  struct imapc_mailbox *mbox)
+{
+	if (mbox != NULL)
+		imapc_search_reply_search(reply->args, mbox);
+}
+
+static void imapc_untagged_esearch(const struct imapc_untagged_reply *reply,
+				   struct imapc_mailbox *mbox)
+{
+	const struct imap_arg *tag_list;
+	const char *str;
+
+	if (mbox == NULL || !imap_arg_get_list(reply->args, &tag_list))
+		return;
+
+	/* ESEARCH begins with (TAG <tag>) */
+	if (!imap_arg_atom_equals(&tag_list[0], "TAG") ||
+	    !imap_arg_get_string(&tag_list[1], &str) ||
+	    tag_list[2].type != IMAP_ARG_EOL)
+		return;
+
+	/* for now the only ESEARCH reply that we have is for getting GMail's
+	   list of hidden POP3 messages. */
+	if (mbox->sync_gmail_pop3_search_tag != NULL &&
+	    strcmp(mbox->sync_gmail_pop3_search_tag, str) == 0)
+		imapc_untagged_esearch_gmail_pop3(reply->args+1, mbox);
+	else
+		imapc_search_reply_esearch(reply->args+1, mbox);
+}
+
+static void imapc_sync_uid_validity(struct imapc_mailbox *mbox)
+{
+	const struct mail_index_header *hdr;
+
+	imapc_mailbox_init_delayed_trans(mbox);
+	hdr = mail_index_get_header(mbox->delayed_sync_view);
+	if (hdr->uid_validity != mbox->sync_uid_validity &&
+	    mbox->sync_uid_validity != 0) {
+		if (hdr->uid_validity != 0) {
+			/* uidvalidity changed, reset the entire mailbox */
+			mail_index_reset(mbox->delayed_sync_trans);
+			mbox->sync_fetch_first_uid = 1;
+			/* The reset needs to be committed before FETCH 1:*
+			   results are received. */
+			bool changes;
+			if (imapc_mailbox_commit_delayed_trans(mbox, TRUE, &changes) < 0)
+				mail_index_mark_corrupted(mbox->box.index);
+			imapc_mailbox_init_delayed_trans(mbox);
+		}
+		mail_index_update_header(mbox->delayed_sync_trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&mbox->sync_uid_validity,
+			sizeof(mbox->sync_uid_validity), TRUE);
+	}
+}
+
+static void
+imapc_resp_text_uidvalidity(const struct imapc_untagged_reply *reply,
+			    struct imapc_mailbox *mbox)
+{
+	uint32_t uid_validity;
+
+	if (mbox == NULL ||
+	    str_to_uint32(reply->resp_text_value, &uid_validity) < 0 ||
+	    uid_validity == 0)
+		return;
+
+	if (mbox->sync_uid_validity != uid_validity) {
+		mbox->sync_uid_validity = uid_validity;
+		imapc_mail_cache_free(&mbox->prev_mail_cache);
+		imapc_sync_uid_validity(mbox);
+	}
+}
+
+static void
+imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply,
+			struct imapc_mailbox *mbox)
+{
+	uint32_t uid_next;
+
+	if (mbox == NULL ||
+	    str_to_uint32(reply->resp_text_value, &uid_next) < 0)
+		return;
+
+	mbox->sync_uid_next = uid_next;
+}
+
+static void
+imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply,
+			      struct imapc_mailbox *mbox)
+{
+	uint64_t highestmodseq;
+
+	if (mbox == NULL ||
+	    str_to_uint64(reply->resp_text_value, &highestmodseq) < 0)
+		return;
+
+	mbox->sync_highestmodseq = highestmodseq;
+}
+
+static void
+imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply,
+			       struct imapc_mailbox *mbox)
+{
+	const struct imap_arg *flags_args, *arg;
+	const char *flag;
+	unsigned int idx;
+
+	i_assert(reply->args[0].type == IMAP_ARG_ATOM);
+
+	if (mbox == NULL || !imap_arg_get_list(&reply->args[1], &flags_args))
+		return;
+
+	mbox->permanent_flags = 0;
+	mbox->box.disallow_new_keywords = TRUE;
+
+	for (arg = flags_args; arg->type != IMAP_ARG_EOL; arg++) {
+		if (!imap_arg_get_atom(arg, &flag))
+			continue;
+
+		if (strcmp(flag, "\\*") == 0)
+			mbox->box.disallow_new_keywords = FALSE;
+		else if (*flag == '\\')
+			mbox->permanent_flags |= imap_parse_system_flag(flag);
+		else {
+			/* we'll simply make sure that it exists in the index */
+			mail_index_keyword_lookup_or_create(mbox->box.index,
+							    flag, &idx);
+		}
+	}
+}
+
+void imapc_mailbox_register_untagged(struct imapc_mailbox *mbox,
+				     const char *key,
+				     imapc_mailbox_callback_t *callback)
+{
+	struct imapc_mailbox_event_callback *cb;
+
+	cb = array_append_space(&mbox->untagged_callbacks);
+	cb->name = p_strdup(mbox->box.pool, key);
+	cb->callback = callback;
+}
+
+void imapc_mailbox_register_resp_text(struct imapc_mailbox *mbox,
+				      const char *key,
+				      imapc_mailbox_callback_t *callback)
+{
+	struct imapc_mailbox_event_callback *cb;
+
+	cb = array_append_space(&mbox->resp_text_callbacks);
+	cb->name = p_strdup(mbox->box.pool, key);
+	cb->callback = callback;
+}
+
+void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox)
+{
+	imapc_mailbox_register_untagged(mbox, "EXISTS",
+					imapc_untagged_exists);
+	imapc_mailbox_register_untagged(mbox, "FETCH",
+					imapc_untagged_fetch);
+	imapc_mailbox_register_untagged(mbox, "EXPUNGE",
+					imapc_untagged_expunge);
+	imapc_mailbox_register_untagged(mbox, "SEARCH",
+					imapc_untagged_search);
+	imapc_mailbox_register_untagged(mbox, "ESEARCH",
+					imapc_untagged_esearch);
+	imapc_mailbox_register_resp_text(mbox, "UIDVALIDITY",
+					 imapc_resp_text_uidvalidity);
+	imapc_mailbox_register_resp_text(mbox, "UIDNEXT",
+					 imapc_resp_text_uidnext);
+	imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ",
+					 imapc_resp_text_highestmodseq);
+	imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS",
+					 imapc_resp_text_permanentflags);
+}
diff --git a/src/lib-storage/index/imapc/imapc-save.c b/src/lib-storage/index/imapc/imapc-save.c
new file mode 100644
index 0000000..c50f46b
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-save.c
@@ -0,0 +1,829 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "ostream.h"
+#include "imap-date.h"
+#include "imap-util.h"
+#include "imap-seqset.h"
+#include "imap-quote.h"
+#include "index-mail.h"
+#include "mail-copy.h"
+#include "mailbox-list-private.h"
+#include "imapc-msgmap.h"
+#include "imapc-storage.h"
+#include "imapc-sync.h"
+#include "imapc-mail.h"
+#include "seq-set-builder.h"
+
+struct imapc_save_context {
+	struct mail_save_context ctx;
+
+	struct imapc_mailbox *mbox;
+	struct imapc_mailbox *src_mbox;
+	struct mail_index_transaction *trans;
+
+	int fd;
+	char *temp_path;
+	struct istream *input;
+
+	uint32_t dest_uid_validity;
+	ARRAY_TYPE(seq_range) dest_saved_uids;
+	unsigned int save_count;
+
+	bool failed:1;
+	bool finished:1;
+};
+
+struct imapc_save_cmd_context {
+	struct imapc_save_context *ctx;
+	int ret;
+};
+
+#define IMAPC_SAVECTX(s)	container_of(s, struct imapc_save_context, ctx)
+#define IMAPC_SERVER_CMDLINE_MAX_LEN 	8000
+
+void imapc_transaction_save_rollback(struct mail_save_context *_ctx);
+static void imapc_mail_copy_bulk_flush(struct imapc_mailbox *mbox);
+
+struct mail_save_context *
+imapc_save_alloc(struct mailbox_transaction_context *t)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(t->box);
+	struct imapc_save_context *ctx;
+
+	i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	if (t->save_ctx == NULL) {
+		ctx = i_new(struct imapc_save_context, 1);
+		ctx->ctx.transaction = t;
+		ctx->mbox = mbox;
+		ctx->src_mbox = NULL;
+		ctx->trans = t->itrans;
+		ctx->fd = -1;
+		t->save_ctx = &ctx->ctx;
+	}
+	return t->save_ctx;
+}
+
+int imapc_save_begin(struct mail_save_context *_ctx, struct istream *input)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+	const char *path;
+
+	i_assert(ctx->fd == -1);
+
+	if (imapc_storage_client_handle_auth_failure(ctx->mbox->storage->client))
+		return -1;
+
+	ctx->fd = imapc_client_create_temp_fd(ctx->mbox->storage->client->client,
+					      &path);
+	if (ctx->fd == -1) {
+		mail_set_critical(_ctx->dest_mail,
+				  "Couldn't create temp file %s", path);
+		ctx->failed = TRUE;
+		return -1;
+	}
+	/* we may not know the size of the input, or be sure that it contains
+	   only CRLFs. so we'll always first write the mail to a temp file and
+	   upload it from there to remote server. */
+	ctx->finished = FALSE;
+	ctx->temp_path = i_strdup(path);
+	ctx->input = i_stream_create_crlf(input);
+	_ctx->data.output = o_stream_create_fd_file(ctx->fd, 0, FALSE);
+	o_stream_cork(_ctx->data.output);
+	return 0;
+}
+
+int imapc_save_continue(struct mail_save_context *_ctx)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+
+	if (ctx->failed)
+		return -1;
+
+	if (index_storage_save_continue(_ctx, ctx->input, NULL) < 0) {
+		ctx->failed = TRUE;
+		return -1;
+	}
+	return 0;
+}
+
+static void imapc_save_appenduid(struct imapc_save_context *ctx,
+				 const struct imapc_command_reply *reply,
+				 uint32_t *uid_r)
+{
+	const char *const *args;
+	uint32_t uid_validity, dest_uid;
+
+	*uid_r = 0;
+
+	/* <uidvalidity> <dest uid-set> */
+	args = t_strsplit(reply->resp_text_value, " ");
+	if (str_array_length(args) != 2)
+		return;
+
+	if (str_to_uint32(args[0], &uid_validity) < 0)
+		return;
+	if (ctx->dest_uid_validity == 0)
+		ctx->dest_uid_validity = uid_validity;
+	else if (ctx->dest_uid_validity != uid_validity)
+		return;
+
+	if (str_to_uint32(args[1], &dest_uid) == 0) {
+		seq_range_array_add_with_init(&ctx->dest_saved_uids,
+					      32, dest_uid);
+		*uid_r = dest_uid;
+	}
+}
+
+static void
+imapc_save_add_to_index(struct imapc_save_context *ctx, uint32_t uid)
+{
+	struct mail *_mail = ctx->ctx.dest_mail;
+	struct index_mail *imail = INDEX_MAIL(_mail);
+	uint32_t seq;
+
+	/* we'll temporarily append messages and at commit time expunge
+	   them all, since we can't guarantee that no one else has saved
+	   messages to remote server during our transaction */
+	mail_index_append(ctx->trans, uid, &seq);
+	mail_set_seq_saving(_mail, seq);
+	imail->data.no_caching = TRUE;
+	imail->data.forced_no_caching = TRUE;
+
+	if (ctx->fd != -1) {
+		struct imapc_mail *imapc_mail = IMAPC_MAIL(_mail);
+		imail->data.stream = i_stream_create_fd_autoclose(&ctx->fd, 0);
+		imapc_mail->header_fetched = TRUE;
+		imapc_mail->body_fetched = TRUE;
+		/* The saved stream wasn't actually read, but it needs to be
+		   set accessed to avoid assert-crash. */
+		_mail->mail_stream_accessed = TRUE;
+		imapc_mail_init_stream(imapc_mail);
+	}
+
+	ctx->save_count++;
+}
+
+static void imapc_save_callback(const struct imapc_command_reply *reply,
+				void *context)
+{
+	struct imapc_save_cmd_context *ctx = context;
+	uint32_t uid = 0;
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK) {
+		if (reply->resp_text_key != NULL &&
+		    strcasecmp(reply->resp_text_key, "APPENDUID") == 0)
+			imapc_save_appenduid(ctx->ctx, reply, &uid);
+		imapc_save_add_to_index(ctx->ctx, uid);
+		ctx->ret = 0;
+	} else if (imapc_storage_client_handle_auth_failure(ctx->ctx->mbox->storage->client)) {
+		ctx->ret = -1;
+	} else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		imapc_copy_error_from_reply(ctx->ctx->mbox->storage,
+					    MAIL_ERROR_PARAMS, reply);
+		ctx->ret = -1;
+	} else {
+		mailbox_set_critical(&ctx->ctx->mbox->box,
+			"imapc: APPEND failed: %s", reply->text_full);
+		ctx->ret = -1;
+	}
+	imapc_client_stop(ctx->ctx->mbox->storage->client->client);
+}
+
+static void
+imapc_save_noop_callback(const struct imapc_command_reply *reply ATTR_UNUSED,
+			 void *context)
+{
+	struct imapc_save_cmd_context *ctx = context;
+
+	/* we don't really care about the reply */
+	ctx->ret = 0;
+	imapc_client_stop(ctx->ctx->mbox->storage->client->client);
+}
+
+static void
+imapc_copy_rollback_store_callback(const struct imapc_command_reply *reply ATTR_UNUSED,
+				   void *context)
+{
+	struct imapc_save_context *ctx = context;
+	/* Can't do much about a non successful STORE here */
+	if (reply->state != IMAPC_COMMAND_STATE_OK) {
+		e_error(ctx->src_mbox->box.event,
+			"imapc: Failed to set \\Deleted flag for rolling back "
+			"failed copy: %s", reply->text_full);
+		ctx->src_mbox->rollback_pending = FALSE;
+		ctx->finished = TRUE;
+		ctx->failed = TRUE;
+	} else {
+		i_assert(ctx->src_mbox->rollback_pending);
+	}
+	/* No need stop the imapc client here there is always an additional
+	   expunge callback after this. */
+}
+
+static void
+imapc_copy_rollback_expunge_callback(const struct imapc_command_reply *reply ATTR_UNUSED,
+				     void *context)
+{
+	struct imapc_save_context *ctx = context;
+
+	/* Can't do much about a non successful EXPUNGE here */
+	if (reply->state != IMAPC_COMMAND_STATE_OK) {
+		e_error(ctx->src_mbox->box.event,
+			"imapc: Failed to expunge messages for rolling back "
+			"failed copy: %s", reply->text_full);
+		ctx->src_mbox->rollback_pending = FALSE;
+		ctx->finished = TRUE;
+		ctx->failed = TRUE;
+	} else {
+		ctx->finished = TRUE;
+		ctx->src_mbox->rollback_pending = FALSE;
+	}
+	imapc_client_stop(ctx->src_mbox->storage->client->client);
+}
+
+static void
+imapc_append_keywords(string_t *str, struct mail_keywords *kw)
+{
+	const ARRAY_TYPE(keywords) *kw_arr;
+	const char *kw_str;
+	unsigned int i;
+
+	kw_arr = mail_index_get_keywords(kw->index);
+	for (i = 0; i < kw->count; i++) {
+		kw_str = array_idx_elem(kw_arr, kw->idx[i]);
+		if (str_len(str) > 1)
+			str_append_c(str, ' ');
+		str_append(str, kw_str);
+	}
+}
+
+static int imapc_save_append(struct imapc_save_context *ctx)
+{
+	struct mail_save_context *_ctx = &ctx->ctx;
+	struct mail_save_data *mdata = &_ctx->data;
+	struct imapc_command *cmd;
+	struct imapc_save_cmd_context sctx;
+	struct istream *input;
+	const char *flags = "", *internaldate = "";
+
+	if (mdata->flags != 0 || mdata->keywords != NULL) {
+		string_t *str = t_str_new(64);
+
+		str_append(str, " (");
+		imap_write_flags(str, mdata->flags & ENUM_NEGATE(MAIL_RECENT),
+				 NULL);
+		if (mdata->keywords != NULL)
+			imapc_append_keywords(str, mdata->keywords);
+		str_append_c(str, ')');
+		flags = str_c(str);
+	}
+	if (mdata->received_date != (time_t)-1) {
+		internaldate = t_strdup_printf(" \"%s\"",
+			imap_to_datetime(mdata->received_date));
+	}
+
+	ctx->mbox->exists_received = FALSE;
+
+	input = i_stream_create_fd(ctx->fd, IO_BLOCK_SIZE);
+	sctx.ctx = ctx;
+	sctx.ret = -2;
+	cmd = imapc_client_cmd(ctx->mbox->storage->client->client,
+			       imapc_save_callback, &sctx);
+	imapc_command_sendf(cmd, "APPEND %s%1s%1s %p",
+		imapc_mailbox_get_remote_name(ctx->mbox),
+		flags, internaldate, input);
+	i_stream_unref(&input);
+	while (sctx.ret == -2)
+		imapc_mailbox_run(ctx->mbox);
+
+	if (sctx.ret == 0 && ctx->mbox->selected &&
+	    !ctx->mbox->exists_received) {
+		/* e.g. Courier doesn't send EXISTS reply before the tagged
+		   APPEND reply. That isn't exactly required by the IMAP RFC,
+		   but it makes the behavior better. See if NOOP finds
+		   the mail. */
+		sctx.ret = -2;
+		cmd = imapc_client_cmd(ctx->mbox->storage->client->client,
+				       imapc_save_noop_callback, &sctx);
+		imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+		imapc_command_send(cmd, "NOOP");
+		while (sctx.ret == -2)
+			imapc_mailbox_run(ctx->mbox);
+	}
+	return sctx.ret;
+}
+
+int imapc_save_finish(struct mail_save_context *_ctx)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+	struct mail_storage *storage = _ctx->transaction->box->storage;
+
+	ctx->finished = TRUE;
+
+	if (!ctx->failed) {
+		if (o_stream_finish(_ctx->data.output) < 0) {
+			if (!mail_storage_set_error_from_errno(storage)) {
+				mail_set_critical(_ctx->dest_mail,
+					"write(%s) failed: %s", ctx->temp_path,
+					o_stream_get_error(_ctx->data.output));
+			}
+			ctx->failed = TRUE;
+		}
+	}
+
+	if (!ctx->failed) {
+		if (imapc_save_append(ctx) < 0)
+			ctx->failed = TRUE;
+	}
+
+	o_stream_unref(&_ctx->data.output);
+	i_stream_unref(&ctx->input);
+	i_close_fd_path(&ctx->fd, ctx->temp_path);
+	i_free(ctx->temp_path);
+	index_save_context_free(_ctx);
+	return ctx->failed ? -1 : 0;
+}
+
+void imapc_save_cancel(struct mail_save_context *_ctx)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+
+	ctx->failed = TRUE;
+	(void)imapc_transaction_save_commit_pre(_ctx);
+	(void)imapc_save_finish(_ctx);
+}
+
+static void imapc_copy_bulk_finish(struct imapc_save_context *ctx)
+{
+	while (ctx->src_mbox != NULL && ctx->src_mbox->pending_copy_request != NULL)
+		imapc_mailbox_run_nofetch(ctx->src_mbox);
+}
+
+int imapc_transaction_save_commit_pre(struct mail_save_context *_ctx)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+	struct mail_transaction_commit_changes *changes =
+		_ctx->transaction->changes;
+	uint32_t i, last_seq;
+
+	i_assert(ctx->finished || ctx->failed);
+
+	/* expunge all added messages from index before commit */
+	last_seq = mail_index_view_get_messages_count(_ctx->transaction->view);
+	if (last_seq == 0)
+		return -1;
+	for (i = 0; i < ctx->save_count; i++)
+		mail_index_expunge(ctx->trans, last_seq - i);
+
+	if (!ctx->failed && array_is_created(&ctx->dest_saved_uids)) {
+		changes->uid_validity = ctx->dest_uid_validity;
+		array_append_array(&changes->saved_uids, &ctx->dest_saved_uids);
+	}
+	return 0;
+}
+
+int imapc_transaction_save_commit(struct mailbox_transaction_context *t)
+{
+       struct imapc_save_context *ctx = NULL;
+       struct imapc_mailbox *src_mbox = NULL;
+
+       if (t->save_ctx != NULL) {
+               ctx = IMAPC_SAVECTX(t->save_ctx);
+               src_mbox = ctx->src_mbox;
+       }
+
+       if (src_mbox != NULL && src_mbox->pending_copy_request != NULL) {
+	       /* If there is still a copy command to send flush it now */
+	       imapc_mail_copy_bulk_flush(src_mbox);
+	       imapc_copy_bulk_finish(ctx);
+       }
+
+       if (ctx != NULL)
+	       return ctx->failed ? -1 : 0;
+       return 0;
+}
+
+void imapc_transaction_save_commit_post(struct mail_save_context *_ctx,
+					struct mail_index_transaction_commit_result *result ATTR_UNUSED)
+{
+	imapc_transaction_save_rollback(_ctx);
+}
+
+static void
+imapc_expunge_construct_cmd_str(string_t *store_cmd,
+				string_t *expunge_cmd,
+				string_t *uids)
+{
+	str_append(store_cmd, "UID STORE ");
+	str_append_str(store_cmd, uids);
+	str_append(store_cmd, " +FLAGS (\\Deleted)");
+	str_append(expunge_cmd, "UID EXPUNGE ");
+	str_append_str(expunge_cmd, uids);
+	/* Clear already appened uids */
+	str_truncate(uids, 0);
+}
+
+static void
+imapc_expunge_send_cmd_str(struct imapc_save_context *ctx,
+			   string_t *uids)
+{
+	struct imapc_command *store_cmd, *expunge_cmd;
+
+	string_t *store_cmd_str, *expunge_cmd_str;
+	store_cmd_str = t_str_new(128);
+	expunge_cmd_str = t_str_new(128);
+
+	imapc_expunge_construct_cmd_str(store_cmd_str, expunge_cmd_str, uids);
+	/* Make sure line length is less than 8k */
+	i_assert(str_len(store_cmd_str) < IMAPC_SERVER_CMDLINE_MAX_LEN);
+	i_assert(str_len(expunge_cmd_str) < IMAPC_SERVER_CMDLINE_MAX_LEN);
+
+	store_cmd = imapc_client_mailbox_cmd(ctx->src_mbox->client_box,
+					     imapc_copy_rollback_store_callback,
+					     ctx);
+	expunge_cmd = imapc_client_mailbox_cmd(ctx->src_mbox->client_box,
+					       imapc_copy_rollback_expunge_callback,
+					       ctx);
+	ctx->src_mbox->rollback_pending = TRUE;
+	imapc_command_send(store_cmd, str_c(store_cmd_str));
+	imapc_command_send(expunge_cmd, str_c(expunge_cmd_str));
+}
+
+static void
+imapc_rollback_send_expunge(struct imapc_save_context *ctx)
+{
+	string_t *uids_str;
+	struct seqset_builder *seqset_builder;
+	struct seq_range_iter iter;
+	unsigned int i = 0;
+	uint32_t uid;
+
+	if (!array_not_empty(&ctx->src_mbox->copy_rollback_expunge_uids))
+		return;
+
+	uids_str = t_str_new(128);
+	seqset_builder = seqset_builder_init(uids_str);
+	seq_range_array_iter_init(&iter, &ctx->src_mbox->copy_rollback_expunge_uids);
+
+	/* Iterate over all uids that must be rolled back */
+	while (seq_range_array_iter_nth(&iter, i++, &uid)) {
+		/* Try to add the to the seqset builder while respecting
+		   the maximum length of IMAPC_SERVER_CMDLINE_MAX_LEN. */
+		if (!seqset_builder_try_add(seqset_builder,
+					    IMAPC_SERVER_CMDLINE_MAX_LEN -
+					    strlen("UID STORE  +FLAGS (\\Deleted)"),
+					    uid)) {
+			/* Maximum length is reached send the rollback
+			   and wait for it to be finished. */
+			imapc_expunge_send_cmd_str(ctx, uids_str);
+			while (ctx->src_mbox->rollback_pending)
+				imapc_mailbox_run_nofetch(ctx->src_mbox);
+
+			/* Truncate the uids_str and create a new
+			   seqset_builder for the next command */
+			seqset_builder_deinit(&seqset_builder);
+			str_truncate(uids_str, 0);
+			seqset_builder = seqset_builder_init(uids_str);
+			/* Make sure the current uid which is part of
+			   the next uid_str */
+			seqset_builder_add(seqset_builder, uid);
+		}
+	}
+	if (str_len(uids_str) > 0)
+		imapc_expunge_send_cmd_str(ctx, uids_str);
+	while (ctx->src_mbox->rollback_pending)
+		imapc_mailbox_run_nofetch(ctx->src_mbox);
+}
+
+static void imapc_copy_bulk_ctx_deinit(struct imapc_save_context *ctx)
+{
+	/* Clean up the pending copy and the context attached to it */
+	str_truncate(ctx->src_mbox->pending_copy_cmd, 0);
+	i_free(ctx->src_mbox->copy_dest_box);
+}
+
+void imapc_transaction_save_rollback(struct mail_save_context *_ctx)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+
+       if ((ctx->src_mbox != NULL && ctx->src_mbox->pending_copy_request != NULL) ||
+	   !ctx->finished) {
+	       /* There is still a pending copy which should not be send
+		  as rollback() is called or the transaction has not yet
+		  finished and rollback is called */
+	       ctx->failed = TRUE;
+	       (void)imapc_transaction_save_commit_pre(_ctx);
+
+	       i_assert(ctx->finished || ctx->src_mbox != NULL);
+	       /* Clean up the pending copy and the context attached to it */
+	       if (ctx->src_mbox != NULL) {
+		       if (ctx->src_mbox->pending_copy_request != NULL) {
+			       seqset_builder_deinit(&ctx->src_mbox->pending_copy_request->uidset_builder);
+			       i_free(ctx->src_mbox->pending_copy_request);
+		       }
+		       imapc_copy_bulk_ctx_deinit(ctx);
+		       imapc_client_stop(ctx->src_mbox->storage->client->client);
+	       }
+       }
+
+	/* Expunge all added messages from index */
+	if (ctx->failed && array_is_created(&ctx->dest_saved_uids)) {
+		i_assert(ctx->src_mbox != NULL);
+		seq_range_array_merge(&ctx->src_mbox->copy_rollback_expunge_uids, &ctx->dest_saved_uids);
+		/* Make sure context is not finished already */
+		ctx->finished = FALSE;
+		imapc_rollback_send_expunge(ctx);
+		array_free(&ctx->dest_saved_uids);
+	}
+
+	if (ctx->finished || ctx->failed) {
+		array_free(&ctx->dest_saved_uids);
+		i_free(ctx);
+	}
+}
+
+static bool imapc_save_copyuid(struct imapc_save_context *ctx,
+			       const struct imapc_command_reply *reply,
+			       uint32_t *uid_r)
+{
+	ARRAY_TYPE(seq_range) dest_uidset, source_uidset;
+	struct seq_range_iter iter;
+	const char *const *args;
+	uint32_t uid_validity;
+
+	*uid_r = 0;
+
+	/* <uidvalidity> <source uid-set> <dest uid-set> */
+	args = t_strsplit(reply->resp_text_value, " ");
+	if (str_array_length(args) != 3)
+		return FALSE;
+
+	if (str_to_uint32(args[0], &uid_validity) < 0)
+		return FALSE;
+	if (ctx->dest_uid_validity == 0)
+		ctx->dest_uid_validity = uid_validity;
+	else if (ctx->dest_uid_validity != uid_validity)
+		return FALSE;
+
+	t_array_init(&source_uidset, 8);
+	t_array_init(&dest_uidset, 8);
+
+	if (imap_seq_set_nostar_parse(args[1], &source_uidset) < 0)
+		return FALSE;
+	if (imap_seq_set_nostar_parse(args[2], &dest_uidset) < 0)
+		return FALSE;
+
+	if (!array_is_created(&ctx->dest_saved_uids))
+		i_array_init(&ctx->dest_saved_uids, 8);
+
+	seq_range_array_merge(&ctx->dest_saved_uids, &dest_uidset);
+
+	seq_range_array_iter_init(&iter, &dest_uidset);
+	(void)seq_range_array_iter_nth(&iter, 0, uid_r);
+	return TRUE;
+}
+
+static void imapc_copy_set_error(struct imapc_save_context *sctx,
+				 const struct imapc_command_reply *reply)
+{
+	sctx->failed = TRUE;
+
+	if (reply->state != IMAPC_COMMAND_STATE_BAD)
+		imapc_copy_error_from_reply(sctx->mbox->storage,
+					    MAIL_ERROR_PARAMS, reply);
+	else
+		mailbox_set_critical(&sctx->mbox->box,
+				     "imapc: COPY failed: %s",
+				     reply->text_full);
+}
+
+static void
+imapc_copy_simple_callback(const struct imapc_command_reply *reply,
+			   void *context)
+{
+	struct imapc_save_cmd_context *ctx = context;
+	uint32_t uid = 0;
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK) {
+		if (reply->resp_text_key != NULL &&
+		    strcasecmp(reply->resp_text_key, "COPYUID") == 0)
+			imapc_save_copyuid(ctx->ctx, reply, &uid);
+		imapc_save_add_to_index(ctx->ctx, uid);
+		ctx->ret = 0;
+	} else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		imapc_copy_error_from_reply(ctx->ctx->mbox->storage,
+					    MAIL_ERROR_PARAMS, reply);
+		ctx->ret = -1;
+	} else {
+		mailbox_set_critical(&ctx->ctx->mbox->box,
+			"imapc: COPY failed: %s", reply->text_full);
+		ctx->ret = -1;
+	}
+	imapc_client_stop(ctx->ctx->mbox->storage->client->client);
+}
+
+static int
+imapc_copy_simple(struct mail_save_context *_ctx, struct mail *mail)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+	struct mailbox_transaction_context *_t = _ctx->transaction;
+	struct imapc_save_cmd_context sctx;
+	struct imapc_command *cmd;
+
+	sctx.ret = -2;
+	sctx.ctx = ctx;
+	cmd = imapc_client_mailbox_cmd(ctx->src_mbox->client_box,
+				       imapc_copy_simple_callback,
+				       &sctx);
+	imapc_command_sendf(cmd, "UID COPY %u %s", mail->uid, _t->box->name);
+	while (sctx.ret == -2)
+		imapc_mailbox_run(ctx->src_mbox);
+	ctx->finished = TRUE;
+	return sctx.ret;
+}
+
+static void imapc_copy_bulk_callback(const struct imapc_command_reply *reply,
+				     void *context)
+{
+	struct imapc_copy_request *request = context;
+	struct imapc_save_context *ctx = request->sctx;
+	struct imapc_mailbox *mbox = ctx->src_mbox;
+	unsigned int uid;
+
+	i_assert(mbox != NULL);
+	i_assert(request == mbox->pending_copy_request);
+
+	/* Check the reply state and add uid's to index and
+	   dest_saved_uids. */
+	if (ctx->failed) {
+		/* If the saving already failed try to find UIDs already
+		   copied from the reply so that rollback can expunge
+		   them */
+		if (null_strcasecmp(reply->resp_text_key, "COPYUID") == 0) {
+			(void)imapc_save_copyuid(ctx, reply, &uid);
+			imapc_transaction_save_rollback(&ctx->ctx);
+		}
+	} else if (reply->state == IMAPC_COMMAND_STATE_OK) {
+		if (reply->resp_text_key != NULL &&
+		   strcasecmp(reply->resp_text_key, "COPYUID") == 0 &&
+		   imapc_save_copyuid(ctx, reply, &uid)) {
+			ctx->finished = TRUE;
+		}
+	} else {
+		imapc_copy_set_error(ctx, reply);
+	}
+
+	ctx->src_mbox->pending_copy_request = NULL;
+	i_free(request);
+	imapc_client_stop(mbox->storage->client->client);
+}
+
+static void imapc_mail_copy_bulk_flush(struct imapc_mailbox *mbox)
+{
+	struct imapc_command *cmd;
+
+	i_assert(mbox != NULL);
+	i_assert(mbox->pending_copy_request != NULL);
+	i_assert(mbox->client_box != NULL);
+
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_copy_bulk_callback,
+				       mbox->pending_copy_request);
+
+	seqset_builder_deinit(&mbox->pending_copy_request->uidset_builder);
+
+	str_append(mbox->pending_copy_cmd, " ");
+	imap_append_astring(mbox->pending_copy_cmd, mbox->copy_dest_box);
+
+	imapc_command_send(cmd, str_c(mbox->pending_copy_cmd));
+
+	imapc_copy_bulk_ctx_deinit(mbox->pending_copy_request->sctx);
+}
+
+static bool
+imapc_mail_copy_bulk_try_merge(struct imapc_mailbox *mbox, uint32_t uid,
+			       const char *box)
+{
+	i_assert(str_begins(str_c(mbox->pending_copy_cmd), "UID COPY "));
+
+	if (strcmp(box, mbox->copy_dest_box) != 0) {
+		/* Not the same mailbox merging not possible */
+		return FALSE;
+	}
+	return seqset_builder_try_add(mbox->pending_copy_request->uidset_builder,
+				      IMAPC_SERVER_CMDLINE_MAX_LEN, uid);
+}
+
+static void
+imapc_mail_copy_bulk_delayed_send_or_merge(struct imapc_save_context *ctx,
+					   uint32_t uid,
+					   const char *box)
+{
+	struct imapc_mailbox *mbox = ctx->src_mbox;
+
+	if (mbox->pending_copy_request != NULL &&
+	    !imapc_mail_copy_bulk_try_merge(mbox, uid, box)) {
+		/* send the previous COPY and create new one after
+		   waiting for this one to be finished. */
+		imapc_mail_copy_bulk_flush(mbox);
+		imapc_copy_bulk_finish(mbox->pending_copy_request->sctx);
+	}
+	if (mbox->pending_copy_request == NULL) {
+		mbox->pending_copy_request =
+			i_new(struct imapc_copy_request, 1);
+		str_printfa(mbox->pending_copy_cmd, "UID COPY ");
+		mbox->pending_copy_request->uidset_builder =
+			seqset_builder_init(mbox->pending_copy_cmd);
+		seqset_builder_add(mbox->pending_copy_request->uidset_builder,
+				   uid);
+		mbox->copy_dest_box = i_strdup(box);
+	} else {
+		i_assert(mbox->pending_copy_request->sctx == ctx);
+	}
+	mbox->pending_copy_request->sctx = ctx;
+}
+
+static int
+imapc_copy_bulk(struct imapc_save_context *ctx, struct mail *mail)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(ctx->ctx.transaction->box);
+
+	imapc_mail_copy_bulk_delayed_send_or_merge(ctx, mail->uid,
+						   imapc_mailbox_get_remote_name(mbox));
+	imapc_save_add_to_index(ctx, 0);
+
+	return ctx->failed ? -1 : 0;
+}
+
+static bool imapc_is_mail_expunged(struct imapc_mailbox *mbox, uint32_t uid)
+{
+	if (array_is_created(&mbox->delayed_expunged_uids) &&
+	    seq_range_exists(&mbox->delayed_expunged_uids, uid))
+		return TRUE;
+	if (mbox->delayed_sync_trans == NULL)
+		return FALSE;
+
+	struct mail_index_view *view =
+		mail_index_transaction_get_view(mbox->delayed_sync_trans);
+	uint32_t seq;
+	return mail_index_lookup_seq(view, uid, &seq) &&
+		mail_index_transaction_is_expunged(mbox->delayed_sync_trans, seq);
+}
+
+int imapc_copy(struct mail_save_context *_ctx, struct mail *mail)
+{
+	struct imapc_save_context *ctx = IMAPC_SAVECTX(_ctx);
+	struct mailbox_transaction_context *_t = _ctx->transaction;
+	struct imapc_msgmap *src_msgmap;
+	uint32_t rseq;
+	int ret;
+
+	i_assert((_t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	if (_t->box->storage == mail->box->storage) {
+		/* Currently we don't support copying mails from multiple
+		   different source mailboxes within the same transaction. */
+		i_assert(ctx->src_mbox == NULL || &ctx->src_mbox->box == mail->box);
+		ctx->src_mbox = IMAPC_MAILBOX(mail->box);
+		if (!mail->expunged && imapc_is_mail_expunged(ctx->mbox, mail->uid))
+			mail_set_expunged(mail);
+		/* same server, we can use COPY for the mail */
+		src_msgmap =
+			imapc_client_mailbox_get_msgmap(ctx->src_mbox->client_box);
+		if (mail->expunged ||
+		    !imapc_msgmap_uid_to_rseq(src_msgmap, mail->uid, &rseq)) {
+			mail_storage_set_error(mail->box->storage,
+					       MAIL_ERROR_EXPUNGED,
+					       "Some of the requested messages no longer exist.");
+			ctx->finished = TRUE;
+			index_save_context_free(_ctx);
+			return -1;
+		}
+		/* Mail has not been expunged and can be copied. */
+		if (ctx->mbox->capabilities == 0) {
+			/* The destination mailbox has not yet been selected
+			   so the capabilities are unknown */
+			if (imapc_client_get_capabilities(ctx->mbox->storage->client->client,
+						      &ctx->mbox->capabilities) < 0) {
+				mail_storage_set_error(mail->box->storage,
+						       MAIL_ERROR_UNAVAILABLE,
+						       "Failed to determine capabilities for mailbox.");
+				ctx->finished = TRUE;
+				index_save_context_free(_ctx);
+				return -1;
+			}
+		}
+		if ((ctx->mbox->capabilities & IMAPC_CAPABILITY_UIDPLUS) != 0)
+			ret = imapc_copy_bulk(ctx, mail);
+		else
+			ret = imapc_copy_simple(_ctx, mail);
+		index_save_context_free(_ctx);
+		return ret;
+	}
+	return mail_storage_copy(_ctx, mail);
+}
diff --git a/src/lib-storage/index/imapc/imapc-search.c b/src/lib-storage/index/imapc/imapc-search.c
new file mode 100644
index 0000000..e395919
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-search.c
@@ -0,0 +1,330 @@
+/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "imap-arg.h"
+#include "imap-seqset.h"
+#include "imap-util.h"
+#include "mail-search.h"
+#include "imapc-msgmap.h"
+#include "imapc-storage.h"
+#include "imapc-search.h"
+
+#define IMAPC_SEARCHCTX(obj) \
+	MODULE_CONTEXT(obj, imapc_storage_module)
+
+struct imapc_search_context {
+	union mail_search_module_context module_ctx;
+
+	ARRAY_TYPE(seq_range) rseqs;
+	struct seq_range_iter iter;
+	unsigned int n;
+	bool finished;
+	bool success;
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(imapc_storage_module,
+				  &mail_storage_module_register);
+
+static bool
+imapc_build_search_query_args(struct imapc_mailbox *mbox,
+			      const struct mail_search_arg *args,
+			      bool parent_or, string_t *str);
+
+static bool imapc_search_is_fast_local(const struct mail_search_arg *args)
+{
+	const struct mail_search_arg *arg;
+
+	for (arg = args; arg != NULL; arg = arg->next) {
+		switch (arg->type) {
+		case SEARCH_OR:
+		case SEARCH_SUB:
+			if (!imapc_search_is_fast_local(arg->value.subargs))
+				return FALSE;
+			break;
+		case SEARCH_ALL:
+		case SEARCH_SEQSET:
+		case SEARCH_UIDSET:
+		case SEARCH_FLAGS:
+		case SEARCH_KEYWORDS:
+		case SEARCH_MODSEQ:
+		case SEARCH_MAILBOX:
+		case SEARCH_MAILBOX_GUID:
+		case SEARCH_MAILBOX_GLOB:
+		case SEARCH_REAL_UID:
+			break;
+		default:
+			return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+static bool
+imapc_build_search_query_arg(struct imapc_mailbox *mbox,
+			     const struct mail_search_arg *arg,
+			     string_t *str)
+{
+	struct mail_search_arg arg2 = *arg;
+	const char *error;
+
+	if (arg->match_not)
+		str_append(str, "NOT ");
+	arg2.match_not = FALSE;
+	arg = &arg2;
+
+	switch (arg->type) {
+	case SEARCH_OR:
+		imapc_build_search_query_args(mbox, arg->value.subargs, TRUE, str);
+		return TRUE;
+	case SEARCH_SUB:
+		str_append_c(str, '(');
+		imapc_build_search_query_args(mbox, arg->value.subargs, FALSE, str);
+		str_append_c(str, ')');
+		return TRUE;
+	case SEARCH_SEQSET:
+		/* translate to UIDs */
+		T_BEGIN {
+			ARRAY_TYPE(seq_range) uids;
+
+			t_array_init(&uids, 64);
+			mailbox_get_uid_range(&mbox->box, &arg->value.seqset,
+					      &uids);
+			str_append(str, "UID ");
+			imap_write_seq_range(str, &uids);
+		} T_END;
+		return TRUE;
+	case SEARCH_BEFORE:
+	case SEARCH_SINCE:
+	case SEARCH_ON:
+		if (arg->type != SEARCH_ON &&
+		    (mbox->capabilities & IMAPC_CAPABILITY_WITHIN) == 0) {
+			/* a bit kludgy way to check this.. */
+			size_t pos = str_len(str);
+			if (!mail_search_arg_to_imap(str, arg, &error))
+				return FALSE;
+			if (strncasecmp(str_c(str) + pos, "OLDER", 5) == 0 ||
+			    strncasecmp(str_c(str) + pos, "YOUNGER", 7) == 0)
+				return FALSE;
+			return TRUE;
+		}
+		if (arg->value.date_type == MAIL_SEARCH_DATE_TYPE_SAVED &&
+		    (mbox->capabilities & IMAPC_CAPABILITY_SAVEDATE) == 0) {
+			/* Fall back to internal date if save date is not
+			   supported. */
+			arg2.value.date_type = MAIL_SEARCH_DATE_TYPE_RECEIVED;
+		}
+		/* fall through */
+	case SEARCH_ALL:
+	case SEARCH_UIDSET:
+	case SEARCH_FLAGS:
+	case SEARCH_KEYWORDS:
+	case SEARCH_SMALLER:
+	case SEARCH_LARGER:
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+	case SEARCH_BODY:
+	case SEARCH_TEXT:
+		return mail_search_arg_to_imap(str, arg, &error);
+	/* extensions */
+	case SEARCH_MODSEQ:
+		if ((mbox->capabilities & IMAPC_CAPABILITY_CONDSTORE) == 0)
+			return FALSE;
+		return mail_search_arg_to_imap(str, arg, &error);
+	case SEARCH_SAVEDATESUPPORTED:
+		if ((mbox->capabilities & IMAPC_CAPABILITY_SAVEDATE) == 0)
+			return FALSE;
+		return mail_search_arg_to_imap(str, arg, &error);
+	case SEARCH_INTHREAD:
+	case SEARCH_GUID:
+	case SEARCH_MAILBOX:
+	case SEARCH_MAILBOX_GUID:
+	case SEARCH_MAILBOX_GLOB:
+	case SEARCH_REAL_UID:
+	case SEARCH_MIMEPART:
+		/* not supported for now */
+		break;
+	}
+	return FALSE;
+}
+
+static bool
+imapc_build_search_query_args(struct imapc_mailbox *mbox,
+			      const struct mail_search_arg *args,
+			      bool parent_or, string_t *str)
+{
+	const struct mail_search_arg *arg;
+
+	for (arg = args; arg != NULL; arg = arg->next) {
+		if (parent_or && arg->next != NULL)
+			str_append(str, "OR ");
+		if (!imapc_build_search_query_arg(mbox, arg, str))
+			return FALSE;
+		str_append_c(str, ' ');
+	}
+	str_truncate(str, str_len(str)-1);
+	return TRUE;
+}
+
+static bool imapc_build_search_query(struct imapc_mailbox *mbox,
+				     const struct mail_search_args *args,
+				     const char **query_r)
+{
+	string_t *str = t_str_new(128);
+
+	if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_SEARCH)) {
+		/* SEARCH command passthrough not enabled */
+		return FALSE;
+	}
+	if (imapc_search_is_fast_local(args->args))
+		return FALSE;
+
+	if ((mbox->capabilities & IMAPC_CAPABILITY_ESEARCH) != 0)
+		str_append(str, "SEARCH RETURN (ALL) ");
+	else
+		str_append(str, "UID SEARCH ");
+	if (!imapc_build_search_query_args(mbox, args->args, FALSE, str))
+		return FALSE;
+	*query_r = str_c(str);
+	return TRUE;
+}
+
+static void imapc_search_callback(const struct imapc_command_reply *reply,
+				  void *context)
+{
+	struct mail_search_context *ctx = context;
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(ctx->transaction->box);
+	struct imapc_search_context *ictx = IMAPC_SEARCHCTX(ctx);
+	i_assert(ictx != NULL);
+
+	ictx->finished = TRUE;
+	if (reply->state == IMAPC_COMMAND_STATE_OK) {
+		seq_range_array_iter_init(&ictx->iter, &ictx->rseqs);
+		ictx->success = TRUE;
+	} else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		mail_storage_set_internal_error(mbox->box.storage);
+	} else {
+		mailbox_set_critical(&mbox->box,
+			"imapc: Command failed: %s", reply->text_full);
+	}
+	imapc_client_stop(mbox->storage->client->client);
+}
+
+struct mail_search_context *
+imapc_search_init(struct mailbox_transaction_context *t,
+		  struct mail_search_args *args,
+		  const enum mail_sort_type *sort_program,
+		  enum mail_fetch_field wanted_fields,
+		  struct mailbox_header_lookup_ctx *wanted_headers)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(t->box);
+	struct mail_search_context *ctx;
+	struct imapc_search_context *ictx;
+	struct imapc_command *cmd;
+	const char *search_query;
+
+	ctx = index_storage_search_init(t, args, sort_program,
+					wanted_fields, wanted_headers);
+
+	if (!imapc_build_search_query(mbox, args, &search_query)) {
+		/* can't optimize this with SEARCH */
+		return ctx;
+	}
+
+	ictx = i_new(struct imapc_search_context, 1);
+	i_array_init(&ictx->rseqs, 64);
+	MODULE_CONTEXT_SET(ctx, imapc_storage_module, ictx);
+
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_search_callback, ctx);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_send(cmd, search_query);
+
+	i_assert(mbox->search_ctx == NULL);
+	mbox->search_ctx = ictx;
+	while (!ictx->finished)
+		imapc_client_run(mbox->storage->client->client);
+	mbox->search_ctx = NULL;
+	return ctx;
+}
+
+static void imapc_search_set_matches(struct mail_search_arg *args)
+{
+	for (; args != NULL; args = args->next) {
+		if (args->type == SEARCH_OR ||
+		    args->type == SEARCH_SUB)
+			imapc_search_set_matches(args->value.subargs);
+		args->match_always = TRUE;
+		args->result = 1;
+	}
+}
+
+bool imapc_search_next_update_seq(struct mail_search_context *ctx)
+{
+	struct imapc_search_context *ictx = IMAPC_SEARCHCTX(ctx);
+
+	if (ictx == NULL || !ictx->success)
+		return index_storage_search_next_update_seq(ctx);
+
+	if (!seq_range_array_iter_nth(&ictx->iter, ictx->n++, &ctx->seq))
+		return FALSE;
+	ctx->progress_cur = ctx->seq;
+
+	imapc_search_set_matches(ctx->args->args);
+	return TRUE;
+}
+
+int imapc_search_deinit(struct mail_search_context *ctx)
+{
+	struct imapc_search_context *ictx = IMAPC_SEARCHCTX(ctx);
+
+	if (ictx != NULL) {
+		array_free(&ictx->rseqs);
+		i_free(ictx);
+	}
+	return index_storage_search_deinit(ctx);
+}
+
+void imapc_search_reply_search(const struct imap_arg *args,
+			       struct imapc_mailbox *mbox)
+{
+	struct imapc_msgmap *msgmap =
+		imapc_client_mailbox_get_msgmap(mbox->client_box);
+	const char *atom;
+	uint32_t uid, rseq;
+
+	if (mbox->search_ctx == NULL) {
+		i_error("Unexpected SEARCH reply");
+		return;
+	}
+
+	/* we're doing UID SEARCH, so need to convert UIDs to sequences */
+	for (unsigned int i = 0; args[i].type != IMAP_ARG_EOL; i++) {
+		if (!imap_arg_get_atom(&args[i], &atom) ||
+		    str_to_uint32(atom, &uid) < 0 || uid == 0) {
+			i_error("Invalid SEARCH reply");
+			break;
+		}
+		if (imapc_msgmap_uid_to_rseq(msgmap, uid, &rseq))
+			seq_range_array_add(&mbox->search_ctx->rseqs, rseq);
+	}
+}
+
+void imapc_search_reply_esearch(const struct imap_arg *args,
+				struct imapc_mailbox *mbox)
+{
+	const char *atom;
+
+	if (mbox->search_ctx == NULL) {
+		i_error("Unexpected ESEARCH reply");
+		return;
+	}
+
+	/* It should contain ALL <seqset> or nonexistent if nothing matched */
+	if (args[0].type != IMAP_ARG_EOL &&
+	    (!imap_arg_atom_equals(&args[0], "ALL") ||
+	     !imap_arg_get_atom(&args[1], &atom) ||
+	     imap_seq_set_nostar_parse(atom, &mbox->search_ctx->rseqs) < 0))
+		i_error("Invalid ESEARCH reply");
+}
diff --git a/src/lib-storage/index/imapc/imapc-search.h b/src/lib-storage/index/imapc/imapc-search.h
new file mode 100644
index 0000000..0966569
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-search.h
@@ -0,0 +1,18 @@
+#ifndef IMAPC_SEARCH_H
+#define IMAPC_SEARCH_H
+
+struct mail_search_context *
+imapc_search_init(struct mailbox_transaction_context *t,
+		  struct mail_search_args *args,
+		  const enum mail_sort_type *sort_program,
+		  enum mail_fetch_field wanted_fields,
+		  struct mailbox_header_lookup_ctx *wanted_headers);
+bool imapc_search_next_update_seq(struct mail_search_context *ctx);
+int imapc_search_deinit(struct mail_search_context *ctx);
+
+void imapc_search_reply_search(const struct imap_arg *args,
+			       struct imapc_mailbox *mbox);
+void imapc_search_reply_esearch(const struct imap_arg *args,
+				struct imapc_mailbox *mbox);
+
+#endif
diff --git a/src/lib-storage/index/imapc/imapc-settings.c b/src/lib-storage/index/imapc/imapc-settings.c
new file mode 100644
index 0000000..6183f8f
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-settings.c
@@ -0,0 +1,173 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "settings-parser.h"
+#include "mail-storage-settings.h"
+#include "imapc-settings.h"
+
+#include <stddef.h>
+
+#undef DEF
+#define DEF(type, name) \
+	SETTING_DEFINE_STRUCT_##type(#name, name, struct imapc_settings)
+
+static bool imapc_settings_check(void *_set, pool_t pool, const char **error_r);
+
+static const struct setting_define imapc_setting_defines[] = {
+	DEF(STR, imapc_host),
+	DEF(IN_PORT, imapc_port),
+
+	DEF(STR_VARS, imapc_user),
+	DEF(STR_VARS, imapc_master_user),
+	DEF(STR, imapc_password),
+	DEF(STR, imapc_sasl_mechanisms),
+
+	DEF(ENUM, imapc_ssl),
+	DEF(BOOL, imapc_ssl_verify),
+
+	DEF(STR, imapc_features),
+	DEF(STR, imapc_rawlog_dir),
+	DEF(STR, imapc_list_prefix),
+	DEF(TIME, imapc_cmd_timeout),
+	DEF(TIME, imapc_max_idle_time),
+	DEF(UINT, imapc_connection_retry_count),
+	DEF(TIME_MSECS, imapc_connection_retry_interval),
+	DEF(SIZE, imapc_max_line_length),
+
+	DEF(STR, pop3_deleted_flag),
+
+	SETTING_DEFINE_LIST_END
+};
+
+static const struct imapc_settings imapc_default_settings = {
+	.imapc_host = "",
+	.imapc_port = 143,
+
+	.imapc_user = "",
+	.imapc_master_user = "",
+	.imapc_password = "",
+	.imapc_sasl_mechanisms = "",
+
+	.imapc_ssl = "no:imaps:starttls",
+	.imapc_ssl_verify = TRUE,
+
+	.imapc_features = "",
+	.imapc_rawlog_dir = "",
+	.imapc_list_prefix = "",
+	.imapc_cmd_timeout = 5*60,
+	.imapc_max_idle_time = 60*29,
+	.imapc_connection_retry_count = 1,
+	.imapc_connection_retry_interval = 1000,
+	.imapc_max_line_length = 0,
+
+	.pop3_deleted_flag = ""
+};
+
+static const struct setting_parser_info imapc_setting_parser_info = {
+	.module_name = "imapc",
+	.defines = imapc_setting_defines,
+	.defaults = &imapc_default_settings,
+
+	.type_offset = SIZE_MAX,
+	.struct_size = sizeof(struct imapc_settings),
+
+	.parent_offset = SIZE_MAX,
+	.parent = &mail_user_setting_parser_info,
+
+	.check_func = imapc_settings_check
+};
+
+const struct setting_parser_info *imapc_get_setting_parser_info(void)
+{
+	return &imapc_setting_parser_info;
+}
+
+/* <settings checks> */
+struct imapc_feature_list {
+	const char *name;
+	enum imapc_features num;
+};
+
+static const struct imapc_feature_list imapc_feature_list[] = {
+	{ "rfc822.size", IMAPC_FEATURE_RFC822_SIZE },
+	{ "guid-forced", IMAPC_FEATURE_GUID_FORCED },
+	{ "fetch-headers", IMAPC_FEATURE_FETCH_HEADERS },
+	{ "gmail-migration", IMAPC_FEATURE_GMAIL_MIGRATION },
+	{ "search", IMAPC_FEATURE_SEARCH },
+	{ "zimbra-workarounds", IMAPC_FEATURE_ZIMBRA_WORKAROUNDS },
+	{ "no-examine", IMAPC_FEATURE_NO_EXAMINE },
+	{ "proxyauth", IMAPC_FEATURE_PROXYAUTH },
+	{ "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS },
+	{ "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS },
+	{ "modseq", IMAPC_FEATURE_MODSEQ },
+	{ "delay-login", IMAPC_FEATURE_DELAY_LOGIN },
+	{ "fetch-bodystructure", IMAPC_FEATURE_FETCH_BODYSTRUCTURE },
+	{ "send-id", IMAPC_FEATURE_SEND_ID },
+	{ "fetch-empty-is-expunged", IMAPC_FEATURE_FETCH_EMPTY_IS_EXPUNGED },
+	{ "no-msn-updates", IMAPC_FEATURE_NO_MSN_UPDATES },
+	{ "acl", IMAPC_FEATURE_ACL },
+	{ NULL, 0 }
+};
+
+static int
+imapc_settings_parse_throttle(struct imapc_settings *set,
+			      const char *throttle_str, const char **error_r)
+{
+	const char *const *tmp;
+
+	tmp = t_strsplit(throttle_str, ":");
+	if (str_array_length(tmp) != 3 ||
+	    str_to_uint(tmp[0], &set->throttle_init_msecs) < 0 ||
+	    str_to_uint(tmp[1], &set->throttle_max_msecs) < 0 ||
+	    str_to_uint(tmp[2], &set->throttle_shrink_min_msecs) < 0) {
+		*error_r = "imapc_features: Invalid throttle settings";
+		return -1;
+	}
+	return 0;
+}
+
+static int
+imapc_settings_parse_features(struct imapc_settings *set,
+			      const char **error_r)
+{
+        enum imapc_features features = 0;
+        const struct imapc_feature_list *list;
+	const char *const *str;
+
+        str = t_strsplit_spaces(set->imapc_features, " ,");
+	for (; *str != NULL; str++) {
+		list = imapc_feature_list;
+		for (; list->name != NULL; list++) {
+			if (strcasecmp(*str, list->name) == 0) {
+				features |= list->num;
+				break;
+			}
+		}
+		if (strncasecmp(*str, "throttle:", 9) == 0) {
+			if (imapc_settings_parse_throttle(set, *str + 9, error_r) < 0)
+				return -1;
+			continue;
+		}
+		if (list->name == NULL) {
+			*error_r = t_strdup_printf("imapc_features: "
+				"Unknown feature: %s", *str);
+			return -1;
+		}
+	}
+	set->parsed_features = features;
+	return 0;
+}
+
+static bool imapc_settings_check(void *_set, pool_t pool ATTR_UNUSED,
+				 const char **error_r)
+{
+	struct imapc_settings *set = _set;
+
+	if (set->imapc_max_idle_time == 0) {
+		*error_r = "imapc_max_idle_time must not be 0";
+		return FALSE;
+	}
+	if (imapc_settings_parse_features(set, error_r) < 0)
+		return FALSE;
+	return TRUE;
+}
diff --git a/src/lib-storage/index/imapc/imapc-settings.h b/src/lib-storage/index/imapc/imapc-settings.h
new file mode 100644
index 0000000..e4c1da1
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-settings.h
@@ -0,0 +1,63 @@
+#ifndef IMAPC_SETTINGS_H
+#define IMAPC_SETTINGS_H
+
+#include "net.h"
+
+/* <settings checks> */
+enum imapc_features {
+	IMAPC_FEATURE_RFC822_SIZE		= 0x01,
+	IMAPC_FEATURE_GUID_FORCED		= 0x02,
+	IMAPC_FEATURE_FETCH_HEADERS		= 0x04,
+	IMAPC_FEATURE_GMAIL_MIGRATION		= 0x08,
+	IMAPC_FEATURE_SEARCH			= 0x10,
+	IMAPC_FEATURE_ZIMBRA_WORKAROUNDS	= 0x20,
+	IMAPC_FEATURE_NO_EXAMINE		= 0x40,
+	IMAPC_FEATURE_PROXYAUTH			= 0x80,
+	IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS	= 0x100,
+	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200,
+	IMAPC_FEATURE_MODSEQ			= 0x400,
+	IMAPC_FEATURE_DELAY_LOGIN		= 0x800,
+	IMAPC_FEATURE_FETCH_BODYSTRUCTURE	= 0x1000,
+	IMAPC_FEATURE_SEND_ID			= 0x2000,
+	IMAPC_FEATURE_FETCH_EMPTY_IS_EXPUNGED	= 0x4000,
+	IMAPC_FEATURE_NO_MSN_UPDATES		= 0x8000,
+	IMAPC_FEATURE_ACL 			= 0x10000,
+};
+/* </settings checks> */
+
+/*
+ * NOTE: Any additions here should be reflected in imapc_storage_create's
+ * serialization of settings.
+ */
+struct imapc_settings {
+	const char *imapc_host;
+	in_port_t imapc_port;
+
+	const char *imapc_user;
+	const char *imapc_master_user;
+	const char *imapc_password;
+	const char *imapc_sasl_mechanisms;
+
+	const char *imapc_ssl;
+	bool imapc_ssl_verify;
+
+	const char *imapc_features;
+	const char *imapc_rawlog_dir;
+	const char *imapc_list_prefix;
+	unsigned int imapc_cmd_timeout;
+	unsigned int imapc_max_idle_time;
+	unsigned int imapc_connection_retry_count;
+	unsigned int imapc_connection_retry_interval;
+	uoff_t imapc_max_line_length;
+
+	const char *pop3_deleted_flag;
+
+	enum imapc_features parsed_features;
+	unsigned int throttle_init_msecs;
+	unsigned int throttle_max_msecs;
+	unsigned int throttle_shrink_min_msecs;
+};
+
+const struct setting_parser_info *imapc_get_setting_parser_info(void);
+
+#endif
diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
new file mode 100644
index 0000000..43d90de
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -0,0 +1,1344 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "str.h"
+#include "imap-arg.h"
+#include "imap-resp-code.h"
+#include "mailbox-tree.h"
+#include "imapc-connection.h"
+#include "imapc-msgmap.h"
+#include "imapc-mail.h"
+#include "imapc-list.h"
+#include "imapc-search.h"
+#include "imapc-sync.h"
+#include "imapc-settings.h"
+#include "imapc-storage.h"
+
+#define DNS_CLIENT_SOCKET_NAME "dns-client"
+
+struct imapc_open_context {
+	struct imapc_mailbox *mbox;
+	int ret;
+};
+
+struct imapc_resp_code_map {
+	const char *code;
+	enum mail_error error;
+};
+
+extern struct mail_storage imapc_storage;
+extern struct mailbox imapc_mailbox;
+
+static struct event_category event_category_imapc = {
+	.name = "imapc",
+	.parent = &event_category_storage,
+};
+
+static struct imapc_resp_code_map imapc_resp_code_map[] = {
+	{ IMAP_RESP_CODE_UNAVAILABLE, MAIL_ERROR_TEMP },
+	{ IMAP_RESP_CODE_AUTHFAILED, MAIL_ERROR_PERM },
+	{ IMAP_RESP_CODE_AUTHZFAILED, MAIL_ERROR_PERM },
+	{ IMAP_RESP_CODE_EXPIRED, MAIL_ERROR_PERM },
+	{ IMAP_RESP_CODE_PRIVACYREQUIRED, MAIL_ERROR_PERM },
+	{ IMAP_RESP_CODE_CONTACTADMIN, MAIL_ERROR_PERM },
+	{ IMAP_RESP_CODE_NOPERM, MAIL_ERROR_PERM },
+	{ IMAP_RESP_CODE_INUSE, MAIL_ERROR_INUSE },
+	{ IMAP_RESP_CODE_EXPUNGEISSUED, MAIL_ERROR_EXPUNGED },
+	{ IMAP_RESP_CODE_CORRUPTION, MAIL_ERROR_TEMP },
+	{ IMAP_RESP_CODE_SERVERBUG, MAIL_ERROR_TEMP },
+	/* { IMAP_RESP_CODE_CLIENTBUG, 0 }, */
+	{ IMAP_RESP_CODE_CANNOT, MAIL_ERROR_NOTPOSSIBLE },
+	{ IMAP_RESP_CODE_LIMIT, MAIL_ERROR_LIMIT },
+	{ IMAP_RESP_CODE_OVERQUOTA, MAIL_ERROR_NOQUOTA },
+	{ IMAP_RESP_CODE_ALREADYEXISTS, MAIL_ERROR_EXISTS },
+	{ IMAP_RESP_CODE_NONEXISTENT, MAIL_ERROR_NOTFOUND }
+};
+
+static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
+				  struct imapc_storage_client *client);
+static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
+				     struct imapc_storage_client *client);
+static int imapc_mailbox_run_status(struct mailbox *box,
+				    enum mailbox_status_items items,
+				    struct mailbox_status *status_r);
+
+bool imapc_resp_text_code_parse(const char *str, enum mail_error *error_r)
+{
+	unsigned int i;
+
+	if (str == NULL)
+		return FALSE;
+
+	for (i = 0; i < N_ELEMENTS(imapc_resp_code_map); i++) {
+		if (strcmp(imapc_resp_code_map[i].code, str) == 0) {
+			*error_r = imapc_resp_code_map[i].error;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+bool imapc_mail_error_to_resp_text_code(enum mail_error error, const char **str_r)
+{
+	unsigned int i;
+
+	for (i = 0; i < N_ELEMENTS(imapc_resp_code_map); i++) {
+		if (imapc_resp_code_map[i].error == error) {
+			*str_r = imapc_resp_code_map[i].code;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+bool imapc_mailbox_has_modseqs(struct imapc_mailbox *mbox)
+{
+	return (mbox->capabilities & (IMAPC_CAPABILITY_CONDSTORE |
+				      IMAPC_CAPABILITY_QRESYNC)) != 0 &&
+		IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_MODSEQ);
+}
+
+static struct mail_storage *imapc_storage_alloc(void)
+{
+	struct imapc_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("imapc storage", 2048);
+	storage = p_new(pool, struct imapc_storage, 1);
+	storage->storage = imapc_storage;
+	storage->storage.pool = pool;
+	storage->root_ioloop = current_ioloop;
+	return &storage->storage;
+}
+
+void imapc_copy_error_from_reply(struct imapc_storage *storage,
+				 enum mail_error default_error,
+				 const struct imapc_command_reply *reply)
+{
+	enum mail_error error;
+
+	if (imapc_resp_text_code_parse(reply->resp_text_key, &error)) {
+		mail_storage_set_error(&storage->storage, error,
+				       reply->text_without_resp);
+	} else {
+		mail_storage_set_error(&storage->storage, default_error,
+				       reply->text_without_resp);
+	}
+}
+
+void imapc_simple_context_init(struct imapc_simple_context *sctx,
+			       struct imapc_storage_client *client)
+{
+	i_zero(sctx);
+	sctx->client = client;
+	sctx->ret = -2;
+}
+
+void imapc_simple_run(struct imapc_simple_context *sctx,
+		      struct imapc_command **cmd)
+{
+	if (imapc_storage_client_handle_auth_failure(sctx->client)) {
+		imapc_command_abort(cmd);
+		imapc_client_logout(sctx->client->client);
+		sctx->ret = -1;
+	}
+	*cmd = NULL;
+	while (sctx->ret == -2)
+		imapc_client_run(sctx->client->client);
+}
+
+void imapc_mailbox_run(struct imapc_mailbox *mbox)
+{
+	imapc_mail_fetch_flush(mbox);
+	imapc_mailbox_run_nofetch(mbox);
+}
+
+void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox)
+{
+	do {
+		imapc_client_run(mbox->storage->client->client);
+	} while (mbox->storage->reopen_count > 0 ||
+		 mbox->state_fetching_uid1);
+}
+
+void imapc_simple_callback(const struct imapc_command_reply *reply,
+			   void *context)
+{
+	struct imapc_simple_context *ctx = context;
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		ctx->ret = 0;
+	else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		imapc_copy_error_from_reply(ctx->client->_storage,
+					    MAIL_ERROR_PARAMS, reply);
+		ctx->ret = -1;
+	} else if (imapc_storage_client_handle_auth_failure(ctx->client)) {
+		ctx->ret = -1;
+	} else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		mail_storage_set_internal_error(&ctx->client->_storage->storage);
+		ctx->ret = -1;
+	} else {
+		mail_storage_set_critical(&ctx->client->_storage->storage,
+			"imapc: Command failed: %s", reply->text_full);
+		ctx->ret = -1;
+	}
+	imapc_client_stop(ctx->client->client);
+}
+
+void imapc_mailbox_noop(struct imapc_mailbox *mbox)
+{
+	struct imapc_command *cmd;
+	struct imapc_simple_context sctx;
+
+	if (mbox->client_box == NULL) {
+		/* mailbox opening hasn't finished yet */
+		return;
+	}
+
+	imapc_simple_context_init(&sctx, mbox->storage->client);
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_simple_callback, &sctx);
+	imapc_command_send(cmd, "NOOP");
+	imapc_simple_run(&sctx, &cmd);
+}
+
+static void
+imapc_storage_client_untagged_cb(const struct imapc_untagged_reply *reply,
+				 void *context)
+{
+	struct imapc_storage_client *client = context;
+	struct imapc_mailbox *mbox = reply->untagged_box_context;
+	const struct imapc_storage_event_callback *cb;
+	const struct imapc_mailbox_event_callback *mcb;
+
+	array_foreach(&client->untagged_callbacks, cb) {
+		if (strcasecmp(reply->name, cb->name) == 0)
+			cb->callback(reply, client);
+	}
+
+	if (mbox == NULL)
+		return;
+
+	array_foreach(&mbox->untagged_callbacks, mcb) {
+		if (strcasecmp(reply->name, mcb->name) == 0)
+			mcb->callback(reply, mbox);
+	}
+
+	if (reply->resp_text_key != NULL) {
+		array_foreach(&mbox->resp_text_callbacks, mcb) {
+			if (strcasecmp(reply->resp_text_key, mcb->name) == 0)
+				mcb->callback(reply, mbox);
+		}
+	}
+}
+
+static void
+imapc_storage_client_login_callback(const struct imapc_command_reply *reply,
+				    void *context)
+{
+	struct imapc_storage_client *client = context;
+
+	client->auth_returned = TRUE;
+	imapc_client_stop(client->client);
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		return;
+	if (client->destroying &&
+	    reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		/* user's work was finished before imapc login finished -
+		   it's not an error */
+		return;
+	}
+
+	client->auth_failed_state = reply->state;
+	client->auth_failed_reason = i_strdup(reply->text_full);
+	if (!imapc_storage_client_handle_auth_failure(client))
+		i_unreached();
+}
+
+bool imapc_storage_client_handle_auth_failure(struct imapc_storage_client *client)
+{
+	if (client->auth_failed_state == IMAPC_COMMAND_STATE_OK)
+		return FALSE;
+
+	/* We need to set the error to either storage or to list, depending on
+	   whether the caller is from mail-storage.h API or mailbox-list.h API.
+	   We don't know here what the caller is though, so just set the error
+	   to both of them. */
+	if (client->_storage != NULL) {
+		if (client->auth_failed_state == IMAPC_COMMAND_STATE_DISCONNECTED)
+			mail_storage_set_internal_error(&client->_storage->storage);
+		else {
+			mail_storage_set_error(&client->_storage->storage,
+				MAIL_ERROR_PERM, client->auth_failed_reason);
+		}
+	}
+	if (client->_list != NULL) {
+		if (client->auth_failed_state == IMAPC_COMMAND_STATE_DISCONNECTED)
+			mailbox_list_set_internal_error(&client->_list->list);
+		else {
+			mailbox_list_set_error(&client->_list->list,
+				MAIL_ERROR_PERM, client->auth_failed_reason);
+		}
+	}
+	return TRUE;
+}
+
+static void imapc_storage_client_login(struct imapc_storage_client *client,
+				       struct mail_user *user, const char *host)
+{
+	imapc_client_login(client->client);
+	if (!user->namespaces_created) {
+		/* we're still initializing the user. wait for the
+		   login to finish, so we can fail the user creation
+		   if it fails. */
+		while (!client->auth_returned)
+			imapc_client_run(client->client);
+		if (imapc_storage_client_handle_auth_failure(client)) {
+			user->error = p_strdup_printf(user->pool,
+				"imapc: Login to %s failed: %s",
+				host, client->auth_failed_reason);
+		}
+	}
+}
+
+int imapc_storage_client_create(struct mail_namespace *ns,
+				const struct imapc_settings *imapc_set,
+				const struct mail_storage_settings *mail_set,
+				struct imapc_storage_client **client_r,
+				const char **error_r)
+{
+	struct imapc_storage_client *client;
+	struct imapc_client_settings set;
+	string_t *str;
+
+	i_zero(&set);
+	set.host = imapc_set->imapc_host;
+	if (*set.host == '\0') {
+		*error_r = "missing imapc_host";
+		return -1;
+	}
+	set.port = imapc_set->imapc_port;
+	if (imapc_set->imapc_user[0] != '\0')
+		set.username = imapc_set->imapc_user;
+	else if (ns->owner != NULL)
+		set.username = ns->owner->username;
+	else
+		set.username = ns->user->username;
+	set.master_user = imapc_set->imapc_master_user;
+	set.password = imapc_set->imapc_password;
+	if (*set.password == '\0') {
+		*error_r = "missing imapc_password";
+		return -1;
+	}
+	set.sasl_mechanisms = imapc_set->imapc_sasl_mechanisms;
+	set.use_proxyauth = (imapc_set->parsed_features & IMAPC_FEATURE_PROXYAUTH) != 0;
+	set.cmd_timeout_msecs = imapc_set->imapc_cmd_timeout * 1000;
+	set.connect_retry_count = imapc_set->imapc_connection_retry_count;
+	set.connect_retry_interval_msecs = imapc_set->imapc_connection_retry_interval;
+	set.max_idle_time = imapc_set->imapc_max_idle_time;
+	set.max_line_length = imapc_set->imapc_max_line_length;
+	set.dns_client_socket_path = *ns->user->set->base_dir == '\0' ? "" :
+		t_strconcat(ns->user->set->base_dir, "/",
+			    DNS_CLIENT_SOCKET_NAME, NULL);
+	set.debug = mail_set->mail_debug;
+	set.rawlog_dir = mail_user_home_expand(ns->user,
+					       imapc_set->imapc_rawlog_dir);
+	if ((imapc_set->parsed_features & IMAPC_FEATURE_SEND_ID) != 0)
+		set.session_id_prefix = ns->user->session_id;
+
+	str = t_str_new(128);
+	mail_user_set_get_temp_prefix(str, ns->user->set);
+	set.temp_path_prefix = str_c(str);
+
+	mail_user_init_ssl_client_settings(ns->user, &set.ssl_set);
+	if (!imapc_set->imapc_ssl_verify)
+		set.ssl_set.allow_invalid_cert = TRUE;
+
+	if (strcmp(imapc_set->imapc_ssl, "imaps") == 0)
+		set.ssl_mode = IMAPC_CLIENT_SSL_MODE_IMMEDIATE;
+	else if (strcmp(imapc_set->imapc_ssl, "starttls") == 0)
+		set.ssl_mode = IMAPC_CLIENT_SSL_MODE_STARTTLS;
+	else
+		set.ssl_mode = IMAPC_CLIENT_SSL_MODE_NONE;
+
+	set.throttle_set.init_msecs = imapc_set->throttle_init_msecs;
+	set.throttle_set.max_msecs = imapc_set->throttle_max_msecs;
+	set.throttle_set.shrink_min_msecs = imapc_set->throttle_shrink_min_msecs;
+
+	client = i_new(struct imapc_storage_client, 1);
+	client->refcount = 1;
+	i_array_init(&client->untagged_callbacks, 16);
+	/* FIXME: storage->event would be better, but we first get here when
+	   creating mailbox_list, and storage doesn't even exist yet. */
+	client->client = imapc_client_init(&set, ns->user->event);
+	imapc_client_register_untagged(client->client,
+				       imapc_storage_client_untagged_cb, client);
+
+	imapc_client_set_login_callback(client->client, imapc_storage_client_login_callback, client);
+
+	if ((ns->flags & NAMESPACE_FLAG_LIST_PREFIX) != 0 &&
+	    (imapc_set->parsed_features & IMAPC_FEATURE_DELAY_LOGIN) == 0) {
+		/* start logging in immediately */
+		imapc_storage_client_login(client, ns->user, set.host);
+	}
+
+	*client_r = client;
+	return 0;
+}
+
+void imapc_storage_client_unref(struct imapc_storage_client **_client)
+{
+	struct imapc_storage_client *client = *_client;
+	struct imapc_storage_event_callback *cb;
+
+	*_client = NULL;
+
+	i_assert(client->refcount > 0);
+	if (--client->refcount > 0)
+		return;
+	imapc_client_deinit(&client->client);
+	array_foreach_modifiable(&client->untagged_callbacks, cb)
+		i_free(cb->name);
+	array_free(&client->untagged_callbacks);
+	i_free(client->auth_failed_reason);
+	i_free(client);
+}
+
+static int
+imapc_storage_create(struct mail_storage *_storage,
+		     struct mail_namespace *ns,
+		     const char **error_r)
+{
+	struct imapc_storage *storage = IMAPC_STORAGE(_storage);
+	struct imapc_mailbox_list *imapc_list = NULL;
+
+	storage->set = mail_namespace_get_driver_settings(ns, _storage);
+
+	/* serialize all the settings */
+	_storage->unique_root_dir = p_strdup_printf(_storage->pool,
+						    "%s%s://(%s|%s):%s@%s:%u/%s mechs:%s features:%s "
+						    "rawlog:%s cmd_timeout:%u maxidle:%u maxline:%zuu "
+						    "pop3delflg:%s root_dir:%s",
+						    storage->set->imapc_ssl,
+						    storage->set->imapc_ssl_verify ? "(verify)" : "",
+						    storage->set->imapc_user,
+						    storage->set->imapc_master_user,
+						    storage->set->imapc_password,
+						    storage->set->imapc_host,
+						    storage->set->imapc_port,
+						    storage->set->imapc_list_prefix,
+						    storage->set->imapc_sasl_mechanisms,
+						    storage->set->imapc_features,
+						    storage->set->imapc_rawlog_dir,
+						    storage->set->imapc_cmd_timeout,
+						    storage->set->imapc_max_idle_time,
+						    (size_t) storage->set->imapc_max_line_length,
+						    storage->set->pop3_deleted_flag,
+						    ns->list->set.root_dir);
+
+	if (strcmp(ns->list->name, MAILBOX_LIST_NAME_IMAPC) == 0) {
+		imapc_list = (struct imapc_mailbox_list *)ns->list;
+		storage->client = imapc_list->client;
+		storage->client->refcount++;
+	} else {
+		if (imapc_storage_client_create(ns, storage->set, _storage->set,
+						&storage->client, error_r) < 0)
+			return -1;
+	}
+	storage->client->_storage = storage;
+	p_array_init(&storage->remote_namespaces, _storage->pool, 4);
+	if (IMAPC_HAS_FEATURE(storage, IMAPC_FEATURE_FETCH_BODYSTRUCTURE)) {
+		_storage->nonbody_access_fields |=
+			MAIL_FETCH_IMAP_BODY | MAIL_FETCH_IMAP_BODYSTRUCTURE;
+	}
+
+	imapc_storage_client_register_untagged(storage->client, "STATUS",
+					       imapc_untagged_status);
+	imapc_storage_client_register_untagged(storage->client, "NAMESPACE",
+					       imapc_untagged_namespace);
+
+	return 0;
+}
+
+static void imapc_storage_destroy(struct mail_storage *_storage)
+{
+	struct imapc_storage *storage = IMAPC_STORAGE(_storage);
+
+	storage->client->destroying = TRUE;
+
+	/* make sure all pending commands are aborted before anything is
+	   deinitialized */
+	imapc_client_logout(storage->client->client);
+
+	imapc_storage_client_unref(&storage->client);
+	index_storage_destroy(_storage);
+}
+
+void imapc_storage_client_register_untagged(struct imapc_storage_client *client,
+					    const char *name,
+					    imapc_storage_callback_t *callback)
+{
+	struct imapc_storage_event_callback *cb;
+
+	cb = array_append_space(&client->untagged_callbacks);
+	cb->name = i_strdup(name);
+	cb->callback = callback;
+}
+
+void imapc_storage_client_unregister_untagged(struct imapc_storage_client *client,
+					      const char *name)
+{
+	struct imapc_storage_event_callback *cb;
+	unsigned int idx;
+	array_foreach_modifiable(&client->untagged_callbacks, cb) {
+		if (strcmp(cb->name, name) == 0) {
+			 idx = array_foreach_idx(&client->untagged_callbacks, cb);
+			 i_free(cb->name);
+			 array_delete(&client->untagged_callbacks, idx, 1);
+			 return;
+		}
+	}
+	i_unreached();
+}
+
+static void
+imapc_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED,
+				struct mailbox_list_settings *set)
+{
+	if (set->layout == NULL)
+		set->layout = MAILBOX_LIST_NAME_IMAPC;
+	set->storage_name_escape_char = IMAPC_LIST_STORAGE_NAME_ESCAPE_CHAR;
+	/* We want to have all imapc mailboxes accessible, so escape them if
+	   necessary. */
+	if (set->vname_escape_char == '\0')
+		set->vname_escape_char = IMAPC_LIST_VNAME_ESCAPE_CHAR;
+}
+
+static struct mailbox *
+imapc_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
+		    const char *vname, enum mailbox_flags flags)
+{
+	struct imapc_mailbox *mbox;
+	pool_t pool;
+
+	pool = pool_alloconly_create("imapc mailbox", 1024*4);
+	mbox = p_new(pool, struct imapc_mailbox, 1);
+	mbox->box = imapc_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.mail_vfuncs = &imapc_mail_vfuncs;
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
+
+	mbox->storage = IMAPC_STORAGE(storage);
+
+	p_array_init(&mbox->untagged_callbacks, pool, 16);
+	p_array_init(&mbox->resp_text_callbacks, pool, 16);
+	p_array_init(&mbox->fetch_requests, pool, 16);
+	p_array_init(&mbox->untagged_fetch_contexts, pool, 16);
+	p_array_init(&mbox->delayed_expunged_uids, pool, 16);
+	p_array_init(&mbox->copy_rollback_expunge_uids, pool, 16);
+	mbox->pending_fetch_cmd = str_new(pool, 128);
+	mbox->pending_copy_cmd = str_new(pool, 128);
+	mbox->prev_mail_cache.fd = -1;
+	imapc_mailbox_register_callbacks(mbox);
+	return &mbox->box;
+}
+
+const char *imapc_mailbox_get_remote_name(struct imapc_mailbox *mbox)
+{
+	struct imapc_mailbox_list *list =
+		container_of(mbox->box.list, struct imapc_mailbox_list, list);
+
+	if (strcmp(mbox->box.list->name, MAILBOX_LIST_NAME_IMAPC) != 0)
+		return mbox->box.name;
+	return imapc_list_storage_to_remote_name(list, mbox->box.name);
+}
+
+static int
+imapc_mailbox_exists(struct mailbox *box, bool auto_boxes,
+		     enum mailbox_existence *existence_r)
+{
+	if (auto_boxes && mailbox_is_autocreated(box)) {
+		*existence_r = MAILBOX_EXISTENCE_SELECT;
+		return 0;
+	}
+
+	if (strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPC) != 0) {
+		if (box->inbox_any)
+			*existence_r = MAILBOX_EXISTENCE_SELECT;
+		else
+			*existence_r = MAILBOX_EXISTENCE_NONE;
+		return 0;
+	}
+
+	enum mailbox_info_flags flags;
+
+	struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)box->list;
+
+	if (imapc_storage_client_handle_auth_failure(list->client)) {
+		mail_storage_copy_list_error(box->storage, box->list);
+		return -1;
+	}
+	if (imapc_list_get_mailbox_flags(box->list, box->name, &flags) < 0) {
+		mail_storage_copy_list_error(box->storage, box->list);
+		return -1;
+	}
+	if ((flags & MAILBOX_NONEXISTENT) != 0)
+		*existence_r = MAILBOX_EXISTENCE_NONE;
+	else if ((flags & MAILBOX_NOSELECT) != 0)
+		*existence_r = MAILBOX_EXISTENCE_NOSELECT;
+	else
+		*existence_r = MAILBOX_EXISTENCE_SELECT;
+	return 0;
+}
+
+static bool imapc_mailbox_want_examine(struct imapc_mailbox *mbox)
+{
+	if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_EXAMINE)) {
+		/* mainly a Courier-workaround: With POP3-only Maildir that
+		   doesn't have UIDVALIDITY set, EXAMINE won't generate a
+		   permanent UIDVALIDITY while SELECT will. */
+		return FALSE;
+	}
+	return (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0 &&
+		((mbox->box.flags & MAILBOX_FLAG_READONLY) != 0 ||
+		 (mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0);
+}
+
+static bool
+imapc_mailbox_verify_select(struct imapc_mailbox *mbox, const char **error_r)
+{
+	if (!mbox->exists_received)
+		*error_r = "EXISTS not received";
+	else if (mbox->sync_uid_validity == 0)
+		*error_r = "UIDVALIDITY not received";
+	else
+		return TRUE;
+	return FALSE;
+}
+
+static void
+imapc_mailbox_reopen_callback(const struct imapc_command_reply *reply,
+			      void *context)
+{
+	struct imapc_mailbox *mbox = context;
+	const char *errmsg;
+
+	i_assert(mbox->storage->reopen_count > 0);
+	mbox->storage->reopen_count--;
+	mbox->selecting = FALSE;
+	if (reply->state != IMAPC_COMMAND_STATE_OK)
+		errmsg = reply->text_full;
+	else if (imapc_mailbox_verify_select(mbox, &errmsg)) {
+		imap_mailbox_select_finish(mbox);
+		errmsg = NULL;
+	}
+
+	if (errmsg != NULL) {
+		imapc_client_mailbox_reconnect(mbox->client_box,
+			t_strdup_printf("Reopening mailbox '%s' failed: %s",
+					mbox->box.name, errmsg));
+	}
+
+	imapc_client_stop(mbox->storage->client->client);
+}
+
+static void imapc_mailbox_reopen(void *context)
+{
+	struct imapc_mailbox *mbox = context;
+	struct imapc_command *cmd;
+
+	/* we're reconnecting and need to reopen the mailbox */
+	mbox->prev_skipped_rseq = 0;
+	mbox->prev_skipped_uid = 0;
+	imapc_msgmap_reset(imapc_client_mailbox_get_msgmap(mbox->client_box));
+
+	if (mbox->selecting) {
+		/* We reconnected during the initial SELECT/EXAMINE. It'll be
+		   automatically resent by lib-imap-client, so we don't need to
+		   send it again here. */
+		i_assert(!mbox->initial_sync_done);
+		return;
+	}
+	if (!mbox->initial_sync_done) {
+		/* Initial FETCH 1:* didn't fully succeed. We're reconnecting
+		   and lib-imap-client is automatically resending it. But we
+		   need to reset the sync_next_* state so that if any of the
+		   mails are now expunged we won't get confused and crash. */
+		mbox->sync_next_lseq = 1;
+		mbox->sync_next_rseq = 1;
+	}
+
+	mbox->state_fetched_success = FALSE;
+	mbox->initial_sync_done = FALSE;
+	mbox->selecting = TRUE;
+	mbox->selected = FALSE;
+	mbox->exists_received = FALSE;
+
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_mailbox_reopen_callback, mbox);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT);
+	if (imapc_mailbox_want_examine(mbox)) {
+		imapc_command_sendf(cmd, "EXAMINE %s",
+				    imapc_mailbox_get_remote_name(mbox));
+	} else {
+		imapc_command_sendf(cmd, "SELECT %s",
+				    imapc_mailbox_get_remote_name(mbox));
+	}
+	mbox->storage->reopen_count++;
+}
+
+static void
+imapc_mailbox_open_callback(const struct imapc_command_reply *reply,
+			    void *context)
+{
+	struct imapc_open_context *ctx = context;
+	const char *error;
+
+	ctx->mbox->selecting = FALSE;
+	if (reply->state == IMAPC_COMMAND_STATE_OK) {
+		if (!imapc_mailbox_verify_select(ctx->mbox, &error)) {
+			mailbox_set_critical(&ctx->mbox->box,
+				"imapc: Opening mailbox failed: %s", error);
+			ctx->ret = -1;
+		} else {
+			imap_mailbox_select_finish(ctx->mbox);
+			ctx->ret = 0;
+		}
+	} else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		/* Unless the remote IMAP server supports sending
+		   resp-text-code, we don't know if the NO reply is because
+		   the mailbox doesn't exist or because of some internal error.
+		   We'll default to assuming it doesn't exist, so e.g.
+		   mailbox { auto=create } will auto-create missing mailboxes.
+		   However, INBOX is a special mailbox, which is always
+		   autocreated if it doesn't exist. This is true in both the
+		   local Dovecot and the remote IMAP server. This means that
+		   there's no point in trying to send CREATE INBOX to the
+		   remote server. We'll avoid that by defaulting to temporary
+		   failure with INBOX. */
+		enum mail_error default_error =
+			ctx->mbox->box.inbox_any ?
+			MAIL_ERROR_TEMP : MAIL_ERROR_NOTFOUND;
+		imapc_copy_error_from_reply(ctx->mbox->storage,
+					    default_error, reply);
+		ctx->ret = -1;
+	} else if (imapc_storage_client_handle_auth_failure(ctx->mbox->storage->client)) {
+		ctx->ret = -1;
+	} else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		ctx->ret = -1;
+		mail_storage_set_internal_error(ctx->mbox->box.storage);
+	} else {
+		mailbox_set_critical(&ctx->mbox->box,
+			"imapc: Opening mailbox failed: %s", reply->text_full);
+		ctx->ret = -1;
+	}
+	imapc_client_stop(ctx->mbox->storage->client->client);
+}
+
+static int imapc_mailbox_get_capabilities(struct imapc_mailbox *mbox)
+{
+	/* If authentication failed, don't check again. */
+	if (imapc_storage_client_handle_auth_failure(mbox->storage->client))
+		return -1;
+
+	return imapc_client_get_capabilities(mbox->storage->client->client,
+					     &mbox->capabilities);
+
+}
+
+static void imapc_mailbox_get_extensions(struct imapc_mailbox *mbox)
+{
+	if (mbox->guid_fetch_field_name == NULL) {
+		/* see if we can get message GUIDs somehow */
+		if ((mbox->capabilities & IMAPC_CAPABILITY_X_GM_EXT_1) != 0) {
+			/* GMail */
+			mbox->guid_fetch_field_name = "X-GM-MSGID";
+		}
+	}
+}
+
+int imapc_mailbox_select(struct imapc_mailbox *mbox)
+{
+	struct imapc_command *cmd;
+	struct imapc_open_context ctx;
+
+	i_assert(mbox->client_box == NULL);
+
+	if (imapc_mailbox_get_capabilities(mbox) < 0)
+		return -1;
+
+	if (imapc_mailbox_has_modseqs(mbox)) {
+		if (!array_is_created(&mbox->rseq_modseqs))
+			i_array_init(&mbox->rseq_modseqs, 32);
+		else
+			array_clear(&mbox->rseq_modseqs);
+	}
+
+	mbox->client_box =
+		imapc_client_mailbox_open(mbox->storage->client->client, mbox);
+	imapc_client_mailbox_set_reopen_cb(mbox->client_box,
+					   imapc_mailbox_reopen, mbox);
+
+	imapc_mailbox_get_extensions(mbox);
+
+	mbox->selecting = TRUE;
+	mbox->exists_received = FALSE;
+	ctx.mbox = mbox;
+	ctx.ret = -2;
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_mailbox_open_callback, &ctx);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT |
+				IMAPC_COMMAND_FLAG_RETRIABLE);
+	if (imapc_mailbox_want_examine(mbox)) {
+		imapc_command_sendf(cmd, "EXAMINE %s",
+			imapc_mailbox_get_remote_name(mbox));
+	} else {
+		imapc_command_sendf(cmd, "SELECT %s",
+			imapc_mailbox_get_remote_name(mbox));
+	}
+
+	while (ctx.ret == -2 || mbox->state_fetching_uid1)
+		imapc_mailbox_run(mbox);
+	if (!mbox->state_fetched_success)
+		ctx.ret = -1;
+	return ctx.ret;
+}
+
+static int imapc_mailbox_open(struct mailbox *box)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+
+	if (index_storage_mailbox_open(box, FALSE) < 0)
+		return -1;
+
+	if (box->deleting || (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) {
+		/* We don't actually want to SELECT the mailbox. */
+		return 0;
+	}
+
+	if (*box->name == '\0' &&
+	    (box->list->ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) {
+		/* trying to open INBOX as the namespace prefix.
+		   Don't allow this. */
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+				       "Mailbox isn't selectable");
+		mailbox_close(box);
+		return -1;
+	}
+
+	if (imapc_mailbox_select(mbox) < 0) {
+		mailbox_close(box);
+		return -1;
+	}
+	return 0;
+}
+
+void imapc_mail_cache_free(struct imapc_mail_cache *cache)
+{
+	i_close_fd(&cache->fd);
+	buffer_free(&cache->buf);
+	cache->uid = 0;
+}
+
+static void imapc_mailbox_close(struct mailbox *box)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+	bool changes;
+
+	(void)imapc_mailbox_commit_delayed_trans(mbox, FALSE, &changes);
+	imapc_mail_fetch_flush(mbox);
+
+	/* Arriving here we may have fetch contexts still unprocessed,
+	   if there have been no mailbox_sync() after receiving the untagged replies.
+	   Losing these changes isn't a problem, since the same changes will be found
+	   out after connecting to the server the next time. */
+	struct imapc_untagged_fetch_ctx *untagged_fetch_context;
+	array_foreach_elem(&mbox->untagged_fetch_contexts, untagged_fetch_context)
+		imapc_untagged_fetch_ctx_free(&untagged_fetch_context);
+	array_clear(&mbox->untagged_fetch_contexts);
+
+	if (mbox->client_box != NULL)
+		imapc_client_mailbox_close(&mbox->client_box);
+	if (array_is_created(&mbox->rseq_modseqs))
+		array_free(&mbox->rseq_modseqs);
+	if (mbox->sync_view != NULL)
+		mail_index_view_close(&mbox->sync_view);
+	timeout_remove(&mbox->to_idle_delay);
+	timeout_remove(&mbox->to_idle_check);
+	imapc_mail_cache_free(&mbox->prev_mail_cache);
+	index_storage_mailbox_close(box);
+}
+
+static int
+imapc_mailbox_create(struct mailbox *box,
+		     const struct mailbox_update *update ATTR_UNUSED,
+		     bool directory)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+	struct imapc_command *cmd;
+	struct imapc_simple_context sctx;
+	const char *remote_name = imapc_mailbox_get_remote_name(mbox);
+
+	if (!directory)
+		;
+	else if (strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPC) == 0) {
+		struct imapc_mailbox_list *imapc_list =
+			(struct imapc_mailbox_list *)box->list;
+		remote_name = t_strdup_printf("%s%c", remote_name,
+					      imapc_list->root_sep);
+	} else {
+		remote_name = t_strdup_printf("%s%c", remote_name,
+			mailbox_list_get_hierarchy_sep(box->list));
+	}
+	imapc_simple_context_init(&sctx, mbox->storage->client);
+	cmd = imapc_client_cmd(mbox->storage->client->client,
+			       imapc_simple_callback, &sctx);
+	imapc_command_sendf(cmd, "CREATE %s", remote_name);
+	imapc_simple_run(&sctx, &cmd);
+	return sctx.ret;
+}
+
+static int imapc_mailbox_update(struct mailbox *box,
+				const struct mailbox_update *update)
+{
+	if (!guid_128_is_empty(update->mailbox_guid) ||
+	    update->uid_validity != 0 || update->min_next_uid != 0 ||
+	    update->min_first_recent_uid != 0) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE,
+				       "Not supported");
+	}
+	return index_storage_mailbox_update(box, update);
+}
+
+static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
+				  struct imapc_storage_client *client)
+{
+	struct imapc_storage *storage = client->_storage;
+	struct mailbox_status *status;
+	const struct imap_arg *list;
+	const char *remote_name, *key, *value;
+	uint32_t num;
+	unsigned int i;
+
+	if (!imap_arg_get_astring(&reply->args[0], &remote_name) ||
+	    !imap_arg_get_list(&reply->args[1], &list))
+		return;
+
+	if (storage->cur_status_box == NULL)
+		return;
+
+	if (!imapc_mailbox_name_equals(storage->cur_status_box,
+				       remote_name))
+		return;
+
+	status = storage->cur_status;
+	for (i = 0; list[i].type != IMAP_ARG_EOL; i += 2) {
+		if (!imap_arg_get_atom(&list[i], &key) ||
+		    !imap_arg_get_atom(&list[i+1], &value) ||
+		    str_to_uint32(value, &num) < 0)
+			return;
+
+		if (strcasecmp(key, "MESSAGES") == 0)
+			status->messages = num;
+		else if (strcasecmp(key, "RECENT") == 0)
+			status->recent = num;
+		else if (strcasecmp(key, "UIDNEXT") == 0)
+			status->uidnext = num;
+		else if (strcasecmp(key, "UIDVALIDITY") == 0)
+			status->uidvalidity = num;
+		else if (strcasecmp(key, "UNSEEN") == 0)
+			status->unseen = num;
+		else if (strcasecmp(key, "HIGHESTMODSEQ") == 0 &&
+			 imapc_mailbox_has_modseqs(storage->cur_status_box))
+			status->highest_modseq = num;
+	}
+}
+
+static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
+				     struct imapc_storage_client *client)
+{
+	struct imapc_storage *storage = client->_storage;
+	static enum mail_namespace_type ns_types[] = {
+		MAIL_NAMESPACE_TYPE_PRIVATE,
+		MAIL_NAMESPACE_TYPE_SHARED,
+		MAIL_NAMESPACE_TYPE_PUBLIC
+	};
+	struct imapc_namespace *ns;
+	const struct imap_arg *list, *list2;
+	const char *prefix, *sep;
+	unsigned int i;
+
+	array_clear(&storage->remote_namespaces);
+	for (i = 0; i < N_ELEMENTS(ns_types); i++) {
+		if (reply->args[i].type == IMAP_ARG_NIL)
+			continue;
+		if (!imap_arg_get_list(&reply->args[i], &list))
+			break;
+
+		for (; list->type != IMAP_ARG_EOL; list++) {
+			if (!imap_arg_get_list(list, &list2) ||
+			    !imap_arg_get_astring(&list2[0], &prefix) ||
+			    !imap_arg_get_nstring(&list2[1], &sep))
+				break;
+
+			ns = array_append_space(&storage->remote_namespaces);
+			ns->prefix = p_strdup(storage->storage.pool, prefix);
+			ns->separator = sep == NULL ? '\0' : sep[0];
+			ns->type = ns_types[i];
+		}
+	}
+}
+
+static void imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
+					      enum mailbox_status_items items,
+					      struct mailbox_status *status_r)
+{
+	index_storage_get_open_status(&mbox->box, items, status_r);
+	if ((items & STATUS_PERMANENT_FLAGS) != 0)
+		status_r->permanent_flags = mbox->permanent_flags;
+	if ((items & STATUS_FIRST_RECENT_UID) != 0)
+		status_r->first_recent_uid = mbox->highest_nonrecent_uid + 1;
+	if ((items & STATUS_HIGHESTMODSEQ) != 0) {
+		/* FIXME: this doesn't work perfectly. we're now just returning
+		   the HIGHESTMODSEQ from the current index, which may or may
+		   not be correct. with QRESYNC enabled we could be returning
+		   sync_highestmodseq, but that would require implementing
+		   VANISHED replies. and without QRESYNC we'd have to issue
+		   STATUS (HIGHESTMODSEQ), which isn't efficient since we get
+		   here constantly (after every IMAP command). */
+	}
+	if (imapc_mailbox_has_modseqs(mbox)) {
+		/* even if local indexes are only in memory, we still
+		   have modseqs on the IMAP server itself. */
+		status_r->nonpermanent_modseqs = FALSE;
+	}
+}
+
+static int imapc_mailbox_delete(struct mailbox *box)
+{
+	box->delete_skip_empty_check = TRUE;
+	return index_storage_mailbox_delete(box);
+}
+
+static int imapc_mailbox_run_status(struct mailbox *box,
+				    enum mailbox_status_items items,
+				    struct mailbox_status *status_r)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+	struct imapc_command *cmd;
+	struct imapc_simple_context sctx;
+	string_t *str;
+
+	if (imapc_mailbox_get_capabilities(mbox) < 0)
+		return -1;
+
+	str = t_str_new(256);
+	if ((items & STATUS_MESSAGES) != 0)
+		str_append(str, " MESSAGES");
+	if ((items & STATUS_RECENT) != 0)
+		str_append(str, " RECENT");
+	if ((items & STATUS_UIDNEXT) != 0)
+		str_append(str, " UIDNEXT");
+	if ((items & STATUS_UIDVALIDITY) != 0)
+		str_append(str, " UIDVALIDITY");
+	if ((items & STATUS_UNSEEN) != 0)
+		str_append(str, " UNSEEN");
+	if ((items & STATUS_HIGHESTMODSEQ) != 0 &&
+	    imapc_mailbox_has_modseqs(mbox))
+		str_append(str, " HIGHESTMODSEQ");
+
+	if (str_len(str) == 0) {
+		/* nothing requested */
+		return 0;
+	}
+
+	imapc_simple_context_init(&sctx, mbox->storage->client);
+	mbox->storage->cur_status_box = mbox;
+	mbox->storage->cur_status = status_r;
+	cmd = imapc_client_cmd(mbox->storage->client->client,
+			       imapc_simple_callback, &sctx);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_sendf(cmd, "STATUS %s (%1s)",
+			    imapc_mailbox_get_remote_name(mbox), str_c(str)+1);
+	imapc_simple_run(&sctx, &cmd);
+	mbox->storage->cur_status_box = NULL;
+	mbox->storage->cur_status = NULL;
+	return sctx.ret;
+}
+
+static int imapc_mailbox_get_status(struct mailbox *box,
+				    enum mailbox_status_items items,
+				    struct mailbox_status *status_r)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+
+	if (mbox->guid_fetch_field_name != NULL ||
+	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GUID_FORCED))
+		status_r->have_guids = TRUE;
+
+	if (box->opened) {
+		imapc_mailbox_get_selected_status(mbox, items, status_r);
+	} else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS |
+			     STATUS_PERMANENT_FLAGS |
+			     STATUS_FIRST_RECENT_UID)) != 0) {
+		/* getting these requires opening the mailbox */
+		if (mailbox_open(box) < 0)
+			return -1;
+		imapc_mailbox_get_selected_status(mbox, items, status_r);
+	} else {
+		if (imapc_mailbox_run_status(box, items, status_r) < 0)
+			return -1;
+	}
+
+	if (box->opened && !box->deleting && (items & STATUS_UIDNEXT) != 0 &&
+	    mbox->sync_uid_next == 0) {
+		/* Courier-workaround, it doesn't send UIDNEXT on SELECT */
+		if (imapc_mailbox_run_status(box, STATUS_UIDNEXT, status_r) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int imapc_mailbox_get_namespaces(struct imapc_mailbox *mbox)
+{
+	struct imapc_storage *storage = mbox->storage;
+	struct imapc_command *cmd;
+	struct imapc_simple_context sctx;
+
+	if (storage->namespaces_requested)
+		return 0;
+
+	if (imapc_mailbox_get_capabilities(mbox) < 0)
+		return -1;
+	if ((mbox->capabilities & IMAPC_CAPABILITY_NAMESPACE) == 0) {
+		/* NAMESPACE capability not supported */
+		return 0;
+	}
+
+	imapc_simple_context_init(&sctx, storage->client);
+	cmd = imapc_client_cmd(storage->client->client,
+			       imapc_simple_callback, &sctx);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_send(cmd, "NAMESPACE");
+	imapc_simple_run(&sctx, &cmd);
+
+	if (sctx.ret < 0)
+		return -1;
+	storage->namespaces_requested = TRUE;
+	return 0;
+}
+
+static const struct imapc_namespace *
+imapc_namespace_find_mailbox(struct imapc_storage *storage,
+			     const char *remote_name)
+{
+	const struct imapc_namespace *ns, *best_ns = NULL;
+	size_t best_len = UINT_MAX, len;
+
+	array_foreach(&storage->remote_namespaces, ns) {
+		len = strlen(ns->prefix);
+		if (str_begins(remote_name, ns->prefix)) {
+			if (best_len > len) {
+				best_ns = ns;
+				best_len = len;
+			}
+		}
+	}
+	return best_ns;
+}
+
+static int imapc_mailbox_get_metadata(struct mailbox *box,
+				      enum mailbox_metadata_items items,
+				      struct mailbox_metadata *metadata_r)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+	const struct imapc_namespace *ns;
+
+	if ((items & MAILBOX_METADATA_GUID) != 0) {
+		/* a bit ugly way to do this, but better than nothing for now.
+		   FIXME: if indexes are enabled, keep this there. */
+		mail_generate_guid_128_hash(box->name, metadata_r->guid);
+		items &= ENUM_NEGATE(MAILBOX_METADATA_GUID);
+	}
+	if ((items & MAILBOX_METADATA_BACKEND_NAMESPACE) != 0) {
+		if (imapc_mailbox_get_namespaces(mbox) < 0)
+			return -1;
+
+		const char *remote_name = imapc_mailbox_get_remote_name(mbox);
+		ns = imapc_namespace_find_mailbox(mbox->storage, remote_name);
+		if (ns != NULL) {
+			metadata_r->backend_ns_prefix = ns->prefix;
+			metadata_r->backend_ns_type = ns->type;
+		}
+		items &= ENUM_NEGATE(MAILBOX_METADATA_BACKEND_NAMESPACE);
+	}
+	if (items != 0) {
+		if (index_mailbox_get_metadata(box, items, metadata_r) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static void imapc_noop_callback(const struct imapc_command_reply *reply,
+				void *context)
+
+{
+	struct imapc_storage *storage = context;
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		;
+	else if (reply->state == IMAPC_COMMAND_STATE_NO)
+		imapc_copy_error_from_reply(storage, MAIL_ERROR_PARAMS, reply);
+	else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED)
+		mail_storage_set_internal_error(&storage->storage);
+	else {
+		mail_storage_set_critical(&storage->storage,
+			"imapc: NOOP failed: %s", reply->text_full);
+	}
+}
+
+static void imapc_idle_timeout(struct imapc_mailbox *mbox)
+{
+	struct imapc_command *cmd;
+
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_noop_callback, mbox->storage);
+	imapc_command_send(cmd, "NOOP");
+}
+
+static void imapc_idle_noop_callback(const struct imapc_command_reply *reply,
+				     void *context)
+
+{
+	struct imapc_mailbox *mbox = context;
+
+	imapc_noop_callback(reply, mbox->box.storage);
+	if (mbox->client_box != NULL)
+		imapc_client_mailbox_idle(mbox->client_box);
+}
+
+static void imapc_notify_changes(struct mailbox *box)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+	const struct mail_storage_settings *set = box->storage->set;
+	struct imapc_command *cmd;
+
+	if (box->notify_callback == NULL) {
+		timeout_remove(&mbox->to_idle_check);
+		return;
+	}
+
+	if ((mbox->capabilities & IMAPC_CAPABILITY_IDLE) != 0) {
+		/* remote server is already in IDLE. but since some servers
+		   don't notice changes immediately, we'll force them to check
+		   here by sending a NOOP. this helps with clients that break
+		   IDLE when clicking "get mail". */
+		cmd = imapc_client_mailbox_cmd(mbox->client_box,
+					       imapc_idle_noop_callback, mbox);
+		imapc_command_send(cmd, "NOOP");
+	} else {
+		/* remote server doesn't support IDLE.
+		   check for changes with NOOP every once in a while. */
+		i_assert(!imapc_client_is_running(mbox->storage->client->client));
+		mbox->to_idle_check =
+			timeout_add(set->mailbox_idle_check_interval * 1000,
+				    imapc_idle_timeout, mbox);
+	}
+}
+
+static bool imapc_is_inconsistent(struct mailbox *box)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+
+	if (box->view != NULL &&
+	    mail_index_view_is_inconsistent(box->view))
+		return TRUE;
+
+	return mbox->client_box == NULL ? FALSE :
+		!imapc_client_mailbox_is_opened(mbox->client_box);
+}
+
+struct mail_storage imapc_storage = {
+	.name = IMAPC_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT |
+		       MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT |
+		       MAIL_STORAGE_CLASS_FLAG_SECONDARY_INDEX,
+	.event_category = &event_category_imapc,
+
+	.v = {
+		imapc_get_setting_parser_info,
+		imapc_storage_alloc,
+		imapc_storage_create,
+		imapc_storage_destroy,
+		NULL,
+		imapc_storage_get_list_settings,
+		NULL,
+		imapc_mailbox_alloc,
+		NULL,
+		NULL,
+	}
+};
+
+static int
+imapc_mailbox_transaction_commit(struct mailbox_transaction_context *t,
+				 struct mail_transaction_commit_changes *changes_r)
+{
+	int ret = imapc_transaction_save_commit(t);
+	int ret2 = index_transaction_commit(t, changes_r);
+	return ret >= 0 && ret2 >= 0 ? 0 : -1;
+}
+
+struct mailbox imapc_mailbox = {
+	.v = {
+		index_storage_is_readonly,
+		index_storage_mailbox_enable,
+		imapc_mailbox_exists,
+		imapc_mailbox_open,
+		imapc_mailbox_close,
+		index_storage_mailbox_free,
+		imapc_mailbox_create,
+		imapc_mailbox_update,
+		imapc_mailbox_delete,
+		index_storage_mailbox_rename,
+		imapc_mailbox_get_status,
+		imapc_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		NULL,
+		NULL,
+		imapc_mailbox_sync_init,
+		index_mailbox_sync_next,
+		imapc_mailbox_sync_deinit,
+		NULL,
+		imapc_notify_changes,
+		index_transaction_begin,
+		imapc_mailbox_transaction_commit,
+		index_transaction_rollback,
+		NULL,
+		imapc_mail_alloc,
+		imapc_search_init,
+		imapc_search_deinit,
+		index_storage_search_next_nonblock,
+		imapc_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		imapc_save_alloc,
+		imapc_save_begin,
+		imapc_save_continue,
+		imapc_save_finish,
+		imapc_save_cancel,
+		imapc_copy,
+		imapc_transaction_save_commit_pre,
+		imapc_transaction_save_commit_post,
+		imapc_transaction_save_rollback,
+		imapc_is_inconsistent
+	}
+};
diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
new file mode 100644
index 0000000..e02c568
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -0,0 +1,268 @@
+#ifndef IMAPC_STORAGE_H
+#define IMAPC_STORAGE_H
+
+#include "index-storage.h"
+#include "imapc-settings.h"
+#include "imapc-client.h"
+
+#define IMAPC_STORAGE_NAME "imapc"
+/* storage_name separator */
+#define IMAPC_LIST_STORAGE_NAME_ESCAPE_CHAR '%'
+/* fs_name separator */
+#define IMAPC_LIST_FS_NAME_ESCAPE_CHAR '%'
+/* vname separator */
+#define IMAPC_LIST_VNAME_ESCAPE_CHAR '~'
+
+struct imap_arg;
+struct imapc_untagged_reply;
+struct imapc_command_reply;
+struct imapc_mailbox;
+struct imapc_storage_client;
+
+typedef void imapc_storage_callback_t(const struct imapc_untagged_reply *reply,
+				      struct imapc_storage_client *client);
+typedef void imapc_mailbox_callback_t(const struct imapc_untagged_reply *reply,
+				      struct imapc_mailbox *mbox);
+
+struct imapc_storage_event_callback {
+	char *name;
+	imapc_storage_callback_t *callback;
+};
+
+struct imapc_mailbox_event_callback {
+	const char *name;
+	imapc_mailbox_callback_t *callback;
+};
+
+#define IMAPC_HAS_FEATURE(mstorage, feature) \
+	(((mstorage)->set->parsed_features & feature) != 0)
+#define IMAPC_BOX_HAS_FEATURE(mbox, feature) \
+	(((mbox)->storage->set->parsed_features & feature) != 0)
+
+struct imapc_namespace {
+	const char *prefix;
+	char separator;
+	enum mail_namespace_type type;
+};
+
+struct imapc_storage_client {
+	int refcount;
+
+	/* either one of these may not be available: */
+	struct imapc_storage *_storage;
+	struct imapc_mailbox_list *_list;
+
+	struct imapc_client *client;
+
+	ARRAY(struct imapc_storage_event_callback) untagged_callbacks;
+
+	/* IMAPC_COMMAND_STATE_OK if no auth failure (yet), otherwise result to
+	   the LOGIN/AUTHENTICATE command. */
+	enum imapc_command_state auth_failed_state;
+	char *auth_failed_reason;
+
+	/* Authentication reply was received (success or failure) */
+	bool auth_returned:1;
+	bool destroying:1;
+};
+
+struct imapc_storage {
+	struct mail_storage storage;
+	const struct imapc_settings *set;
+
+	struct ioloop *root_ioloop;
+	struct imapc_storage_client *client;
+
+	struct imapc_mailbox *cur_status_box;
+	struct mailbox_status *cur_status;
+	unsigned int reopen_count;
+
+	ARRAY(struct imapc_namespace) remote_namespaces;
+
+	bool namespaces_requested:1;
+};
+
+struct imapc_mail_cache {
+	uint32_t uid;
+
+	/* either fd != -1 or buf != NULL */
+	int fd;
+	buffer_t *buf;
+};
+
+struct imapc_fetch_request {
+	ARRAY(struct imapc_mail *) mails;
+};
+
+struct imapc_untagged_fetch_ctx {
+	pool_t pool;
+
+	/* keywords, flags, guid, modseq and fetch_uid may or may not be
+	   received with an untagged fetch response */
+	ARRAY_TYPE(const_string) keywords;
+	/* Is set if have_flags is TRUE */
+	enum mail_flags flags;
+	const char *guid;
+	uint64_t modseq;
+	uint32_t fetch_uid;
+
+	/* uid is generated locally based on the remote MSN or fetch_uid */
+	uint32_t uid;
+
+	bool have_gmail_labels:1;
+	bool have_flags:1;
+};
+
+struct imapc_copy_request {
+	struct imapc_save_context *sctx;
+	struct seqset_builder *uidset_builder;
+};
+
+struct imapc_mailbox {
+	struct mailbox box;
+	struct imapc_storage *storage;
+	struct imapc_client_mailbox *client_box;
+	enum imapc_capability capabilities;
+
+	struct mail_index_transaction *delayed_sync_trans;
+	struct mail_index_view *sync_view, *delayed_sync_view;
+	struct mail_cache_view *delayed_sync_cache_view;
+	struct mail_cache_transaction_ctx *delayed_sync_cache_trans;
+	struct timeout *to_idle_check, *to_idle_delay;
+
+	ARRAY(struct imapc_fetch_request *) fetch_requests;
+	ARRAY(struct imapc_untagged_fetch_ctx *) untagged_fetch_contexts;
+	/* if non-empty, contains the latest FETCH command we're going to be
+	   sending soon (but still waiting to see if we can increase its
+	   UID range) */
+	string_t *pending_fetch_cmd;
+	/* if non-empty, contains the latest COPY command we're going to be
+	   sending soon. */
+	string_t *pending_copy_cmd;
+	char *copy_dest_box;
+	struct imapc_fetch_request *pending_fetch_request;
+	struct imapc_copy_request *pending_copy_request;
+	struct timeout *to_pending_fetch_send;
+
+	ARRAY(struct imapc_mailbox_event_callback) untagged_callbacks;
+	ARRAY(struct imapc_mailbox_event_callback) resp_text_callbacks;
+
+	enum mail_flags permanent_flags;
+	uint32_t highest_nonrecent_uid;
+
+	ARRAY(uint64_t) rseq_modseqs;
+	ARRAY_TYPE(seq_range) delayed_expunged_uids;
+	ARRAY_TYPE(seq_range) copy_rollback_expunge_uids;
+	uint32_t sync_uid_validity;
+	uint32_t sync_uid_next;
+	uint64_t sync_highestmodseq;
+	uint32_t sync_fetch_first_uid;
+	uint32_t sync_next_lseq;
+	uint32_t sync_next_rseq;
+	uint32_t exists_count;
+	uint32_t min_append_uid;
+	char *sync_gmail_pop3_search_tag;
+
+	/* keep the previous fetched message body cached,
+	   mainly for partial IMAP fetches */
+	struct imapc_mail_cache prev_mail_cache;
+
+	uint32_t prev_skipped_rseq, prev_skipped_uid;
+	struct imapc_sync_context *sync_ctx;
+
+	const char *guid_fetch_field_name;
+	struct imapc_search_context *search_ctx;
+
+	bool selecting:1;
+	bool syncing:1;
+	bool initial_sync_done:1;
+	bool selected:1;
+	bool exists_received:1;
+	bool state_fetching_uid1:1;
+	bool state_fetched_success:1;
+	bool rollback_pending:1;
+	bool delayed_untagged_exists:1;
+};
+
+struct imapc_simple_context {
+	struct imapc_storage_client *client;
+	int ret;
+};
+
+#define IMAPC_STORAGE(s)	container_of(s, struct imapc_storage, storage)
+#define IMAPC_MAILBOX(s)	container_of(s, struct imapc_mailbox, box)
+
+int imapc_storage_client_create(struct mail_namespace *ns,
+				const struct imapc_settings *imapc_set,
+				const struct mail_storage_settings *mail_set,
+				struct imapc_storage_client **client_r,
+				const char **error_r);
+void imapc_storage_client_unref(struct imapc_storage_client **client);
+bool imapc_storage_client_handle_auth_failure(struct imapc_storage_client *client);
+
+struct mail_save_context *
+imapc_save_alloc(struct mailbox_transaction_context *_t);
+int imapc_save_begin(struct mail_save_context *ctx, struct istream *input);
+int imapc_save_continue(struct mail_save_context *ctx);
+int imapc_save_finish(struct mail_save_context *ctx);
+void imapc_save_cancel(struct mail_save_context *ctx);
+int imapc_copy(struct mail_save_context *ctx, struct mail *mail);
+
+int imapc_transaction_save_commit(struct mailbox_transaction_context *t);
+int imapc_transaction_save_commit_pre(struct mail_save_context *ctx);
+void imapc_transaction_save_commit_post(struct mail_save_context *ctx,
+					struct mail_index_transaction_commit_result *result);
+void imapc_transaction_save_rollback(struct mail_save_context *ctx);
+
+void imapc_mailbox_run(struct imapc_mailbox *mbox);
+void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox);
+void imapc_mail_cache_free(struct imapc_mail_cache *cache);
+int imapc_mailbox_select(struct imapc_mailbox *mbox);
+void imap_mailbox_select_finish(struct imapc_mailbox *mbox);
+
+bool imapc_mailbox_has_modseqs(struct imapc_mailbox *mbox);
+bool imapc_resp_text_code_parse(const char *str, enum mail_error *error_r);
+bool imapc_mail_error_to_resp_text_code(enum mail_error error, const char **str_r);
+void imapc_copy_error_from_reply(struct imapc_storage *storage,
+				 enum mail_error default_error,
+				 const struct imapc_command_reply *reply);
+void imapc_simple_context_init(struct imapc_simple_context *sctx,
+			       struct imapc_storage_client *client);
+void imapc_simple_run(struct imapc_simple_context *sctx,
+		      struct imapc_command **cmd);
+void imapc_simple_callback(const struct imapc_command_reply *reply,
+			   void *context);
+int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox,
+				       bool force, bool *changes_r);
+bool imapc_mailbox_name_equals(struct imapc_mailbox *mbox,
+			       const char *remote_name);
+void imapc_mailbox_noop(struct imapc_mailbox *mbox);
+void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox,
+				 const char *reason, ...) ATTR_FORMAT(2, 3);
+const char *imapc_mailbox_get_remote_name(struct imapc_mailbox *mbox);
+
+void imapc_storage_client_register_untagged(struct imapc_storage_client *client,
+					    const char *name,
+					    imapc_storage_callback_t *callback);
+void imapc_storage_client_unregister_untagged(struct imapc_storage_client *client,
+					      const char *name);
+void imapc_mailbox_register_untagged(struct imapc_mailbox *mbox,
+				     const char *name,
+				     imapc_mailbox_callback_t *callback);
+void imapc_mailbox_register_resp_text(struct imapc_mailbox *mbox,
+				      const char *key,
+				      imapc_mailbox_callback_t *callback);
+
+void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox);
+
+struct mail_index_view *
+imapc_mailbox_get_sync_view(struct imapc_mailbox *mbox);
+
+void imapc_untagged_fetch_ctx_free(struct imapc_untagged_fetch_ctx **_ctx);
+void imapc_untagged_fetch_update_flags(struct imapc_mailbox *mbox,
+				       struct imapc_untagged_fetch_ctx *ctx,
+				       struct mail_index_view *view,
+				       uint32_t lseq);
+bool imapc_mailbox_fetch_state(struct imapc_mailbox *mbox, uint32_t first_uid);
+
+#endif
diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
new file mode 100644
index 0000000..3130121
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -0,0 +1,702 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "hash.h"
+#include "str.h"
+#include "sort.h"
+#include "imap-util.h"
+#include "mail-cache.h"
+#include "mail-index-modseq.h"
+#include "index-sync-private.h"
+#include "imapc-msgmap.h"
+#include "imapc-list.h"
+#include "imapc-storage.h"
+#include "imapc-sync.h"
+
+struct imapc_sync_command {
+	struct imapc_sync_context *ctx;
+	char *cmd_str;
+	bool ignore_no;
+};
+
+static void imapc_sync_callback(const struct imapc_command_reply *reply,
+				void *context)
+{
+	struct imapc_sync_command *cmd = context;
+	struct imapc_sync_context *ctx = cmd->ctx;
+
+	i_assert(ctx->sync_command_count > 0);
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		;
+	else if (reply->state == IMAPC_COMMAND_STATE_NO && cmd->ignore_no) {
+		/* maybe the message was expunged already.
+		   some servers fail STOREs with NO in such situation. */
+	} else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		/* the disconnection is already logged, don't flood
+		   the logs unnecessarily */
+		mail_storage_set_internal_error(&ctx->mbox->storage->storage);
+		ctx->failed = TRUE;
+	} else {
+		mailbox_set_critical(&ctx->mbox->box,
+				     "imapc: Sync command '%s' failed: %s",
+				     cmd->cmd_str, reply->text_full);
+		ctx->failed = TRUE;
+	}
+
+	if (--ctx->sync_command_count == 0)
+		imapc_client_stop(ctx->mbox->storage->client->client);
+	i_free(cmd->cmd_str);
+	i_free(cmd);
+}
+
+static struct imapc_command *
+imapc_sync_cmd_full(struct imapc_sync_context *ctx, const char *cmd_str,
+		    bool ignore_no)
+{
+	struct imapc_sync_command *sync_cmd;
+	struct imapc_command *cmd;
+
+	sync_cmd = i_new(struct imapc_sync_command, 1);
+	sync_cmd->ctx = ctx;
+	sync_cmd->cmd_str = i_strdup(cmd_str);
+	sync_cmd->ignore_no = ignore_no;
+
+	ctx->sync_command_count++;
+	cmd = imapc_client_mailbox_cmd(ctx->mbox->client_box,
+				       imapc_sync_callback, sync_cmd);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_send(cmd, cmd_str);
+	return cmd;
+}
+
+static struct imapc_command *
+imapc_sync_cmd(struct imapc_sync_context *ctx, const char *cmd_str)
+{
+	return imapc_sync_cmd_full(ctx, cmd_str, FALSE);
+}
+
+static unsigned int imapc_sync_store_hash(const struct imapc_sync_store *store)
+{
+	return str_hash(store->flags) ^ store->modify_type;
+}
+
+static int imapc_sync_store_cmp(const struct imapc_sync_store *store1,
+				const struct imapc_sync_store *store2)
+{
+	if (store1->modify_type != store2->modify_type)
+		return 1;
+	return strcmp(store1->flags, store2->flags);
+}
+
+static const char *imapc_sync_flags_sort(const char *flags)
+{
+	if (strchr(flags, ' ') == NULL)
+		return flags;
+
+	const char **str = t_strsplit(flags, " ");
+	i_qsort(str, str_array_length(str), sizeof(const char *),
+		i_strcasecmp_p);
+	return t_strarray_join(str, " ");
+}
+
+static void
+imapc_sync_store_flush(struct imapc_sync_context *ctx)
+{
+	struct imapc_sync_store *store;
+	const char *sorted_flags;
+
+	if (ctx->prev_uid1 == 0)
+		return;
+
+	sorted_flags = imapc_sync_flags_sort(str_c(ctx->prev_flags));
+	struct imapc_sync_store store_lookup = {
+		.modify_type = ctx->prev_modify_type,
+		.flags = sorted_flags,
+	};
+	store = hash_table_lookup(ctx->stores, &store_lookup);
+	if (store == NULL) {
+		store = p_new(ctx->pool, struct imapc_sync_store, 1);
+		store->modify_type = ctx->prev_modify_type;
+		store->flags = p_strdup(ctx->pool, sorted_flags);
+		p_array_init(&store->uids, ctx->pool, 4);
+		hash_table_insert(ctx->stores, store, store);
+	}
+	seq_range_array_add_range(&store->uids, ctx->prev_uid1, ctx->prev_uid2);
+}
+
+static void
+imapc_sync_store(struct imapc_sync_context *ctx,
+		 enum modify_type modify_type, uint32_t uid1, uint32_t uid2,
+		 const char *flags)
+{
+	if (ctx->prev_flags == NULL) {
+		ctx->prev_flags = str_new(ctx->pool, 128);
+		hash_table_create(&ctx->stores, ctx->pool, 0,
+				  imapc_sync_store_hash, imapc_sync_store_cmp);
+	}
+
+	if (ctx->prev_uid1 != uid1 || ctx->prev_uid2 != uid2 ||
+	    ctx->prev_modify_type != modify_type) {
+		imapc_sync_store_flush(ctx);
+		ctx->prev_uid1 = uid1;
+		ctx->prev_uid2 = uid2;
+		ctx->prev_modify_type = modify_type;
+		str_truncate(ctx->prev_flags, 0);
+	}
+	if (str_len(ctx->prev_flags) > 0)
+		str_append_c(ctx->prev_flags, ' ');
+	str_append(ctx->prev_flags, flags);
+}
+
+static void
+imapc_sync_finish_store(struct imapc_sync_context *ctx)
+{
+	struct hash_iterate_context *iter;
+	struct imapc_sync_store *store;
+	string_t *cmd = t_str_new(128);
+
+	imapc_sync_store_flush(ctx);
+
+	if (!hash_table_is_created(ctx->stores))
+		return;
+
+	iter = hash_table_iterate_init(ctx->stores);
+	while (hash_table_iterate(iter, ctx->stores, &store, &store)) {
+		str_truncate(cmd, 0);
+		str_append(cmd, "UID STORE ");
+		imap_write_seq_range(cmd, &store->uids);
+		str_printfa(cmd, " %cFLAGS (%s)",
+			    store->modify_type == MODIFY_ADD ? '+' : '-',
+			    store->flags);
+		imapc_sync_cmd_full(ctx, str_c(cmd), TRUE);
+	}
+	hash_table_iterate_deinit(&iter);
+	hash_table_destroy(&ctx->stores);
+}
+
+static void
+imapc_sync_add_missing_deleted_flags(struct imapc_sync_context *ctx,
+				     uint32_t seq1, uint32_t seq2)
+{
+	const struct mail_index_record *rec;
+	uint32_t seq, uid1, uid2;
+
+	/* if any of them has a missing \Deleted flag,
+	   just add it to all of them. */
+	for (seq = seq1; seq <= seq2; seq++) {
+		rec = mail_index_lookup(ctx->sync_view, seq);
+		if ((rec->flags & MAIL_DELETED) == 0)
+			break;
+	}
+
+	if (seq <= seq2) {
+		mail_index_lookup_uid(ctx->sync_view, seq1, &uid1);
+		mail_index_lookup_uid(ctx->sync_view, seq2, &uid2);
+
+		imapc_sync_store(ctx, MODIFY_ADD, uid1, uid2, "\\Deleted");
+	}
+}
+
+static void imapc_sync_index_flags(struct imapc_sync_context *ctx,
+				   const struct mail_index_sync_rec *sync_rec)
+{
+	string_t *str = t_str_new(128);
+
+	i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
+
+	if (sync_rec->add_flags != 0) {
+		i_assert((sync_rec->add_flags & MAIL_RECENT) == 0);
+
+		imap_write_flags(str, sync_rec->add_flags, NULL);
+		imapc_sync_store(ctx, MODIFY_ADD, sync_rec->uid1,
+				 sync_rec->uid2, str_c(str));
+	}
+
+	if (sync_rec->remove_flags != 0) {
+		i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0);
+		str_truncate(str, 0);
+		imap_write_flags(str, sync_rec->remove_flags, NULL);
+		imapc_sync_store(ctx, MODIFY_REMOVE, sync_rec->uid1,
+				 sync_rec->uid2, str_c(str));
+	}
+}
+
+static void
+imapc_sync_index_keyword(struct imapc_sync_context *ctx,
+			 const struct mail_index_sync_rec *sync_rec)
+{
+	const char *kw_str;
+	enum modify_type modify_type;
+
+	switch (sync_rec->type) {
+	case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
+		modify_type = MODIFY_ADD;
+		break;
+	case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
+		modify_type = MODIFY_REMOVE;
+		break;
+	default:
+		i_unreached();
+	}
+
+	kw_str = array_idx_elem(ctx->keywords, sync_rec->keyword_idx);
+	imapc_sync_store(ctx, modify_type, sync_rec->uid1,
+			 sync_rec->uid2, kw_str);
+}
+
+static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx)
+{
+	string_t *str;
+
+	if (array_count(&ctx->expunged_uids) == 0)
+		return;
+
+	if ((ctx->mbox->capabilities & IMAPC_CAPABILITY_UIDPLUS) == 0) {
+		/* just expunge everything */
+		imapc_sync_cmd(ctx, "EXPUNGE");
+		return;
+	}
+
+	/* build a list of UIDs to expunge */
+	str = t_str_new(128);
+	str_append(str, "UID EXPUNGE ");
+	imap_write_seq_range(str, &ctx->expunged_uids);
+	imapc_sync_cmd(ctx, str_c(str));
+}
+
+static void imapc_sync_uid_next(struct imapc_sync_context *ctx)
+{
+	struct imapc_mailbox *mbox = ctx->mbox;
+	const struct mail_index_header *hdr;
+	uint32_t uid_next = mbox->sync_uid_next;
+
+	if (uid_next < mbox->min_append_uid)
+		uid_next = mbox->min_append_uid;
+
+	hdr = mail_index_get_header(ctx->sync_view);
+	if (hdr->next_uid < uid_next) {
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, next_uid),
+			&uid_next, sizeof(uid_next), FALSE);
+	}
+}
+
+static void imapc_sync_highestmodseq(struct imapc_sync_context *ctx)
+{
+	if (imapc_mailbox_has_modseqs(ctx->mbox) &&
+	    mail_index_modseq_get_highest(ctx->sync_view) < ctx->mbox->sync_highestmodseq)
+		mail_index_update_highest_modseq(ctx->trans, ctx->mbox->sync_highestmodseq);
+}
+
+static void
+imapc_initial_sync_check(struct imapc_sync_context *ctx, bool nooped)
+{
+	struct imapc_msgmap *msgmap =
+		imapc_client_mailbox_get_msgmap(ctx->mbox->client_box);
+	struct mail_index_view *view = ctx->mbox->delayed_sync_view;
+	const struct mail_index_header *hdr = mail_index_get_header(view);
+	uint32_t rseq, lseq, ruid, luid, rcount, lcount;
+
+	rseq = lseq = 1;
+	rcount = imapc_msgmap_count(msgmap);
+	lcount = mail_index_view_get_messages_count(view);
+	while (rseq <= rcount || lseq <= lcount) {
+		if (rseq <= rcount)
+			ruid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
+		else
+			ruid = (uint32_t)-1;
+		if (lseq <= lcount)
+			mail_index_lookup_uid(view, lseq, &luid);
+		else
+			luid = (uint32_t)-1;
+
+		if (ruid == luid) {
+			/* message exists in index and in remote server */
+			lseq++; rseq++;
+		} else if (luid < ruid) {
+			/* message exists in index but not in remote server */
+			if (luid >= ctx->mbox->sync_uid_next) {
+				/* the message was added to index by another
+				   imapc session, and it's not visible yet
+				   in this session */
+				break;
+			}
+			/* it's already expunged and we should have marked it */
+			i_assert(mail_index_is_expunged(view, lseq) ||
+				 seq_range_exists(&ctx->mbox->delayed_expunged_uids, luid));
+			lseq++;
+		} else {
+			/* message doesn't exist in index, but exists in
+			   remote server */
+			if (lseq > lcount && ruid >= hdr->next_uid) {
+				/* the message hasn't been yet added to index */
+				break;
+			}
+
+			/* another imapc session expunged it =>
+			   NOOP should send us an EXPUNGE event */
+			if (!nooped) {
+				imapc_mailbox_noop(ctx->mbox);
+				imapc_initial_sync_check(ctx, TRUE);
+				return;
+			}
+			/* already nooped => index is corrupted */
+			imapc_mailbox_set_corrupted(ctx->mbox,
+				"Expunged message uid=%u reappeared", ruid);
+			ctx->failed = TRUE;
+			return;
+		}
+	}
+}
+
+static void
+imapc_sync_send_commands(struct imapc_sync_context *ctx)
+{
+	if (ctx->mbox->exists_count == 0) {
+		/* empty mailbox - no point in fetching anything */
+		return;
+	}
+
+	if (IMAPC_BOX_HAS_FEATURE(ctx->mbox, IMAPC_FEATURE_GMAIL_MIGRATION) &&
+	    ctx->mbox->storage->set->pop3_deleted_flag[0] != '\0') {
+		struct imapc_command *cmd;
+
+		cmd = imapc_sync_cmd(ctx, "SEARCH RETURN (ALL) X-GM-RAW \"in:^pop\"");
+		i_free(ctx->mbox->sync_gmail_pop3_search_tag);
+		ctx->mbox->sync_gmail_pop3_search_tag =
+			i_strdup(imapc_command_get_tag(cmd));
+	}
+}
+
+static void imapc_sync_index(struct imapc_sync_context *ctx)
+{
+	struct imapc_mailbox *mbox = ctx->mbox;
+	struct mail_index_sync_rec sync_rec;
+	uint32_t seq1, seq2;
+
+	i_array_init(&ctx->expunged_uids, 64);
+	ctx->keywords = mail_index_get_keywords(mbox->box.index);
+	ctx->pool = pool_alloconly_create("imapc sync pool", 1024);
+
+	while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN {
+		if (!mail_index_lookup_seq_range(ctx->sync_view,
+						 sync_rec.uid1, sync_rec.uid2,
+						 &seq1, &seq2)) {
+			/* already expunged, nothing to do. */
+		} else switch (sync_rec.type) {
+		case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
+			imapc_sync_add_missing_deleted_flags(ctx, seq1, seq2);
+			seq_range_array_add_range(&ctx->expunged_uids,
+						  sync_rec.uid1, sync_rec.uid2);
+			break;
+		case MAIL_INDEX_SYNC_TYPE_FLAGS:
+			imapc_sync_index_flags(ctx, &sync_rec);
+			break;
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
+			imapc_sync_index_keyword(ctx, &sync_rec);
+			break;
+		}
+	} T_END;
+	imapc_sync_finish_store(ctx);
+	pool_unref(&ctx->pool);
+
+	if (!mbox->initial_sync_done)
+		imapc_sync_send_commands(ctx);
+
+	imapc_sync_expunge_finish(ctx);
+	while (ctx->sync_command_count > 0)
+		imapc_mailbox_run(mbox);
+	array_free(&ctx->expunged_uids);
+
+	if (!mbox->state_fetched_success) {
+		/* All the sync commands succeeded, but we got disconnected.
+		   imapc_initial_sync_check() will crash if we go there. */
+		ctx->failed = TRUE;
+	}
+
+	/* add uidnext & highestmodseq after all appends */
+	imapc_sync_uid_next(ctx);
+	imapc_sync_highestmodseq(ctx);
+
+	mailbox_sync_notify(&mbox->box, 0, 0);
+
+	if (!ctx->failed) {
+		/* reset only after a successful sync */
+		mbox->sync_fetch_first_uid = 0;
+	}
+	if (!mbox->initial_sync_done && !ctx->failed) {
+		imapc_initial_sync_check(ctx, FALSE);
+		mbox->initial_sync_done = TRUE;
+	}
+}
+
+static int
+imapc_sync_begin(struct imapc_mailbox *mbox,
+		 struct imapc_sync_context **ctx_r, bool force)
+{
+	struct imapc_sync_context *ctx;
+	enum mail_index_sync_flags sync_flags;
+	int ret;
+
+	i_assert(!mbox->syncing);
+
+	ctx = i_new(struct imapc_sync_context, 1);
+	ctx->mbox = mbox;
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box) |
+		MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
+	if (!force)
+		sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
+
+	ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx,
+				    &ctx->sync_view, &ctx->trans,
+				    sync_flags);
+	if (ret <= 0) {
+		if (ret < 0)
+			mailbox_set_index_error(&mbox->box);
+		i_free(ctx);
+		*ctx_r = NULL;
+		return ret;
+	}
+
+	i_assert(mbox->sync_view == NULL);
+	i_assert(mbox->delayed_sync_trans == NULL);
+	mbox->sync_view = ctx->sync_view;
+	mbox->delayed_sync_view =
+		mail_index_transaction_open_updated_view(ctx->trans);
+	mbox->delayed_sync_trans = ctx->trans;
+	mbox->delayed_sync_cache_view =
+		mail_cache_view_open(mbox->box.cache, mbox->delayed_sync_view);
+	mbox->delayed_sync_cache_trans =
+		mail_cache_get_transaction(mbox->delayed_sync_cache_view,
+					   mbox->delayed_sync_trans);
+	mbox->min_append_uid = mail_index_get_header(ctx->sync_view)->next_uid;
+
+	mbox->syncing = TRUE;
+	mbox->sync_ctx = ctx;
+
+	if (mbox->delayed_untagged_exists) {
+		bool fetch_send = imapc_mailbox_fetch_state(mbox,
+							    mbox->min_append_uid);
+		while (fetch_send && mbox->delayed_untagged_exists)
+			imapc_mailbox_run(mbox);
+	}
+
+	if (!mbox->box.deleting)
+		imapc_sync_index(ctx);
+
+	mail_index_view_close(&mbox->delayed_sync_view);
+	mbox->delayed_sync_trans = NULL;
+	mbox->sync_view = NULL;
+
+	*ctx_r = ctx;
+	return 0;
+}
+
+static int imapc_sync_finish(struct imapc_sync_context **_ctx)
+{
+	struct imapc_sync_context *ctx = *_ctx;
+	bool changes;
+	int ret = ctx->failed ? -1 : 0;
+
+	*_ctx = NULL;
+	/* Commit the transaction even if we failed. This is important, because
+	   during the sync delayed_sync_trans points to the sync transaction.
+	   Even if the syncing doesn't fully succeed, we don't want to lose
+	   changes in delayed_sync_trans. */
+	if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
+		mailbox_set_index_error(&ctx->mbox->box);
+		ret = -1;
+	}
+	if (ctx->mbox->sync_gmail_pop3_search_tag != NULL) {
+		mailbox_set_critical(&ctx->mbox->box,
+			"gmail-pop3 search not successful");
+		i_free_and_null(ctx->mbox->sync_gmail_pop3_search_tag);
+		ret = -1;
+	}
+	mail_cache_view_close(&ctx->mbox->delayed_sync_cache_view);
+	ctx->mbox->delayed_sync_cache_trans = NULL;
+
+	ctx->mbox->syncing = FALSE;
+	ctx->mbox->sync_ctx = NULL;
+
+	/* this is done simply to commit delayed expunges if there are any
+	   (has to be done after sync is committed) */
+	if (imapc_mailbox_commit_delayed_trans(ctx->mbox, FALSE, &changes) < 0)
+		ctx->failed = TRUE;
+
+	i_free(ctx);
+	return ret;
+}
+
+static int imapc_untagged_fetch_uid_cmp(struct imapc_untagged_fetch_ctx *const *ctx1,
+					struct imapc_untagged_fetch_ctx *const *ctx2)
+{
+	return (*ctx1)->uid < (*ctx2)->uid ? -1 :
+		(*ctx1)->uid > (*ctx2)->uid ? 1 : 0;
+}
+
+static void imapc_sync_handle_untagged_fetches(struct imapc_mailbox *mbox)
+{
+	struct imapc_untagged_fetch_ctx *untagged_fetch_context;
+	struct mail_index_view *updated_view;
+	uint32_t lseq;
+
+	i_assert(array_count(&mbox->untagged_fetch_contexts) > 0);
+	i_assert(mbox->delayed_sync_trans == NULL);
+
+	array_sort(&mbox->untagged_fetch_contexts, imapc_untagged_fetch_uid_cmp);
+
+	mbox->delayed_sync_trans =
+		mail_index_transaction_begin(imapc_mailbox_get_sync_view(mbox),
+						     MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+
+	array_foreach_elem(&mbox->untagged_fetch_contexts, untagged_fetch_context) {
+		if (untagged_fetch_context->uid < mbox->min_append_uid ||
+		    untagged_fetch_context->uid < mail_index_get_header(mbox->sync_view)->next_uid) {
+			/* The message was already added */
+			continue;
+		}
+
+		mail_index_append(mbox->delayed_sync_trans,
+				  untagged_fetch_context->uid,
+				  &lseq);
+		mbox->min_append_uid = untagged_fetch_context->uid + 1;
+	}
+
+	updated_view = mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);
+
+	array_foreach_elem(&mbox->untagged_fetch_contexts, untagged_fetch_context) {
+		if (!untagged_fetch_context->have_flags) {
+			imapc_untagged_fetch_ctx_free(&untagged_fetch_context);
+			continue;
+		}
+
+		/* Lookup the mail belonging to this context using the
+		   context->uid */
+		if (!mail_index_lookup_seq(updated_view,
+					   untagged_fetch_context->uid,
+					   &lseq)) {
+			/* mail is expunged already */
+			imapc_untagged_fetch_ctx_free(&untagged_fetch_context);
+			continue;
+		}
+
+		imapc_untagged_fetch_update_flags(mbox, untagged_fetch_context,
+						  updated_view, lseq);
+		imapc_untagged_fetch_ctx_free(&untagged_fetch_context);
+	}
+
+	mail_index_view_close(&updated_view);
+	if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0)
+		mailbox_set_index_error(&mbox->box);
+	array_clear(&mbox->untagged_fetch_contexts);
+}
+
+static int imapc_sync(struct imapc_mailbox *mbox)
+{
+	struct imapc_sync_context *sync_ctx;
+	bool force = mbox->sync_fetch_first_uid != 0;
+
+	if ((mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0) {
+		/* we're only saving mails here - no syncing actually wanted */
+		return 0;
+	}
+
+	if (imapc_sync_begin(mbox, &sync_ctx, force) < 0)
+		return -1;
+
+	if (!array_is_empty(&mbox->untagged_fetch_contexts))
+		imapc_sync_handle_untagged_fetches(mbox);
+
+	if (sync_ctx == NULL)
+		return 0;
+	if (imapc_sync_finish(&sync_ctx) < 0)
+		return -1;
+	return 0;
+}
+
+static void
+imapc_noop_if_needed(struct imapc_mailbox *mbox, enum mailbox_sync_flags flags)
+{
+	if (!mbox->initial_sync_done) {
+		/* we just SELECTed/EXAMINEd the mailbox, don't do another
+		   NOOP. */
+	} else if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 &&
+		   ((mbox->capabilities & IMAPC_CAPABILITY_IDLE) == 0 ||
+		    (flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0)) {
+		/* do NOOP to make sure we have the latest changes before
+		   starting sync. this is necessary either because se don't
+		   support IDLE at all, or because we want to be sure that we
+		   have the latest changes (IDLE is started with a small delay,
+		   so we might not actually even be in IDLE right not) */
+		imapc_mailbox_noop(mbox);
+	}
+}
+
+static bool imapc_mailbox_need_initial_fetch(struct imapc_mailbox *mbox)
+{
+	if (mbox->box.deleting) {
+		/* If the mailbox is about to be deleted there is no need to
+		   expect initial fetch to be done */
+		return FALSE;
+	}
+	if ((mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0) {
+		/* The mailbox is opened only for saving there is no need to
+		   expect initial fetchting do be done. */
+		return FALSE;
+	}
+	return TRUE;
+}
+
+struct mailbox_sync_context *
+imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(box);
+	struct imapc_mailbox_list *list = mbox->storage->client->_list;
+	bool changes;
+	int ret = 0;
+
+	if (list != NULL) {
+		if (!list->refreshed_mailboxes &&
+		    list->last_refreshed_mailboxes < ioloop_time)
+			list->refreshed_mailboxes_recently = FALSE;
+	}
+
+	imapc_noop_if_needed(mbox, flags);
+
+	if (imapc_storage_client_handle_auth_failure(mbox->storage->client))
+		ret = -1;
+	else if (!mbox->state_fetched_success && !mbox->state_fetching_uid1 &&
+		 imapc_mailbox_need_initial_fetch(mbox)) {
+		/* initial FETCH failed already */
+		ret = -1;
+	}
+	if (imapc_mailbox_commit_delayed_trans(mbox, FALSE, &changes) < 0)
+		ret = -1;
+	if ((changes || mbox->sync_fetch_first_uid != 0 ||
+	     index_mailbox_want_full_sync(&mbox->box, flags)) &&
+	    ret == 0)
+		ret = imapc_sync(mbox);
+
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
+
+int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
+			      struct mailbox_sync_status *status_r)
+{
+	struct imapc_mailbox *mbox = IMAPC_MAILBOX(ctx->box);
+	int ret;
+
+	ret = index_mailbox_sync_deinit(ctx, status_r);
+	ctx = NULL;
+
+	if (mbox->client_box == NULL)
+		return ret;
+
+	imapc_client_mailbox_idle(mbox->client_box);
+	return ret;
+}
diff --git a/src/lib-storage/index/imapc/imapc-sync.h b/src/lib-storage/index/imapc/imapc-sync.h
new file mode 100644
index 0000000..54a0368
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-sync.h
@@ -0,0 +1,39 @@
+#ifndef IMAPC_SYNC_H
+#define IMAPC_SYNC_H
+
+struct mailbox;
+struct mailbox_sync_status;
+
+struct imapc_sync_store {
+	enum modify_type modify_type;
+	const char *flags;
+
+	ARRAY_TYPE(seq_range) uids;
+};
+
+struct imapc_sync_context {
+	struct imapc_mailbox *mbox;
+        struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *trans;
+
+	const ARRAY_TYPE(keywords) *keywords;
+	ARRAY_TYPE(seq_range) expunged_uids;
+	unsigned int sync_command_count;
+
+	pool_t pool;
+	HASH_TABLE(struct imapc_sync_store *, struct imapc_sync_store *) stores;
+
+	uint32_t prev_uid1, prev_uid2;
+	enum modify_type prev_modify_type;
+	string_t *prev_flags;
+
+	bool failed:1;
+};
+
+struct mailbox_sync_context *
+imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
+			      struct mailbox_sync_status *status_r);
+
+#endif
diff --git a/src/lib-storage/index/index-attachment.c b/src/lib-storage/index/index-attachment.c
new file mode 100644
index 0000000..6e51fab
--- /dev/null
+++ b/src/lib-storage/index/index-attachment.c
@@ -0,0 +1,446 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "safe-mkstemp.h"
+#include "fs-api.h"
+#include "istream.h"
+#include "ostream.h"
+#include "base64.h"
+#include "hash-format.h"
+#include "str.h"
+#include "message-parser.h"
+#include "rfc822-parser.h"
+#include "fs-api.h"
+#include "istream-fs-file.h"
+#include "istream-attachment-connector.h"
+#include "istream-attachment-extractor.h"
+#include "mail-user.h"
+#include "index-mail.h"
+#include "index-attachment.h"
+
+enum mail_attachment_decode_option {
+	MAIL_ATTACHMENT_DECODE_OPTION_NONE = '-',
+	MAIL_ATTACHMENT_DECODE_OPTION_BASE64 = 'B',
+	MAIL_ATTACHMENT_DECODE_OPTION_CRLF = 'C'
+};
+
+struct mail_save_attachment {
+	pool_t pool;
+	struct fs *fs;
+	struct istream *input;
+
+	struct fs_file *cur_file;
+	ARRAY_TYPE(mail_attachment_extref) extrefs;
+};
+
+static const char *index_attachment_dir_get(struct mail_storage *storage)
+{
+	return mail_user_home_expand(storage->user,
+				     storage->set->mail_attachment_dir);
+}
+
+static bool index_attachment_want(const struct istream_attachment_header *hdr,
+				  void *context)
+{
+	struct mail_save_context *ctx = context;
+	struct mail_attachment_part apart;
+
+	i_zero(&apart);
+	apart.part = hdr->part;
+	apart.content_type = hdr->content_type;
+	apart.content_disposition = hdr->content_disposition;
+
+	if (ctx->part_is_attachment != NULL)
+		return ctx->part_is_attachment(ctx, &apart);
+
+	/* don't treat text/ parts as attachments */
+	return hdr->content_type != NULL &&
+		strncasecmp(hdr->content_type, "text/", 5) != 0;
+}
+
+static int index_attachment_open_temp_fd(void *context)
+{
+	struct mail_save_context *ctx = context;
+	struct mail_storage *storage = ctx->transaction->box->storage;
+	string_t *temp_path;
+	int fd;
+
+	temp_path = t_str_new(256);
+	mail_user_set_get_temp_prefix(temp_path, storage->user->set);
+	fd = safe_mkstemp_hostpid(temp_path, 0600, (uid_t)-1, (gid_t)-1);
+	if (fd == -1) {
+		mailbox_set_critical(ctx->transaction->box,
+			"safe_mkstemp(%s) failed: %m", str_c(temp_path));
+		return -1;
+	}
+	if (unlink(str_c(temp_path)) < 0) {
+		mailbox_set_critical(ctx->transaction->box,
+			"unlink(%s) failed: %m", str_c(temp_path));
+		i_close_fd(&fd);
+		return -1;
+	}
+	return fd;
+}
+
+static int
+index_attachment_open_ostream(struct istream_attachment_info *info,
+			      struct ostream **output_r,
+			      const char **error_r ATTR_UNUSED, void *context)
+{
+	struct mail_save_context *ctx = context;
+	struct mail_save_attachment *attach = ctx->data.attach;
+	struct mail_storage *storage = ctx->transaction->box->storage;
+	struct mail_attachment_extref *extref;
+	enum fs_open_flags flags = 0;
+	const char *attachment_dir, *path, *digest = info->hash;
+	guid_128_t guid_128;
+
+	i_assert(attach->cur_file == NULL);
+
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER)
+		flags |= FS_OPEN_FLAG_FSYNC;
+
+	if (strlen(digest) < 4) {
+		/* make sure we can access first 4 bytes without accessing
+		   out of bounds memory */
+		digest = t_strconcat(digest, "\0\0\0\0", NULL);
+	}
+
+	guid_128_generate(guid_128);
+	attachment_dir = index_attachment_dir_get(storage);
+	path = t_strdup_printf("%s/%c%c/%c%c/%s-%s", attachment_dir,
+			       digest[0], digest[1],
+			       digest[2], digest[3], digest,
+			       guid_128_to_string(guid_128));
+	attach->cur_file = fs_file_init(attach->fs, path,
+					FS_OPEN_MODE_REPLACE | flags);
+
+	extref = array_append_space(&attach->extrefs);
+	extref->start_offset = info->start_offset;
+	extref->size = info->encoded_size;
+	extref->path = p_strdup(attach->pool,
+				path + strlen(attachment_dir) + 1);
+	extref->base64_blocks_per_line = info->base64_blocks_per_line;
+	extref->base64_have_crlf = info->base64_have_crlf;
+
+	*output_r = fs_write_stream(attach->cur_file);
+	return 0;
+}
+
+static int
+index_attachment_close_ostream(struct ostream *output, bool success,
+			       const char **error, void *context)
+{
+	struct mail_save_context *ctx = context;
+	struct mail_save_attachment *attach = ctx->data.attach;
+	int ret = success ? 0 : -1;
+
+	i_assert(attach->cur_file != NULL);
+
+	if (ret < 0)
+		fs_write_stream_abort_error(attach->cur_file, &output, "%s", *error);
+	else if (fs_write_stream_finish(attach->cur_file, &output) < 0) {
+		*error = t_strdup_printf("Couldn't create attachment %s: %s",
+					 fs_file_path(attach->cur_file),
+					 fs_file_last_error(attach->cur_file));
+		ret = -1;
+	}
+	fs_file_deinit(&attach->cur_file);
+
+	if (ret < 0) {
+		array_pop_back(&attach->extrefs);
+	}
+	return ret;
+}
+
+void index_attachment_save_begin(struct mail_save_context *ctx,
+				 struct fs *fs, struct istream *input)
+{
+	struct mail_storage *storage = ctx->transaction->box->storage;
+	struct mail_save_attachment *attach;
+	struct istream_attachment_settings set;
+	const char *error;
+	pool_t pool;
+
+	i_assert(ctx->data.attach == NULL);
+
+	if (*storage->set->mail_attachment_dir == '\0')
+		return;
+
+	i_zero(&set);
+	set.min_size = storage->set->mail_attachment_min_size;
+	if (hash_format_init(storage->set->mail_attachment_hash,
+			     &set.hash_format, &error) < 0) {
+		/* we already checked this when verifying settings */
+		i_panic("mail_attachment_hash=%s unexpectedly failed: %s",
+			storage->set->mail_attachment_hash, error);
+	}
+	set.want_attachment = index_attachment_want;
+	set.open_temp_fd = index_attachment_open_temp_fd;
+	set.open_attachment_ostream = index_attachment_open_ostream;
+	set.close_attachment_ostream = index_attachment_close_ostream;
+
+	pool = pool_alloconly_create("save attachment", 1024);
+	attach = p_new(pool, struct mail_save_attachment, 1);
+	attach->pool = pool;
+	attach->fs = fs;
+	attach->input = i_stream_create_attachment_extractor(input, &set, ctx);
+	p_array_init(&attach->extrefs, attach->pool, 8);
+	ctx->data.attach = attach;
+}
+
+static int save_check_write_error(struct mail_save_context *ctx,
+				  struct ostream *output)
+{
+	struct mail_storage *storage = ctx->transaction->box->storage;
+
+	if (output->stream_errno == 0)
+		return 0;
+
+	if (!mail_storage_set_error_from_errno(storage)) {
+		mail_set_critical(ctx->dest_mail, "write(%s) failed: %s",
+			o_stream_get_name(output), o_stream_get_error(output));
+	}
+	return -1;
+}
+
+int index_attachment_save_continue(struct mail_save_context *ctx)
+{
+	struct mail_save_attachment *attach = ctx->data.attach;
+	const unsigned char *data;
+	size_t size;
+	ssize_t ret;
+
+	if (attach->input->stream_errno != 0)
+		return -1;
+
+	do {
+		ret = i_stream_read(attach->input);
+		if (ret > 0 || ret == -2) {
+			data = i_stream_get_data(attach->input, &size);
+			o_stream_nsend(ctx->data.output, data, size);
+			i_stream_skip(attach->input, size);
+		}
+		index_mail_cache_parse_continue(ctx->dest_mail);
+		if (ret == 0 && !i_stream_attachment_extractor_can_retry(attach->input)) {
+			/* need more input */
+			return 0;
+		}
+	} while (ret != -1);
+
+	if (attach->input->stream_errno != 0) {
+		mail_set_critical(ctx->dest_mail, "read(%s) failed: %s",
+					  i_stream_get_name(attach->input),
+					  i_stream_get_error(attach->input));
+		return -1;
+	}
+	if (ctx->data.output != NULL) {
+		if (save_check_write_error(ctx, ctx->data.output) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+int index_attachment_save_finish(struct mail_save_context *ctx)
+{
+	struct mail_save_attachment *attach = ctx->data.attach;
+
+	(void)i_stream_read(attach->input);
+	i_assert(attach->input->eof);
+	return attach->input->stream_errno == 0 ? 0 : -1;
+}
+
+void index_attachment_save_free(struct mail_save_context *ctx)
+{
+	struct mail_save_attachment *attach = ctx->data.attach;
+
+	if (attach != NULL) {
+		i_stream_unref(&attach->input);
+		pool_unref(&attach->pool);
+		ctx->data.attach = NULL;
+	}
+}
+
+const ARRAY_TYPE(mail_attachment_extref) *
+index_attachment_save_get_extrefs(struct mail_save_context *ctx)
+{
+	return ctx->data.attach == NULL ? NULL :
+		&ctx->data.attach->extrefs;
+}
+
+static int
+index_attachment_delete_real(struct mail_storage *storage,
+			     struct fs *fs, const char *name)
+{
+	struct fs_file *file;
+	const char *path;
+	int ret;
+
+	path = t_strdup_printf("%s/%s", index_attachment_dir_get(storage), name);
+	file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+	if ((ret = fs_delete(file)) < 0)
+		mail_storage_set_critical(storage, "%s", fs_file_last_error(file));
+	fs_file_deinit(&file);
+	return ret;
+}
+
+int index_attachment_delete(struct mail_storage *storage,
+			    struct fs *fs, const char *name)
+{
+	int ret;
+
+	T_BEGIN {
+		ret = index_attachment_delete_real(storage, fs, name);
+	} T_END;
+	return ret;
+}
+
+void index_attachment_append_extrefs(string_t *str,
+	const ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+	const struct mail_attachment_extref *extref;
+	bool add_space = FALSE;
+	unsigned int startpos;
+
+	array_foreach(extrefs, extref) {
+		if (!add_space)
+			add_space = TRUE;
+		else
+			str_append_c(str, ' ');
+		str_printfa(str, "%"PRIuUOFF_T" %"PRIuUOFF_T" ",
+			    extref->start_offset, extref->size);
+
+		startpos = str_len(str);
+		if (extref->base64_have_crlf)
+			str_append_c(str, MAIL_ATTACHMENT_DECODE_OPTION_CRLF);
+		if (extref->base64_blocks_per_line > 0) {
+			str_printfa(str, "%c%u",
+				    MAIL_ATTACHMENT_DECODE_OPTION_BASE64,
+				    extref->base64_blocks_per_line * 4);
+		}
+		if (startpos == str_len(str)) {
+			/* make it clear there are no options */
+			str_append_c(str, MAIL_ATTACHMENT_DECODE_OPTION_NONE);
+		}
+		str_append_c(str, ' ');
+		str_append(str, extref->path);
+	}
+}
+
+static bool
+parse_extref_decode_options(const char *str,
+			    struct mail_attachment_extref *extref)
+{
+	unsigned int num;
+
+	if (*str == MAIL_ATTACHMENT_DECODE_OPTION_NONE)
+		return str[1] == '\0';
+
+	while (*str != '\0') {
+		switch (*str) {
+		case MAIL_ATTACHMENT_DECODE_OPTION_BASE64:
+			str++; num = 0;
+			while (*str >= '0' && *str <= '9') {
+				num = num*10 + (*str-'0');
+				str++;
+			}
+			if (num == 0 || num % 4 != 0)
+				return FALSE;
+
+			extref->base64_blocks_per_line = num/4;
+			break;
+		case MAIL_ATTACHMENT_DECODE_OPTION_CRLF:
+			extref->base64_have_crlf = TRUE;
+			str++;
+			break;
+		default:
+			return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+bool index_attachment_parse_extrefs(const char *line, pool_t pool,
+				    ARRAY_TYPE(mail_attachment_extref) *extrefs)
+{
+	struct mail_attachment_extref extref;
+	const char *const *args;
+	unsigned int i, len;
+	uoff_t last_voffset;
+
+	args = t_strsplit(line, " ");
+	len = str_array_length(args);
+	if ((len % 4) != 0)
+		return FALSE;
+
+	last_voffset = 0;
+	for (i = 0; args[i] != NULL; i += 4) {
+		const char *start_offset_str = args[i+0];
+		const char *size_str = args[i+1];
+		const char *decode_options = args[i+2];
+		const char *path = args[i+3];
+
+		i_zero(&extref);
+		if (str_to_uoff(start_offset_str, &extref.start_offset) < 0 ||
+		    str_to_uoff(size_str, &extref.size) < 0 ||
+		    extref.start_offset < last_voffset ||
+		    !parse_extref_decode_options(decode_options, &extref))
+			return FALSE;
+
+		last_voffset += extref.size +
+			(extref.start_offset - last_voffset);
+
+		extref.path = p_strdup(pool, path);
+		array_push_back(extrefs, &extref);
+	}
+	return TRUE;
+}
+
+int index_attachment_stream_get(struct fs *fs, const char *attachment_dir,
+				const char *path_suffix,
+				struct istream **stream, uoff_t full_size,
+				const char *ext_refs, const char **error_r)
+{
+	ARRAY_TYPE(mail_attachment_extref) extrefs_arr;
+	const struct mail_attachment_extref *extref;
+	struct istream_attachment_connector *conn;
+	struct istream *input;
+	struct fs_file *file;
+	const char *path;
+	int ret;
+
+	*error_r = NULL;
+
+	t_array_init(&extrefs_arr, 16);
+	if (!index_attachment_parse_extrefs(ext_refs, pool_datastack_create(),
+					    &extrefs_arr)) {
+		*error_r = "Broken ext-refs string";
+		return -1;
+	}
+	conn = istream_attachment_connector_begin(*stream, full_size);
+
+	array_foreach(&extrefs_arr, extref) {
+		path = t_strdup_printf("%s/%s%s", attachment_dir,
+				       extref->path, path_suffix);
+		file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY |
+				    FS_OPEN_FLAG_SEEKABLE);
+		input = i_stream_create_fs_file(&file, IO_BLOCK_SIZE);
+
+		ret = istream_attachment_connector_add(conn, input,
+					extref->start_offset, extref->size,
+					extref->base64_blocks_per_line,
+					extref->base64_have_crlf, error_r);
+		i_stream_unref(&input);
+		if (ret < 0) {
+			istream_attachment_connector_abort(&conn);
+			return -1;
+		}
+	}
+
+	input = istream_attachment_connector_finish(&conn);
+	i_stream_set_name(input, t_strdup_printf(
+		"attachments-connector(%s)", i_stream_get_name(*stream)));
+	i_stream_unref(stream);
+	*stream = input;
+	return 0;
+}
diff --git a/src/lib-storage/index/index-attachment.h b/src/lib-storage/index/index-attachment.h
new file mode 100644
index 0000000..a52f158
--- /dev/null
+++ b/src/lib-storage/index/index-attachment.h
@@ -0,0 +1,52 @@
+#ifndef INDEX_ATTACHMENT_H
+#define INDEX_ATTACHMENT_H
+
+#include "sha1.h"
+
+struct fs;
+struct mail_save_context;
+struct mail_storage;
+
+struct mail_attachment_extref {
+	/* path without attachment_dir/ prefix */
+	const char *path;
+	/* offset in input stream where part begins */
+	uoff_t start_offset;
+	uoff_t size;
+
+	/* If non-zero, this attachment was saved as base64-decoded and it
+	   need to be encoded back before presenting it to client. Each line
+	   (except last one) consists of this many base64 blocks (4 chars of
+	   base64 encoded data). */
+	unsigned int base64_blocks_per_line;
+	/* Line feeds are CRLF instead of LF */
+	bool base64_have_crlf;
+};
+ARRAY_DEFINE_TYPE(mail_attachment_extref, struct mail_attachment_extref);
+
+void index_attachment_save_begin(struct mail_save_context *ctx,
+				 struct fs *fs, struct istream *input);
+int index_attachment_save_continue(struct mail_save_context *ctx);
+int index_attachment_save_finish(struct mail_save_context *ctx);
+void index_attachment_save_free(struct mail_save_context *ctx);
+const ARRAY_TYPE(mail_attachment_extref) *
+index_attachment_save_get_extrefs(struct mail_save_context *ctx);
+
+/* Delete a given attachment name from storage
+   (name is same as mail_attachment_extref.name). */
+int index_attachment_delete(struct mail_storage *storage,
+			    struct fs *fs, const char *name);
+
+void index_attachment_append_extrefs(string_t *str,
+	const ARRAY_TYPE(mail_attachment_extref) *extrefs);
+/* Parse extrefs value to given array. Names are allocated from the
+   given pool. */
+bool index_attachment_parse_extrefs(const char *line, pool_t pool,
+				    ARRAY_TYPE(mail_attachment_extref) *extrefs);
+
+int index_attachment_stream_get(struct fs *fs, const char *attachment_dir,
+				const char *path_suffix,
+				struct istream **stream, uoff_t full_size,
+				const char *ext_refs, const char **error_r);
+
+#endif
diff --git a/src/lib-storage/index/index-attribute.c b/src/lib-storage/index/index-attribute.c
new file mode 100644
index 0000000..3d4c415
--- /dev/null
+++ b/src/lib-storage/index/index-attribute.c
@@ -0,0 +1,333 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "dict.h"
+#include "index-storage.h"
+
+struct index_storage_attribute_iter {
+	struct mailbox_attribute_iter iter;
+	struct dict_iterate_context *diter;
+	char *prefix;
+	size_t prefix_len;
+	bool dict_disabled;
+};
+
+static struct mail_namespace *
+mail_user_find_attribute_namespace(struct mail_user *user)
+{
+	struct mail_namespace *ns;
+
+	ns = mail_namespace_find_inbox(user->namespaces);
+	if (ns != NULL)
+		return ns;
+
+	for (ns = user->namespaces; ns != NULL; ns = ns->next) {
+		if (ns->type == MAIL_NAMESPACE_TYPE_PRIVATE)
+			return ns;
+	}
+	return NULL;
+}
+
+static int
+index_storage_get_user_dict(struct mail_storage *err_storage,
+			    struct mail_user *user, struct dict **dict_r)
+{
+	struct dict_settings dict_set;
+	struct mail_namespace *ns;
+	struct mail_storage *attr_storage;
+	const char *error;
+
+	if (user->_attr_dict != NULL) {
+		*dict_r = user->_attr_dict;
+		return 0;
+	}
+	if (user->attr_dict_failed) {
+		mail_storage_set_internal_error(err_storage);
+		return -1;
+	}
+
+	ns = mail_user_find_attribute_namespace(user);
+	if (ns == NULL) {
+		/* probably never happens? */
+		mail_storage_set_error(err_storage, MAIL_ERROR_NOTPOSSIBLE,
+			"Mailbox attributes not available for this mailbox");
+		return -1;
+	}
+	attr_storage = mail_namespace_get_default_storage(ns);
+
+	if (*attr_storage->set->mail_attribute_dict == '\0') {
+		mail_storage_set_error(err_storage, MAIL_ERROR_NOTPOSSIBLE,
+				       "Mailbox attributes not enabled");
+		return -1;
+	}
+
+	i_zero(&dict_set);
+	dict_set.base_dir = user->set->base_dir;
+	dict_set.event_parent = user->event;
+	if (dict_init(attr_storage->set->mail_attribute_dict, &dict_set,
+		      &user->_attr_dict, &error) < 0) {
+		mail_storage_set_critical(err_storage,
+			"mail_attribute_dict: dict_init(%s) failed: %s",
+			attr_storage->set->mail_attribute_dict, error);
+		user->attr_dict_failed = TRUE;
+		return -1;
+	}
+	*dict_r = user->_attr_dict;
+	return 0;
+}
+
+static int
+index_storage_get_dict(struct mailbox *box, enum mail_attribute_type type_flags,
+		       struct dict **dict_r, const char **mailbox_prefix_r)
+{
+	enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK;
+	struct mail_storage *storage = box->storage;
+	struct mail_namespace *ns;
+	struct mailbox_metadata metadata;
+	struct dict_settings set;
+	const char *error;
+
+	if ((type_flags & MAIL_ATTRIBUTE_TYPE_FLAG_VALIDATED) != 0) {
+		/* IMAP METADATA support isn't enabled, so don't allow using
+		   mail_attribute_dict. */
+		mail_storage_set_error(storage, MAIL_ERROR_NOTPOSSIBLE,
+				       "Generic mailbox attributes not enabled");
+		return -1;
+	}
+
+	if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0)
+		return -1;
+	*mailbox_prefix_r = guid_128_to_string(metadata.guid);
+
+	ns = mailbox_get_namespace(box);
+	if (type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
+		/* private attributes are stored in user's own dict */
+		return index_storage_get_user_dict(storage, storage->user, dict_r);
+	} else if (ns->user == ns->owner) {
+		/* user owns the mailbox. shared attributes are stored in
+		   the same dict. */
+		return index_storage_get_user_dict(storage, storage->user, dict_r);
+	} else if (ns->owner != NULL) {
+		/* accessing shared attribute of a shared mailbox.
+		   use the owner's dict. */
+		return index_storage_get_user_dict(storage, ns->owner, dict_r);
+	}
+
+	/* accessing shared attributes of a public mailbox. no user owns it,
+	   so use the storage's dict. */
+	if (storage->_shared_attr_dict != NULL) {
+		*dict_r = storage->_shared_attr_dict;
+		return 0;
+	}
+	if (*storage->set->mail_attribute_dict == '\0') {
+		mail_storage_set_error(storage, MAIL_ERROR_NOTPOSSIBLE,
+				       "Mailbox attributes not enabled");
+		return -1;
+	}
+	if (storage->shared_attr_dict_failed) {
+		mail_storage_set_internal_error(storage);
+		return -1;
+	}
+
+	i_zero(&set);
+	set.base_dir = storage->user->set->base_dir;
+	set.event_parent = storage->user->event;
+	if (dict_init(storage->set->mail_attribute_dict, &set,
+		      &storage->_shared_attr_dict, &error) < 0) {
+		mail_storage_set_critical(storage,
+			"mail_attribute_dict: dict_init(%s) failed: %s",
+			storage->set->mail_attribute_dict, error);
+		storage->shared_attr_dict_failed = TRUE;
+		return -1;
+	}
+	*dict_r = storage->_shared_attr_dict;
+	return 0;
+}
+
+static const char *
+key_get_prefixed(enum mail_attribute_type type_flags, const char *mailbox_prefix,
+		 const char *key)
+{
+	enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK;
+
+	switch (type) {
+	case MAIL_ATTRIBUTE_TYPE_PRIVATE:
+		return t_strconcat(DICT_PATH_PRIVATE, mailbox_prefix, "/",
+				   key, NULL);
+	case MAIL_ATTRIBUTE_TYPE_SHARED:
+		return t_strconcat(DICT_PATH_SHARED, mailbox_prefix, "/",
+				   key, NULL);
+	}
+	i_unreached();
+}
+
+static int
+index_storage_attribute_get_dict_trans(struct mailbox_transaction_context *t,
+				       enum mail_attribute_type type_flags,
+				       struct dict_transaction_context **dtrans_r,
+				       const char **mailbox_prefix_r)
+{
+	enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK;
+	struct dict_transaction_context **dtransp = NULL;
+	struct dict *dict;
+	struct mailbox_metadata metadata;
+
+	switch (type) {
+	case MAIL_ATTRIBUTE_TYPE_PRIVATE:
+		dtransp = &t->attr_pvt_trans;
+		break;
+	case MAIL_ATTRIBUTE_TYPE_SHARED:
+		dtransp = &t->attr_shared_trans;
+		break;
+	}
+	i_assert(dtransp != NULL);
+
+	if (*dtransp != NULL &&
+	    (type_flags & MAIL_ATTRIBUTE_TYPE_FLAG_VALIDATED) == 0) {
+		/* Transaction already created. Even if it was, don't use it
+		   if _FLAG_VALIDATED is being used. It'll be handled below by
+		   returning failure. */
+		if (mailbox_get_metadata(t->box, MAILBOX_METADATA_GUID,
+					 &metadata) < 0)
+			return -1;
+		*mailbox_prefix_r = guid_128_to_string(metadata.guid);
+		*dtrans_r = *dtransp;
+		return 0;
+	}
+
+	if (index_storage_get_dict(t->box, type_flags, &dict, mailbox_prefix_r) < 0)
+		return -1;
+	i_assert(*dtransp == NULL);
+
+	struct mail_user *user = mailbox_list_get_user(t->box->list);
+	const struct dict_op_settings *set = mail_user_get_dict_op_settings(user);
+	*dtransp = *dtrans_r = dict_transaction_begin(dict, set);
+	return 0;
+}
+
+int index_storage_attribute_set(struct mailbox_transaction_context *t,
+				enum mail_attribute_type type_flags,
+				const char *key,
+				const struct mail_attribute_value *value)
+{
+	enum mail_attribute_type type = type_flags & MAIL_ATTRIBUTE_TYPE_MASK;
+	struct dict_transaction_context *dtrans;
+	const char *mailbox_prefix;
+	bool pvt = type == MAIL_ATTRIBUTE_TYPE_PRIVATE;
+	time_t ts = value->last_change != 0 ? value->last_change : ioloop_time;
+	int ret = 0;
+
+	if (index_storage_attribute_get_dict_trans(t, type_flags, &dtrans,
+						   &mailbox_prefix) < 0)
+		return -1;
+
+	T_BEGIN {
+		const char *prefixed_key =
+			key_get_prefixed(type_flags, mailbox_prefix, key);
+		const char *value_str;
+
+		if (mailbox_attribute_value_to_string(t->box->storage, value,
+						      &value_str) < 0) {
+			ret = -1;
+		} else if (value_str != NULL) {
+			dict_set(dtrans, prefixed_key, value_str);
+			mail_index_attribute_set(t->itrans, pvt, key,
+						 ts, strlen(value_str));
+		} else {
+			dict_unset(dtrans, prefixed_key);
+			mail_index_attribute_unset(t->itrans, pvt, key, ts);
+		}
+	} T_END;
+	return ret;
+}
+
+int index_storage_attribute_get(struct mailbox *box,
+				enum mail_attribute_type type_flags,
+				const char *key,
+				struct mail_attribute_value *value_r)
+{
+	struct dict *dict;
+	const char *mailbox_prefix, *error;
+	int ret;
+
+	i_zero(value_r);
+
+	if (index_storage_get_dict(box, type_flags, &dict, &mailbox_prefix) < 0)
+		return -1;
+
+	struct mail_user *user = mailbox_list_get_user(box->list);
+	const struct dict_op_settings *set = mail_user_get_dict_op_settings(user);
+	ret = dict_lookup(dict, set, pool_datastack_create(),
+			  key_get_prefixed(type_flags, mailbox_prefix, key),
+			  &value_r->value, &error);
+	if (ret < 0) {
+		mailbox_set_critical(box,
+			"Failed to get attribute %s: %s", key, error);
+		return -1;
+	}
+	return ret;
+}
+
+struct mailbox_attribute_iter *
+index_storage_attribute_iter_init(struct mailbox *box,
+				  enum mail_attribute_type type_flags,
+				  const char *prefix)
+{
+	struct index_storage_attribute_iter *iter;
+	struct dict *dict;
+	const char *mailbox_prefix;
+
+	iter = i_new(struct index_storage_attribute_iter, 1);
+	iter->iter.box = box;
+	if (index_storage_get_dict(box, type_flags, &dict, &mailbox_prefix) < 0) {
+		if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTPOSSIBLE)
+			iter->dict_disabled = TRUE;
+	} else {
+		iter->prefix = i_strdup(key_get_prefixed(type_flags, mailbox_prefix,
+							 prefix));
+		iter->prefix_len = strlen(iter->prefix);
+		struct mail_user *user = mailbox_list_get_user(box->list);
+		const struct dict_op_settings *set = mail_user_get_dict_op_settings(user);
+		iter->diter = dict_iterate_init(dict, set, iter->prefix,
+						DICT_ITERATE_FLAG_RECURSE |
+						DICT_ITERATE_FLAG_NO_VALUE);
+	}
+	return &iter->iter;
+}
+
+const char *
+index_storage_attribute_iter_next(struct mailbox_attribute_iter *_iter)
+{
+	struct index_storage_attribute_iter *iter =
+		(struct index_storage_attribute_iter *)_iter;
+	const char *key, *value;
+
+	if (iter->diter == NULL || !dict_iterate(iter->diter, &key, &value))
+		return NULL;
+
+	i_assert(strncmp(key, iter->prefix, iter->prefix_len) == 0);
+	key += iter->prefix_len;
+	return key;
+}
+
+int index_storage_attribute_iter_deinit(struct mailbox_attribute_iter *_iter)
+{
+	struct index_storage_attribute_iter *iter =
+		(struct index_storage_attribute_iter *)_iter;
+	const char *error;
+	int ret;
+
+	if (iter->diter == NULL) {
+		ret = iter->dict_disabled ? 0 : -1;
+	} else {
+		if ((ret = dict_iterate_deinit(&iter->diter, &error)) < 0) {
+			mailbox_set_critical(_iter->box,
+				"dict_iterate(%s) failed: %s",
+				iter->prefix, error);
+		}
+	}
+	i_free(iter->prefix);
+	i_free(iter);
+	return ret;
+}
diff --git a/src/lib-storage/index/index-mail-binary.c b/src/lib-storage/index/index-mail-binary.c
new file mode 100644
index 0000000..80c319e
--- /dev/null
+++ b/src/lib-storage/index/index-mail-binary.c
@@ -0,0 +1,598 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "safe-mkstemp.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "istream-seekable.h"
+#include "istream-base64.h"
+#include "istream-qp.h"
+#include "istream-header-filter.h"
+#include "ostream.h"
+#include "message-binary-part.h"
+#include "message-parser.h"
+#include "message-decoder.h"
+#include "mail-user.h"
+#include "index-storage.h"
+#include "index-mail.h"
+
+#define MAIL_BINARY_CACHE_EXPIRE_MSECS (60*1000)
+
+#define IS_CONVERTED_CTE(cte) \
+	((cte) == MESSAGE_CTE_QP || (cte) == MESSAGE_CTE_BASE64)
+
+struct binary_block {
+	struct istream *input;
+	uoff_t physical_pos;
+	unsigned int body_lines_count;
+	bool converted, converted_hdr;
+};
+
+struct binary_ctx {
+	struct mail *mail;
+	struct istream *input;
+	bool has_nuls, converted;
+	/* each block is its own input stream. basically each converted MIME
+	   body has its own block and the parts between the MIME bodies are
+	   unconverted blocks */
+	ARRAY(struct binary_block) blocks;
+
+	uoff_t copy_start_offset;
+};
+
+static void binary_copy_to(struct binary_ctx *ctx, uoff_t end_offset)
+{
+	struct binary_block *block;
+	struct istream *linput, *cinput;
+	uoff_t orig_offset, size;
+
+	i_assert(end_offset >= ctx->copy_start_offset);
+
+	if (end_offset == ctx->copy_start_offset)
+		return;
+
+	size = end_offset - ctx->copy_start_offset;
+	orig_offset = ctx->input->v_offset;
+
+	i_stream_seek(ctx->input, ctx->copy_start_offset);
+	linput = i_stream_create_limit(ctx->input, size);
+	cinput = i_stream_create_crlf(linput);
+	i_stream_unref(&linput);
+
+	block = array_append_space(&ctx->blocks);
+	block->input = cinput;
+
+	i_stream_seek(ctx->input, orig_offset);
+}
+
+static void
+binary_cte_filter_callback(struct header_filter_istream *input,
+			   struct message_header_line *hdr,
+			   bool *matched ATTR_UNUSED, void *context ATTR_UNUSED)
+{
+	static const char *cte_binary = "Content-Transfer-Encoding: binary\r\n";
+
+	if (hdr != NULL && hdr->eoh) {
+		i_stream_header_filter_add(input, cte_binary,
+					   strlen(cte_binary));
+	}
+}
+
+static int
+add_binary_part(struct binary_ctx *ctx, const struct message_part *part,
+		bool include_hdr)
+{
+	static const char *filter_headers[] = {
+		"Content-Transfer-Encoding",
+	};
+	struct message_header_parser_ctx *parser;
+	struct message_header_line *hdr;
+	struct message_part *child;
+	struct message_size hdr_size;
+	struct istream *linput;
+	struct binary_block *block;
+	enum message_cte cte;
+	uoff_t part_end_offset;
+	int ret;
+
+	/* first parse the header to find c-t-e. */
+	i_stream_seek(ctx->input, part->physical_pos);
+
+	cte = MESSAGE_CTE_78BIT;
+	parser = message_parse_header_init(ctx->input, &hdr_size, 0);
+	while ((ret = message_parse_header_next(parser, &hdr)) > 0) {
+		if (strcasecmp(hdr->name, "Content-Transfer-Encoding") == 0)
+			cte = message_decoder_parse_cte(hdr);
+	}
+	i_assert(ret < 0);
+	if (message_parse_header_has_nuls(parser)) {
+		/* we're not converting NULs to 0x80 when doing a binary fetch,
+		   even if they're in the message header. */
+		ctx->has_nuls = TRUE;
+	}
+	message_parse_header_deinit(&parser);
+
+	if (ctx->input->stream_errno != 0) {
+		mail_set_critical(ctx->mail,
+			"read(%s) failed: %s", i_stream_get_name(ctx->input),
+			i_stream_get_error(ctx->input));
+		return -1;
+	}
+
+	if (cte == MESSAGE_CTE_UNKNOWN) {
+		mail_storage_set_error(ctx->mail->box->storage,
+				       MAIL_ERROR_CONVERSION,
+				       "Unknown Content-Transfer-Encoding.");
+		return -1;
+	}
+
+	i_stream_seek(ctx->input, part->physical_pos);
+	if (!include_hdr) {
+		/* body only */
+	} else if (IS_CONVERTED_CTE(cte)) {
+		/* write header with modified content-type */
+		if (ctx->copy_start_offset != 0)
+			binary_copy_to(ctx, part->physical_pos);
+		block = array_append_space(&ctx->blocks);
+		block->physical_pos = part->physical_pos;
+		block->converted = TRUE;
+		block->converted_hdr = TRUE;
+
+		linput = i_stream_create_limit(ctx->input, UOFF_T_MAX);
+		block->input = i_stream_create_header_filter(linput,
+				HEADER_FILTER_EXCLUDE | HEADER_FILTER_HIDE_BODY,
+				filter_headers, N_ELEMENTS(filter_headers),
+				binary_cte_filter_callback, NULL);
+		i_stream_unref(&linput);
+	} else {
+		/* copy everything as-is until the end of this header */
+		binary_copy_to(ctx, part->physical_pos +
+			       part->header_size.physical_size);
+	}
+	ctx->copy_start_offset = part->physical_pos +
+		part->header_size.physical_size;
+	part_end_offset = part->physical_pos +
+		part->header_size.physical_size +
+		part->body_size.physical_size;
+
+	if (part->children != NULL) {
+		/* multipart */
+		for (child = part->children; child != NULL; child = child->next) {
+			if (add_binary_part(ctx, child, TRUE) < 0)
+				return -1;
+		}
+		binary_copy_to(ctx, part_end_offset);
+		ctx->copy_start_offset = part_end_offset;
+		return 0;
+	}
+	if (part->body_size.physical_size == 0) {
+		/* no body */
+		ctx->copy_start_offset = part_end_offset;
+		return 0;
+	}
+
+	/* single part - write decoded data */
+	block = array_append_space(&ctx->blocks);
+	block->physical_pos = part->physical_pos;
+
+	i_stream_seek(ctx->input, part->physical_pos +
+		      part->header_size.physical_size);
+	linput = i_stream_create_limit(ctx->input, part->body_size.physical_size);
+	switch (cte) {
+	case MESSAGE_CTE_UNKNOWN:
+		i_unreached();
+	case MESSAGE_CTE_78BIT:
+	case MESSAGE_CTE_BINARY:
+		/* no conversion necessary */
+		if ((part->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0)
+			ctx->has_nuls = TRUE;
+		block->input = i_stream_create_crlf(linput);
+		break;
+	case MESSAGE_CTE_QP:
+		block->input = i_stream_create_qp_decoder(linput);
+		ctx->converted = block->converted = TRUE;
+		break;
+	case MESSAGE_CTE_BASE64:
+		block->input = i_stream_create_base64_decoder(linput);
+		ctx->converted = block->converted = TRUE;
+		break;
+	}
+	i_stream_unref(&linput);
+
+	ctx->copy_start_offset = part_end_offset;
+	return 0;
+}
+
+static int fd_callback(const char **path_r, void *context)
+{
+	struct mail *_mail = context;
+	string_t *path;
+	int fd;
+
+	path = t_str_new(256);
+	mail_user_set_get_temp_prefix(path, _mail->box->storage->user->set);
+	fd = safe_mkstemp_hostpid(path, 0600, (uid_t)-1, (gid_t)-1);
+	if (fd == -1) {
+		i_error("Temp file creation to %s failed: %m", str_c(path));
+		return -1;
+	}
+
+	/* we just want the fd, unlink it */
+	if (i_unlink(str_c(path)) < 0) {
+		/* shouldn't happen.. */
+		i_close_fd(&fd);
+		return -1;
+	}
+	*path_r = str_c(path);
+	return fd;
+}
+
+static void binary_streams_free(struct binary_ctx *ctx)
+{
+	struct binary_block *block;
+
+	array_foreach_modifiable(&ctx->blocks, block)
+		i_stream_unref(&block->input);
+}
+
+static void
+binary_parts_update(struct binary_ctx *ctx, const struct message_part *part,
+		    struct message_binary_part **msg_bin_parts)
+{
+	struct index_mail *mail = INDEX_MAIL(ctx->mail);
+	struct binary_block *blocks;
+	struct message_binary_part bin_part;
+	unsigned int i, count;
+	uoff_t size;
+	bool found;
+
+	blocks = array_get_modifiable(&ctx->blocks, &count);
+	for (; part != NULL; part = part->next) {
+		binary_parts_update(ctx, part->children, msg_bin_parts);
+
+		i_zero(&bin_part);
+		/* default to unchanged header */
+		bin_part.binary_hdr_size = part->header_size.virtual_size;
+		bin_part.physical_pos = part->physical_pos;
+		found = FALSE;
+		for (i = 0; i < count; i++) {
+			if (blocks[i].physical_pos != part->physical_pos ||
+			    !blocks[i].converted)
+				continue;
+
+			size = blocks[i].input->v_offset;
+			if (blocks[i].converted_hdr)
+				bin_part.binary_hdr_size = size;
+			else
+				bin_part.binary_body_size = size;
+			found = TRUE;
+		}
+		if (found) {
+			bin_part.next = *msg_bin_parts;
+			*msg_bin_parts = p_new(mail->mail.data_pool,
+					       struct message_binary_part, 1);
+			**msg_bin_parts = bin_part;
+		}
+	}
+}
+
+static void binary_parts_cache(struct binary_ctx *ctx)
+{
+	struct index_mail *mail = INDEX_MAIL(ctx->mail);
+	buffer_t *buf;
+
+	buf = t_buffer_create(128);
+	message_binary_part_serialize(mail->data.bin_parts, buf);
+	index_mail_cache_add(mail, MAIL_CACHE_BINARY_PARTS,
+			     buf->data, buf->used);
+}
+
+static struct istream **blocks_get_streams(struct binary_ctx *ctx)
+{
+	struct istream **streams;
+	const struct binary_block *blocks;
+	unsigned int i, count;
+
+	blocks = array_get(&ctx->blocks, &count);
+	streams = t_new(struct istream *, count+1);
+	for (i = 0; i < count; i++) {
+		streams[i] = blocks[i].input;
+		i_assert(streams[i]->v_offset == 0);
+	}
+	return streams;
+}
+
+static int
+blocks_count_lines(struct binary_ctx *ctx, struct istream *full_input)
+{
+	struct binary_block *blocks, *cur_block;
+	unsigned int block_idx, block_count;
+	uoff_t cur_block_offset, cur_block_size;
+	const unsigned char *data, *p;
+	size_t size, skip;
+	ssize_t ret;
+
+	blocks = array_get_modifiable(&ctx->blocks, &block_count);
+	cur_block = blocks;
+	cur_block_offset = 0;
+	block_idx = 0;
+
+	/* count the number of lines each block contains */
+	while ((ret = i_stream_read_more(full_input, &data, &size)) > 0) {
+		i_assert(cur_block_offset <= cur_block->input->v_offset);
+		if (cur_block->input->eof) {
+			/* this is the last input for this block. the input
+			   may also contain the next block's data, which we
+			   don't want to include in this block's line count. */
+			cur_block_size = cur_block->input->v_offset +
+				i_stream_get_data_size(cur_block->input);
+			i_assert(size >= cur_block_size - cur_block_offset);
+			size = cur_block_size - cur_block_offset;
+		}
+		skip = size;
+		while ((p = memchr(data, '\n', size)) != NULL) {
+			size -= p-data+1;
+			data = p+1;
+			cur_block->body_lines_count++;
+		}
+		i_stream_skip(full_input, skip);
+		cur_block_offset += skip;
+
+		if (i_stream_read_eof(cur_block->input)) {
+			/* go to the next block */
+			if (block_idx+1 == block_count) {
+				i_assert(i_stream_read_eof(full_input));
+				ret = -1;
+				break;
+			}
+			block_idx++;
+			cur_block++;
+			cur_block_offset = 0;
+		}
+	}
+	i_assert(ret == -1);
+	if (full_input->stream_errno != 0)
+		return -1;
+	i_assert(block_count == 0 || !i_stream_have_bytes_left(cur_block->input));
+	i_assert(block_count == 0 || block_idx+1 == block_count);
+	return 0;
+}
+
+static int
+index_mail_read_binary_to_cache(struct mail *_mail,
+				const struct message_part *part,
+				bool include_hdr, const char *reason,
+				bool *binary_r, bool *converted_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct mail_binary_cache *cache = &_mail->box->storage->binary_cache;
+	struct binary_ctx ctx;
+	struct istream *is;
+
+	i_zero(&ctx);
+	ctx.mail = _mail;
+	t_array_init(&ctx.blocks, 8);
+
+	mail_storage_free_binary_cache(_mail->box->storage);
+	if (mail_get_stream_because(_mail, NULL, NULL, reason, &ctx.input) < 0)
+		return -1;
+
+	if (add_binary_part(&ctx, part, include_hdr) < 0) {
+		binary_streams_free(&ctx);
+		return -1;
+	}
+
+	if (array_count(&ctx.blocks) != 0) {
+		is = i_streams_merge(blocks_get_streams(&ctx),
+					       IO_BLOCK_SIZE,
+					       fd_callback, _mail);
+	} else {
+		is = i_stream_create_from_data("", 0);
+	}
+	i_stream_set_name(is, t_strdup_printf(
+		"<binary stream of mailbox %s UID %u>",
+		_mail->box->vname, _mail->uid));
+	if (blocks_count_lines(&ctx, is) < 0) {
+		if (is->stream_errno == EINVAL) {
+			/* MIME part contains invalid data */
+			mail_storage_set_error(_mail->box->storage,
+					       MAIL_ERROR_INVALIDDATA,
+					       "Invalid data in MIME part");
+		} else {
+			mail_set_critical(_mail, "read(%s) failed: %s",
+					  i_stream_get_name(is),
+					  i_stream_get_error(is));
+		}
+		i_stream_unref(&is);
+		binary_streams_free(&ctx);
+		return -1;
+	}
+
+	if (_mail->uid > 0) {
+		cache->to = timeout_add(MAIL_BINARY_CACHE_EXPIRE_MSECS,
+					mail_storage_free_binary_cache,
+					_mail->box->storage);
+		cache->box = _mail->box;
+		cache->uid = _mail->uid;
+		cache->orig_physical_pos = part->physical_pos;
+		cache->include_hdr = include_hdr;
+		cache->input = is;
+	}
+
+	i_assert(!i_stream_have_bytes_left(is));
+	cache->size = is->v_offset;
+	i_stream_seek(is, 0);
+
+	if (part->parent == NULL && include_hdr &&
+	    mail->data.bin_parts == NULL) {
+		binary_parts_update(&ctx, part, &mail->data.bin_parts);
+		if (_mail->uid > 0)
+			binary_parts_cache(&ctx);
+	}
+	binary_streams_free(&ctx);
+
+	*binary_r = ctx.converted ? TRUE : ctx.has_nuls;
+	*converted_r = ctx.converted;
+	return 0;
+}
+
+static bool get_cached_binary_parts(struct index_mail *mail)
+{
+	const unsigned int field_idx =
+		mail->ibox->cache_fields[MAIL_CACHE_BINARY_PARTS].idx;
+	buffer_t *part_buf;
+	int ret;
+
+	if (mail->data.bin_parts != NULL)
+		return TRUE;
+
+	part_buf = t_buffer_create(128);
+	ret = index_mail_cache_lookup_field(mail, part_buf, field_idx);
+	if (ret <= 0)
+		return FALSE;
+
+	if (message_binary_part_deserialize(mail->mail.data_pool,
+					    part_buf->data, part_buf->used,
+					    &mail->data.bin_parts) < 0) {
+		mail_set_mail_cache_corrupted(&mail->mail.mail,
+			"Corrupted cached binary.parts data");
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static struct message_part *
+msg_part_find(struct message_part *parts, uoff_t physical_pos)
+{
+	struct message_part *part, *child;
+
+	for (part = parts; part != NULL; part = part->next) {
+		if (part->physical_pos == physical_pos)
+			return part;
+		child = msg_part_find(part->children, physical_pos);
+		if (child != NULL)
+			return child;
+	}
+	return NULL;
+}
+
+static int
+index_mail_get_binary_size(struct mail *_mail,
+			   const struct message_part *part, bool include_hdr,
+			   uoff_t *size_r, unsigned int *lines_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct message_part *all_parts, *msg_part;
+	const struct message_binary_part *bin_part, *root_bin_part;
+	uoff_t size, end_offset;
+	unsigned int lines;
+	bool binary, converted;
+
+	if (mail_get_parts(_mail, &all_parts) < 0)
+		return -1;
+
+	/* first lookup from cache */
+	if (!get_cached_binary_parts(mail)) {
+		/* not found. parse the whole message */
+		if (index_mail_read_binary_to_cache(_mail, all_parts, TRUE,
+						    "binary.size", &binary, &converted) < 0)
+			return -1;
+	}
+
+	size = part->header_size.virtual_size +
+		part->body_size.virtual_size;
+	/* note that we assume here that binary translation doesn't change the
+	   headers' line counts. this isn't true if the original message
+	   contained duplicate Content-Transfer-Encoding lines, but since
+	   that's invalid anyway we don't bother trying to handle it. */
+	lines = part->header_size.lines + part->body_size.lines;
+	end_offset = part->physical_pos + size;
+
+	bin_part = mail->data.bin_parts; root_bin_part = NULL;
+	for (; bin_part != NULL; bin_part = bin_part->next) {
+		msg_part = msg_part_find(all_parts, bin_part->physical_pos);
+		if (msg_part == NULL) {
+			/* either binary.parts or mime.parts is broken */
+			mail_set_cache_corrupted(_mail, MAIL_FETCH_MESSAGE_PARTS, t_strdup_printf(
+				"BINARY part at offset %"PRIuUOFF_T" not found from MIME parts",
+				bin_part->physical_pos));
+			return -1;
+		}
+		if (msg_part->physical_pos >= part->physical_pos &&
+		    msg_part->physical_pos < end_offset) {
+			if (msg_part->physical_pos == part->physical_pos)
+				root_bin_part = bin_part;
+			size -= msg_part->header_size.virtual_size +
+				msg_part->body_size.virtual_size;
+			size += bin_part->binary_hdr_size +
+				bin_part->binary_body_size;
+			lines -= msg_part->body_size.lines;
+			lines += bin_part->binary_body_lines_count;
+		}
+	}
+	if (!include_hdr) {
+		if (root_bin_part != NULL)
+			size -= root_bin_part->binary_hdr_size;
+		else
+			size -= part->header_size.virtual_size;
+		lines -= part->header_size.lines;
+	}
+	*size_r = size;
+	*lines_r = lines;
+	return 0;
+}
+
+int index_mail_get_binary_stream(struct mail *_mail,
+				 const struct message_part *part,
+				 bool include_hdr, uoff_t *size_r,
+				 unsigned int *lines_r, bool *binary_r,
+				 struct istream **stream_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct mail_binary_cache *cache = &_mail->box->storage->binary_cache;
+	struct istream *input;
+	bool binary, converted;
+
+	if (stream_r == NULL) {
+		return index_mail_get_binary_size(_mail, part, include_hdr,
+						  size_r, lines_r);
+	}
+	/* current implementation doesn't bother implementing this,
+	   because it's not needed by anything. */
+	i_assert(lines_r == NULL);
+
+	/* FIXME: always put the header to temp file. skip it when needed. */
+	if (cache->box == _mail->box && cache->uid == _mail->uid &&
+	    cache->orig_physical_pos == part->physical_pos &&
+	    cache->include_hdr == include_hdr) {
+		/* we have this cached already */
+		i_stream_seek(cache->input, 0);
+		timeout_reset(cache->to);
+		binary = TRUE;
+		converted = TRUE;
+	} else {
+		if (index_mail_read_binary_to_cache(_mail, part, include_hdr,
+						    "binary stream", &binary, &converted) < 0)
+			return -1;
+		mail->data.cache_fetch_fields |= MAIL_FETCH_STREAM_BINARY;
+	}
+	*size_r = cache->size;
+	*binary_r = binary;
+	if (!converted) {
+		/* don't keep this cached. it's exactly the same as
+		   the original stream */
+		i_assert(mail->data.stream != NULL);
+		i_stream_seek(mail->data.stream, part->physical_pos +
+			      (include_hdr ? 0 :
+			       part->header_size.physical_size));
+		input = i_stream_create_crlf(mail->data.stream);
+		*stream_r = i_stream_create_limit(input, *size_r);
+		i_stream_unref(&input);
+		mail_storage_free_binary_cache(_mail->box->storage);
+	} else {
+		*stream_r = cache->input;
+		i_stream_ref(cache->input);
+	}
+	return 0;
+}
diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c
new file mode 100644
index 0000000..ce23e9d
--- /dev/null
+++ b/src/lib-storage/index/index-mail-headers.c
@@ -0,0 +1,990 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "array.h"
+#include "buffer.h"
+#include "str.h"
+#include "message-date.h"
+#include "message-part-data.h"
+#include "message-parser.h"
+#include "message-header-decode.h"
+#include "istream-tee.h"
+#include "istream-header-filter.h"
+#include "imap-envelope.h"
+#include "imap-bodystructure.h"
+#include "index-storage.h"
+#include "index-mail.h"
+
+static const struct message_parser_settings msg_parser_set = {
+	.hdr_flags = MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP |
+		MESSAGE_HEADER_PARSER_FLAG_DROP_CR,
+	.flags = MESSAGE_PARSER_FLAG_SKIP_BODY_BLOCK,
+};
+
+static void index_mail_filter_stream_destroy(struct index_mail *mail);
+
+static int header_line_cmp(const struct index_mail_line *l1,
+			   const struct index_mail_line *l2)
+{
+	int diff;
+
+	diff = (int)l1->field_idx - (int)l2->field_idx;
+	return diff != 0 ? diff :
+		(int)l1->line_num - (int)l2->line_num;
+}
+
+void index_mail_parse_header_deinit(struct index_mail *mail)
+{
+	mail->data.header_parser_initialized = FALSE;
+}
+
+static void index_mail_parse_header_finish(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	const struct index_mail_line *lines;
+	const unsigned char *header;
+	const uint8_t *match;
+	buffer_t *buf;
+	unsigned int i, j, count, match_idx, match_count;
+	bool noncontiguous;
+
+	/* sort it first so fields are grouped together and ordered by
+	   line number */
+	array_sort(&mail->header_lines, header_line_cmp);
+
+	lines = array_get(&mail->header_lines, &count);
+	match = array_get(&mail->header_match, &match_count);
+	header = mail->header_data->data;
+	buf = t_buffer_create(256);
+
+	/* go through all the header lines we found */
+	for (i = match_idx = 0; i < count; i = j) {
+		/* matches and header lines are both sorted, all matches
+		   until lines[i] weren't found */
+		while (match_idx < lines[i].field_idx &&
+		       match_idx < match_count) {
+			if (HEADER_MATCH_USABLE(mail, match[match_idx]) &&
+			    mail_cache_field_can_add(_mail->transaction->cache_trans,
+						     _mail->seq, match_idx)) {
+				/* this header doesn't exist. remember that. */
+				i_assert((match[match_idx] &
+					  HEADER_MATCH_FLAG_FOUND) == 0);
+				index_mail_cache_add_idx(mail, match_idx,
+							 "", 0);
+			}
+			match_idx++;
+		}
+
+		if (match_idx < match_count) {
+			/* save index to first header line */
+			i_assert(match_idx == lines[i].field_idx);
+			j = i + 1;
+			array_idx_set(&mail->header_match_lines, match_idx, &j);
+			match_idx++;
+		}
+
+		if (!mail_cache_field_can_add(_mail->transaction->cache_trans,
+					      _mail->seq, lines[i].field_idx)) {
+			/* header is already cached. skip over all the
+			   header lines. */
+			for (j = i+1; j < count; j++) {
+				if (lines[j].field_idx != lines[i].field_idx)
+					break;
+			}
+			continue;
+		}
+
+		/* buffer contains: { uint32_t line_num[], 0, header texts }
+		   noncontiguous is just a small optimization.. */
+		buffer_set_used_size(buf, 0);
+		buffer_append(buf, &lines[i].line_num,
+			      sizeof(lines[i].line_num));
+
+		noncontiguous = FALSE;
+		for (j = i+1; j < count; j++) {
+			if (lines[j].field_idx != lines[i].field_idx)
+				break;
+
+			if (lines[j].start_pos != lines[j-1].end_pos)
+				noncontiguous = TRUE;
+			buffer_append(buf, &lines[j].line_num,
+				      sizeof(lines[j].line_num));
+		}
+		buffer_append_zero(buf, sizeof(uint32_t));
+
+		if (noncontiguous) {
+			for (; i < j; i++) {
+				buffer_append(buf, header + lines[i].start_pos,
+					      lines[i].end_pos -
+					      lines[i].start_pos);
+			}
+			i--;
+		} else {
+			buffer_append(buf, header + lines[i].start_pos,
+				      lines[j-1].end_pos - lines[i].start_pos);
+		}
+
+		index_mail_cache_add_idx(mail, lines[i].field_idx,
+					 buf->data, buf->used);
+	}
+
+	for (; match_idx < match_count; match_idx++) {
+		if (HEADER_MATCH_USABLE(mail, match[match_idx]) &&
+		    mail_cache_field_can_add(_mail->transaction->cache_trans,
+					     _mail->seq, match_idx)) {
+			/* this header doesn't exist. remember that. */
+			i_assert((match[match_idx] &
+				  HEADER_MATCH_FLAG_FOUND) == 0);
+			index_mail_cache_add_idx(mail, match_idx, "", 0);
+		}
+	}
+
+	mail->data.dont_cache_field_idx = UINT_MAX;
+	index_mail_parse_header_deinit(mail);
+}
+
+static unsigned int
+get_header_field_idx(struct mailbox *box, const char *field)
+{
+	struct mail_cache_field header_field;
+
+	i_zero(&header_field);
+	header_field.type = MAIL_CACHE_FIELD_HEADER;
+	/* Always register with NO decision. The field should be added soon
+	   with mail_cache_add(), which changes the decision to TEMP. Most
+	   importantly doing it this way emits mail_cache_decision event. */
+	header_field.decision = MAIL_CACHE_DECISION_NO;
+	T_BEGIN {
+		header_field.name = t_strconcat("hdr.", field, NULL);
+		mail_cache_register_fields(box->cache, &header_field, 1);
+	} T_END;
+	return header_field.idx;
+}
+
+bool index_mail_want_parse_headers(struct index_mail *mail)
+{
+	if (mail->data.wanted_headers != NULL ||
+	    mail->data.save_bodystructure_header)
+		return TRUE;
+
+	if ((mail->data.cache_fetch_fields & MAIL_FETCH_DATE) != 0 &&
+	    !mail->data.sent_date_parsed)
+		return TRUE;
+	return FALSE;
+}
+
+static void index_mail_parse_header_register_all_wanted(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	const struct mail_cache_field *all_cache_fields;
+	unsigned int i, count;
+
+	all_cache_fields =
+		mail_cache_register_get_list(_mail->box->cache,
+					     pool_datastack_create(), &count);
+	for (i = 0; i < count; i++) {
+		if (strncasecmp(all_cache_fields[i].name, "hdr.", 4) != 0)
+			continue;
+		if (!mail_cache_field_want_add(_mail->transaction->cache_trans,
+					       _mail->seq, i))
+			continue;
+
+		array_idx_set(&mail->header_match, all_cache_fields[i].idx,
+			      &mail->header_match_value);
+	}
+}
+
+void index_mail_parse_header_init(struct index_mail *mail,
+				  struct mailbox_header_lookup_ctx *headers)
+{
+	struct index_mail_data *data = &mail->data;
+	const uint8_t *match;
+	unsigned int i, field_idx, match_count;
+
+	index_mail_filter_stream_destroy(mail);
+	i_assert(!mail->data.header_parser_initialized);
+
+	mail->header_seq = mail->mail.mail.seq;
+	if (mail->header_data == NULL) {
+		mail->header_data = buffer_create_dynamic(default_pool, 4096);
+		i_array_init(&mail->header_lines, 32);
+		i_array_init(&mail->header_match, 32);
+		i_array_init(&mail->header_match_lines, 32);
+		mail->header_match_value = HEADER_MATCH_SKIP_COUNT;
+	} else {
+		buffer_set_used_size(mail->header_data, 0);
+		array_clear(&mail->header_lines);
+		array_clear(&mail->header_match_lines);
+
+		i_assert((mail->header_match_value &
+			  (HEADER_MATCH_SKIP_COUNT-1)) == 0);
+		if (mail->header_match_value + HEADER_MATCH_SKIP_COUNT <= UINT8_MAX)
+			mail->header_match_value += HEADER_MATCH_SKIP_COUNT;
+		else {
+			/* wrapped, we'll have to clear the buffer */
+			array_clear(&mail->header_match);
+			mail->header_match_value = HEADER_MATCH_SKIP_COUNT;
+		}
+	}
+
+	if (headers != NULL) {
+		for (i = 0; i < headers->count; i++) {
+			array_idx_set(&mail->header_match, headers->idx[i],
+				      &mail->header_match_value);
+		}
+	}
+
+	if (data->wanted_headers != NULL && data->wanted_headers != headers) {
+		headers = data->wanted_headers;
+		for (i = 0; i < headers->count; i++) {
+			array_idx_set(&mail->header_match, headers->idx[i],
+				      &mail->header_match_value);
+		}
+	}
+
+	/* register also all the other headers that exist in cache file */
+	T_BEGIN {
+		index_mail_parse_header_register_all_wanted(mail);
+	} T_END;
+
+	/* if we want sent date, it doesn't mean that we also want to cache
+	   Date: header. if we have Date field's index set at this point we
+	   know that we want it. otherwise add it and remember that we don't
+	   want it cached. */
+	field_idx = get_header_field_idx(mail->mail.mail.box, "Date");
+	match = array_get(&mail->header_match, &match_count);
+	if (field_idx < match_count &&
+	    match[field_idx] == mail->header_match_value) {
+		/* cache Date: header */
+	} else if ((data->cache_fetch_fields & MAIL_FETCH_DATE) != 0 ||
+		   data->save_sent_date) {
+		/* parse Date: header, but don't cache it. */
+		data->dont_cache_field_idx = field_idx;
+		array_idx_set(&mail->header_match, field_idx,
+			      &mail->header_match_value);
+	}
+	mail->data.header_parser_initialized = TRUE;
+	mail->data.parse_line_num = 0;
+	i_zero(&mail->data.parse_line);
+}
+
+static void index_mail_parse_finish_imap_envelope(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	const unsigned int cache_field_envelope =
+		mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx;
+	string_t *str;
+
+	str = str_new(mail->mail.data_pool, 256);
+	imap_envelope_write(mail->data.envelope_data, str);
+	mail->data.envelope = str_c(str);
+	mail->data.save_envelope = FALSE;
+
+	if (mail_cache_field_can_add(_mail->transaction->cache_trans,
+				     _mail->seq, cache_field_envelope)) {
+		index_mail_cache_add_idx(mail, cache_field_envelope,
+					 str_data(str), str_len(str));
+	}
+}
+
+void index_mail_parse_header(struct message_part *part,
+			     struct message_header_line *hdr,
+			     struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct index_mail_data *data = &mail->data;
+	unsigned int field_idx, count;
+	uint8_t *match;
+
+	i_assert(data->header_parser_initialized);
+
+        data->parse_line_num++;
+
+	if (data->save_bodystructure_header &&
+	    !data->parsed_bodystructure_header) {
+		i_assert(part != NULL);
+		message_part_data_parse_from_header(mail->mail.data_pool, part, hdr);
+	}
+
+	if (data->save_envelope) {
+		message_part_envelope_parse_from_header(mail->mail.data_pool,
+					   &data->envelope_data, hdr);
+
+		if (hdr == NULL)
+                        index_mail_parse_finish_imap_envelope(mail);
+	}
+
+	if (hdr == NULL) {
+		/* end of headers */
+		if (mail->data.save_sent_date)
+			mail->data.sent_date_parsed = TRUE;
+		T_BEGIN {
+			index_mail_parse_header_finish(mail);
+		} T_END;
+		if (data->save_bodystructure_header) {
+			i_assert(data->parser_ctx != NULL);
+			data->parsed_bodystructure_header = TRUE;
+		}
+		return;
+	}
+
+	if (!hdr->continued) {
+		T_BEGIN {
+			const char *cache_field_name =
+				t_strconcat("hdr.", hdr->name, NULL);
+			data->parse_line.field_idx =
+				mail_cache_register_lookup(_mail->box->cache,
+							   cache_field_name);
+		} T_END;
+	}
+	field_idx = data->parse_line.field_idx;
+	match = array_get_modifiable(&mail->header_match, &count);
+	if (field_idx >= count ||
+	    !HEADER_MATCH_USABLE(mail, match[field_idx])) {
+		/* we don't want this header. */
+		return;
+	}
+
+	if (!hdr->continued) {
+		/* beginning of a line. add the header name. */
+		data->parse_line.start_pos = str_len(mail->header_data);
+		data->parse_line.line_num = data->parse_line_num;
+		str_append(mail->header_data, hdr->name);
+		str_append_data(mail->header_data, hdr->middle, hdr->middle_len);
+
+		/* remember that we saw this header so we don't add it to
+		   cache as nonexistent. */
+		match[field_idx] |= HEADER_MATCH_FLAG_FOUND;
+	}
+	str_append_data(mail->header_data, hdr->value, hdr->value_len);
+	if (!hdr->no_newline)
+		str_append(mail->header_data, "\n");
+	if (!hdr->continues) {
+		data->parse_line.end_pos = str_len(mail->header_data);
+		array_push_back(&mail->header_lines, &data->parse_line);
+	}
+}
+
+static void
+index_mail_parse_part_header_cb(struct message_part *part,
+				struct message_header_line *hdr,
+				struct index_mail *mail)
+{
+	index_mail_parse_header(part, hdr, mail);
+}
+
+static void
+index_mail_parse_header_cb(struct message_header_line *hdr,
+			   struct index_mail *mail)
+{
+	index_mail_parse_header(mail->data.parts, hdr, mail);
+}
+
+struct istream *
+index_mail_cache_parse_init(struct mail *_mail, struct istream *input)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct istream *input2;
+
+	i_assert(mail->data.tee_stream == NULL);
+	i_assert(mail->data.parser_ctx == NULL);
+
+	/* we're doing everything for now, figure out later if we want to
+	   save them. */
+	mail->data.save_sent_date = TRUE;
+	mail->data.save_bodystructure_header = TRUE;
+	mail->data.save_bodystructure_body = TRUE;
+	/* Don't unnecessarily waste time generating a snippet, since it's
+	   not as cheap as the others to generate. */
+	if (index_mail_want_cache(mail, MAIL_CACHE_BODY_SNIPPET))
+		mail->data.save_body_snippet = TRUE;
+
+	mail->data.tee_stream = tee_i_stream_create(input);
+	input = tee_i_stream_create_child(mail->data.tee_stream);
+	input2 = tee_i_stream_create_child(mail->data.tee_stream);
+
+	index_mail_parse_header_init(mail, NULL);
+	mail->data.parser_input = input;
+	mail->data.parser_ctx =
+		message_parser_init(mail->mail.data_pool, input,
+				    &msg_parser_set);
+	i_stream_unref(&input);
+	return input2;
+}
+
+static void index_mail_init_parser(struct index_mail *mail)
+{
+	struct index_mail_data *data = &mail->data;
+	struct message_part *parts;
+	const char *error;
+
+	if (data->parser_ctx != NULL) {
+		data->parser_input = NULL;
+		if (message_parser_deinit_from_parts(&data->parser_ctx, &parts, &error) < 0) {
+			index_mail_set_message_parts_corrupted(&mail->mail.mail, error);
+			index_mail_parts_reset(mail);
+		}
+		if (data->parts == NULL || data->parts != parts) {
+			/* The previous parsing didn't finish, so we're
+			   re-parsing the header. The new parts don't have data
+			   filled anymore. */
+			data->parsed_bodystructure_header = FALSE;
+		}
+	}
+
+	/* make sure parsing starts from the beginning of the stream */
+	i_stream_seek(mail->data.stream, 0);
+	if (data->parts == NULL) {
+		data->parser_input = data->stream;
+		data->parser_ctx = message_parser_init(mail->mail.data_pool,
+						       data->stream,
+						       &msg_parser_set);
+	} else {
+		data->parser_ctx =
+			message_parser_init_from_parts(data->parts,
+						       data->stream,
+						       &msg_parser_set);
+	}
+}
+
+int index_mail_parse_headers_internal(struct index_mail *mail,
+				      struct mailbox_header_lookup_ctx *headers)
+{
+	struct index_mail_data *data = &mail->data;
+
+	i_assert(data->stream != NULL);
+
+	index_mail_parse_header_init(mail, headers);
+
+	if (data->parts == NULL || data->save_bodystructure_header ||
+	    (data->access_part & PARSE_BODY) != 0) {
+		/* initialize bodystructure parsing in case we read the whole
+		   message. */
+		index_mail_init_parser(mail);
+		message_parser_parse_header(data->parser_ctx, &data->hdr_size,
+					    index_mail_parse_part_header_cb,
+					    mail);
+	} else {
+		/* just read the header */
+		i_assert(!data->save_bodystructure_body ||
+			 data->parser_ctx != NULL);
+		message_parse_header(data->stream, &data->hdr_size,
+				     msg_parser_set.hdr_flags,
+				     index_mail_parse_header_cb, mail);
+	}
+	if (index_mail_stream_check_failure(mail) < 0) {
+		index_mail_parse_header_deinit(mail);
+		return -1;
+	}
+	i_assert(!mail->data.header_parser_initialized);
+	data->hdr_size_set = TRUE;
+	data->access_part &= ENUM_NEGATE(PARSE_HDR);
+	return 0;
+}
+
+int index_mail_parse_headers(struct index_mail *mail,
+			     struct mailbox_header_lookup_ctx *headers,
+			     const char *reason)
+{
+	struct index_mail_data *data = &mail->data;
+	struct istream *input;
+	uoff_t old_offset;
+
+	old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
+
+	if (mail_get_hdr_stream_because(&mail->mail.mail, NULL, reason, &input) < 0)
+		return -1;
+
+	int ret = index_mail_parse_headers_internal(mail, headers);
+	i_stream_seek(data->stream, old_offset);
+	return ret;
+}
+
+static void
+imap_envelope_parse_callback(struct message_header_line *hdr,
+			     struct index_mail *mail)
+{
+	message_part_envelope_parse_from_header(mail->mail.data_pool,
+				   &mail->data.envelope_data, hdr);
+
+	if (hdr == NULL)
+		index_mail_parse_finish_imap_envelope(mail);
+}
+
+int index_mail_headers_get_envelope(struct index_mail *mail)
+{
+	const unsigned int cache_field_envelope =
+		mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx;
+	struct mailbox_header_lookup_ctx *header_ctx;
+	struct istream *stream;
+	uoff_t old_offset;
+	string_t *str;
+
+	str = str_new(mail->mail.data_pool, 256);
+	if (index_mail_cache_lookup_field(mail, str,
+					  cache_field_envelope) > 0) {
+		mail->data.envelope = str_c(str);
+		return 0;
+	}
+	str_free(&str);
+
+	old_offset = mail->data.stream == NULL ? 0 :
+		mail->data.stream->v_offset;
+
+	/* Make sure header_cache_callback() isn't also parsing the ENVELOPE.
+	   Otherwise two callbacks are doing it and mixing up results. */
+	mail->data.save_envelope = FALSE;
+
+	header_ctx = mailbox_header_lookup_init(mail->mail.mail.box,
+						message_part_envelope_headers);
+	if (mail_get_header_stream(&mail->mail.mail, header_ctx, &stream) < 0) {
+		mailbox_header_lookup_unref(&header_ctx);
+		return -1;
+	}
+	mailbox_header_lookup_unref(&header_ctx);
+
+	if (mail->data.envelope == NULL) {
+		/* we got the headers from cache - parse them to get the
+		   envelope */
+		message_parse_header(stream, NULL, msg_parser_set.hdr_flags,
+				     imap_envelope_parse_callback, mail);
+		if (stream->stream_errno != 0) {
+			index_mail_stream_log_failure_for(mail, stream);
+			return -1;
+		}
+		i_assert(mail->data.envelope != NULL);
+	}
+
+	if (mail->data.stream != NULL)
+		i_stream_seek(mail->data.stream, old_offset);
+	return 0;
+}
+
+static size_t get_header_size(buffer_t *buffer, size_t pos)
+{
+	const unsigned char *data = buffer->data;
+	size_t i, size = buffer->used;
+
+	i_assert(pos <= size);
+
+	for (i = pos; i < size; i++) {
+		if (data[i] == '\n') {
+			if (i+1 == size ||
+			    (data[i+1] != ' ' && data[i+1] != '\t'))
+				return i - pos;
+		}
+	}
+	return size - pos;
+}
+
+static int index_mail_header_is_parsed(struct index_mail *mail,
+				       unsigned int field_idx)
+{
+	const uint8_t *match;
+	unsigned int count;
+
+	match = array_get(&mail->header_match, &count);
+	if (field_idx < count && HEADER_MATCH_USABLE(mail, match[field_idx]))
+		return (match[field_idx] & HEADER_MATCH_FLAG_FOUND) != 0 ? 1 : 0;
+	return -1;
+}
+
+static bool skip_header(const unsigned char **data, size_t len)
+{
+	const unsigned char *p = *data;
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		if (p[i] == ':')
+			break;
+	}
+	if (i == len)
+		return FALSE;
+
+	for (i++; i < len; i++) {
+		if (!IS_LWSP(p[i]))
+			break;
+	}
+
+	*data = p + i;
+	return TRUE;
+}
+
+static const char *const *
+index_mail_get_parsed_header(struct index_mail *mail, unsigned int field_idx)
+{
+	ARRAY(const char *) header_values;
+        const struct index_mail_line *lines;
+	const unsigned char *header, *value_start, *value_end;
+	const unsigned int *line_idx;
+	const char *value;
+	unsigned int i, lines_count, first_line_idx;
+
+	line_idx = array_idx(&mail->header_match_lines, field_idx);
+	i_assert(*line_idx != 0);
+	first_line_idx = *line_idx - 1;
+
+	p_array_init(&header_values, mail->mail.data_pool, 4);
+	header = mail->header_data->data;
+
+	lines = array_get(&mail->header_lines, &lines_count);
+	for (i = first_line_idx; i < lines_count; i++) {
+		if (lines[i].field_idx != lines[first_line_idx].field_idx)
+			break;
+
+		/* skip header: and drop ending LF */
+		value_start = header + lines[i].start_pos;
+		value_end = header + lines[i].end_pos;
+		if (skip_header(&value_start, value_end - value_start)) {
+			if (value_start != value_end && value_end[-1] == '\n')
+				value_end--;
+			value = message_header_strdup(mail->mail.data_pool,
+						      value_start,
+						      value_end - value_start);
+			array_push_back(&header_values, &value);
+		}
+	}
+
+	array_append_zero(&header_values);
+	return array_front(&header_values);
+}
+
+static int
+index_mail_get_raw_headers(struct index_mail *mail, const char *field,
+			   const char *const **value_r)
+{
+	struct mail *_mail = &mail->mail.mail;
+	const char *headers[2], *value;
+	struct mailbox_header_lookup_ctx *headers_ctx;
+	const unsigned char *data;
+	unsigned int field_idx;
+	string_t *dest;
+	size_t i, len, len2;
+	int ret;
+	ARRAY(const char *) header_values;
+
+	i_assert(field != NULL);
+
+	field_idx = get_header_field_idx(_mail->box, field);
+
+	dest = t_str_new(128);
+	if (mail_cache_lookup_headers(_mail->transaction->cache_view, dest,
+				      _mail->seq, &field_idx, 1) <= 0) {
+		/* not in cache / error - first see if it's already parsed */
+		p_free(mail->mail.data_pool, dest);
+		if (mail->data.header_parser_initialized) {
+			/* don't try to parse headers recursively. we're here
+			   because message size was wrong and istream-mail
+			   wants to log some cached headers. */
+			i_assert(mail->mail.mail.lookup_abort >= MAIL_LOOKUP_ABORT_NOT_IN_CACHE);
+			mail_set_aborted(&mail->mail.mail);
+			return -1;
+		}
+		if (mail->header_seq != mail->mail.mail.seq ||
+		    index_mail_header_is_parsed(mail, field_idx) < 0) {
+			/* parse */
+			const char *reason = index_mail_cache_reason(_mail,
+				t_strdup_printf("header %s", field));
+			headers[0] = field; headers[1] = NULL;
+			headers_ctx = mailbox_header_lookup_init(_mail->box,
+								 headers);
+			ret = index_mail_parse_headers(mail, headers_ctx, reason);
+			mailbox_header_lookup_unref(&headers_ctx);
+			if (ret < 0)
+				return -1;
+		}
+
+		if ((ret = index_mail_header_is_parsed(mail, field_idx)) <= 0) {
+			/* not found */
+			i_assert(ret != -1);
+			*value_r = p_new(mail->mail.data_pool, const char *, 1);
+			return 0;
+		}
+		*value_r = index_mail_get_parsed_header(mail, field_idx);
+		return 0;
+	}
+	_mail->transaction->stats.cache_hit_count++;
+	data = buffer_get_data(dest, &len);
+
+	if (len == 0) {
+		/* cached as nonexistent. */
+		*value_r = p_new(mail->mail.data_pool, const char *, 1);
+		return 0;
+	}
+
+	p_array_init(&header_values, mail->mail.data_pool, 4);
+
+	/* cached. skip "header name: " parts in dest. */
+	for (i = 0; i < len; i++) {
+		if (data[i] == ':') {
+			i++;
+			while (i < len && IS_LWSP(data[i])) i++;
+
+			/* @UNSAFE */
+			len2 = get_header_size(dest, i);
+			value = message_header_strdup(mail->mail.data_pool,
+						     data + i, len2);
+			i += len2 + 1;
+
+			array_push_back(&header_values, &value);
+		}
+	}
+
+	array_append_zero(&header_values);
+	*value_r = array_front(&header_values);
+	return 0;
+}
+
+static int unfold_header(pool_t pool, const char **_str)
+{
+	const char *str = *_str;
+	char *new_str;
+	unsigned int i, j;
+
+	for (i = 0; str[i] != '\0'; i++) {
+		if (str[i] == '\n')
+			break;
+	}
+	if (str[i] == '\0')
+		return 0;
+
+	/* @UNSAFE */
+	new_str = p_malloc(pool, i + strlen(str+i) + 1);
+	memcpy(new_str, str, i);
+	for (j = i; str[i] != '\0'; i++) {
+		if (str[i] == '\n') {
+			new_str[j++] = ' ';
+			i++;
+			if (str[i] == '\0')
+				break;
+
+			if (str[i] != ' ' && str[i] != '\t') {
+				/* corrupted */
+				return -1;
+			}
+		} else {
+			new_str[j++] = str[i];
+		}
+	}
+	new_str[j] = '\0';
+	*_str = new_str;
+	return 0;
+}
+
+static void str_replace_nuls(string_t *str)
+{
+	char *data = str_c_modifiable(str);
+	size_t i, len = str_len(str);
+
+	for (i = 0; i < len; i++) {
+		if (data[i] == '\0')
+			data[i] = ' ';
+	}
+}
+
+static int
+index_mail_headers_decode(struct index_mail *mail, const char *const **_list,
+			  unsigned int max_count)
+{
+	const char *const *list = *_list;
+	const char **decoded_list, *input;
+	unsigned int i, count;
+	string_t *str;
+
+	count = str_array_length(list);
+	if (count > max_count)
+		count = max_count;
+	decoded_list = p_new(mail->mail.data_pool, const char *, count + 1);
+
+	str = t_str_new(512);
+	for (i = 0; i < count; i++) {
+		str_truncate(str, 0);
+		input = list[i];
+		/* unfold all lines into a single line */
+		if (unfold_header(mail->mail.data_pool, &input) < 0)
+			return -1;
+
+		/* decode MIME encoded-words. decoding may also add new LFs. */
+		message_header_decode_utf8((const unsigned char *)input,
+					   strlen(input), str, NULL);
+		if (strcmp(str_c(str), input) != 0) {
+			if (strlen(str_c(str)) != str_len(str)) {
+				/* replace NULs with spaces */
+				str_replace_nuls(str);
+			}
+			input = p_strdup(mail->mail.data_pool, str_c(str));
+		}
+		decoded_list[i] = input;
+	}
+	*_list = decoded_list;
+	return 0;
+}
+
+int index_mail_get_headers(struct mail *_mail, const char *field,
+			   bool decode_to_utf8, const char *const **value_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	bool retry = TRUE;
+	int ret;
+
+	for (;; retry = FALSE) {
+		if (index_mail_get_raw_headers(mail, field, value_r) < 0)
+			return -1;
+		if (**value_r == NULL)
+			return 0;
+		if (!decode_to_utf8)
+			return 1;
+
+		T_BEGIN {
+			ret = index_mail_headers_decode(mail, value_r, UINT_MAX);
+		} T_END;
+
+		if (ret < 0 && retry) {
+			mail_set_mail_cache_corrupted(_mail, "Broken header %s",
+						      field);
+		} else {
+			break;
+		}
+	}
+	if (ret < 0) {
+		i_panic("BUG: Broken header %s for mail UID %u "
+			"wasn't fixed by re-parsing the header",
+			field, _mail->uid);
+	}
+	return 1;
+}
+
+int index_mail_get_first_header(struct mail *_mail, const char *field,
+				bool decode_to_utf8, const char **value_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	const char *const *list;
+	bool retry = TRUE;
+	int ret;
+
+	for (;; retry = FALSE) {
+		if (index_mail_get_raw_headers(mail, field, &list) < 0)
+			return -1;
+		if (!decode_to_utf8 || list[0] == NULL) {
+			ret = 0;
+			break;
+		}
+
+		T_BEGIN {
+			ret = index_mail_headers_decode(mail, &list, 1);
+		} T_END;
+
+		if (ret < 0 && retry) {
+			mail_set_mail_cache_corrupted(_mail, "Broken header %s",
+						      field);
+			/* retry by parsing the full header */
+		} else {
+			break;
+		}
+	}
+	if (ret < 0) {
+		i_panic("BUG: Broken header %s for mail UID %u "
+			"wasn't fixed by re-parsing the header",
+			field, _mail->uid);
+	}
+	*value_r = list[0];
+	return list[0] != NULL ? 1 : 0;
+}
+
+static void
+header_cache_callback(struct header_filter_istream *input ATTR_UNUSED,
+		      struct message_header_line *hdr,
+		      bool *matched ATTR_UNUSED, struct index_mail *mail)
+{
+	index_mail_parse_header(NULL, hdr, mail);
+}
+
+static void index_mail_filter_stream_destroy(struct index_mail *mail)
+{
+	if (mail->data.filter_stream == NULL)
+		return;
+
+	const unsigned char *data;
+	size_t size;
+
+	/* read through the previous filter_stream. this makes sure that the
+	   fields are added to cache, and most importantly it resets
+	   header_parser_initialized=FALSE so we don't assert on it. */
+	while (i_stream_read_more(mail->data.filter_stream, &data, &size) > 0)
+		i_stream_skip(mail->data.filter_stream, size);
+	if (mail->data.header_parser_initialized) {
+		/* istream failed while reading the header */
+		i_assert(mail->data.filter_stream->stream_errno != 0);
+		index_mail_parse_header_deinit(mail);
+	}
+	i_stream_destroy(&mail->data.filter_stream);
+}
+
+int index_mail_get_header_stream(struct mail *_mail,
+				 struct mailbox_header_lookup_ctx *headers,
+				 struct istream **stream_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct istream *input;
+	string_t *dest;
+
+	index_mail_filter_stream_destroy(mail);
+
+	if (mail->data.save_bodystructure_header) {
+		/* we have to parse the header. */
+		const char *reason =
+			index_mail_cache_reason(_mail, "bodystructure");
+		mail->data.access_reason_code = "mail:header_fields";
+		if (index_mail_parse_headers(mail, headers, reason) < 0)
+			return -1;
+	}
+
+	dest = str_new(mail->mail.data_pool, 256);
+	if (mail_cache_lookup_headers(_mail->transaction->cache_view, dest,
+				      _mail->seq, headers->idx,
+				      headers->count) > 0) {
+		str_append(dest, "\n");
+		_mail->transaction->stats.cache_hit_count++;
+		mail->data.filter_stream =
+			i_stream_create_from_data(str_data(dest),
+						  str_len(dest));
+		*stream_r = mail->data.filter_stream;
+		return 0;
+	}
+	/* not in cache / error */
+	p_free(mail->mail.data_pool, dest);
+
+	unsigned int first_not_found = UINT_MAX, not_found_count = 0;
+	for (unsigned int i = 0; i < headers->count; i++) {
+		if (mail_cache_field_exists(_mail->transaction->cache_view,
+					    _mail->seq, headers->idx[i]) <= 0) {
+			if (not_found_count++ == 0)
+				first_not_found = i;
+		}
+	}
+
+	const char *reason;
+	if (not_found_count == 0)
+		reason = "BUG: all headers seem to exist in cache";
+	else {
+		i_assert(first_not_found != UINT_MAX);
+		reason = index_mail_cache_reason(_mail, t_strdup_printf(
+			"%u/%u headers not cached (first=%s)",
+			not_found_count, headers->count, headers->name[first_not_found]));
+	}
+	mail->data.access_reason_code = "mail:header_fields";
+	if (mail_get_hdr_stream_because(_mail, NULL, reason, &input) < 0)
+		return -1;
+
+	index_mail_parse_header_init(mail, headers);
+	mail->data.filter_stream =
+		i_stream_create_header_filter(mail->data.stream,
+					      HEADER_FILTER_INCLUDE |
+					      HEADER_FILTER_ADD_MISSING_EOH |
+					      HEADER_FILTER_HIDE_BODY,
+					      headers->name, headers->count,
+					      header_cache_callback, mail);
+	*stream_r = mail->data.filter_stream;
+	return 0;
+}
diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
new file mode 100644
index 0000000..48122b1
--- /dev/null
+++ b/src/lib-storage/index/index-mail.c
@@ -0,0 +1,2619 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "hex-binary.h"
+#include "str.h"
+#include "mailbox-recent-flags.h"
+#include "message-date.h"
+#include "message-part-data.h"
+#include "message-part-serialize.h"
+#include "message-parser.h"
+#include "message-snippet.h"
+#include "imap-bodystructure.h"
+#include "imap-envelope.h"
+#include "mail-cache.h"
+#include "mail-index-modseq.h"
+#include "index-storage.h"
+#include "istream-mail.h"
+#include "index-mail.h"
+
+#include <fcntl.h>
+
+#define BODY_SNIPPET_ALGO_V1 "1"
+#define BODY_SNIPPET_MAX_CHARS 200
+
+struct mail_cache_field global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT] = {
+	{ .name = "flags",
+	  .type = MAIL_CACHE_FIELD_BITMASK,
+	  .field_size = sizeof(uint32_t) },
+	{ .name = "date.sent",
+	  .type = MAIL_CACHE_FIELD_FIXED_SIZE,
+	  .field_size = sizeof(struct mail_sent_date) },
+	{ .name = "date.received",
+	  .type = MAIL_CACHE_FIELD_FIXED_SIZE,
+	  .field_size = sizeof(uint32_t) },
+	{ .name = "date.save",
+	  .type = MAIL_CACHE_FIELD_FIXED_SIZE,
+	  .field_size = sizeof(uint32_t) },
+	{ .name = "size.virtual",
+	  .type = MAIL_CACHE_FIELD_FIXED_SIZE,
+	  .field_size = sizeof(uoff_t) },
+	{ .name = "size.physical",
+	  .type = MAIL_CACHE_FIELD_FIXED_SIZE,
+	  .field_size = sizeof(uoff_t) },
+	{ .name = "imap.body",
+	  .type = MAIL_CACHE_FIELD_STRING },
+	{ .name = "imap.bodystructure",
+	  .type = MAIL_CACHE_FIELD_STRING },
+	{ .name = "imap.envelope",
+	  .type = MAIL_CACHE_FIELD_STRING },
+	{ .name = "pop3.uidl",
+	  .type = MAIL_CACHE_FIELD_STRING },
+	{ .name = "pop3.order",
+	  .type = MAIL_CACHE_FIELD_FIXED_SIZE,
+	  .field_size = sizeof(uint32_t) },
+	{ .name = "guid",
+	  .type = MAIL_CACHE_FIELD_STRING },
+	{ .name = "mime.parts",
+	  .type = MAIL_CACHE_FIELD_VARIABLE_SIZE },
+	{ .name = "binary.parts",
+	  .type = MAIL_CACHE_FIELD_VARIABLE_SIZE },
+	{ .name = "body.snippet",
+	  .type = MAIL_CACHE_FIELD_VARIABLE_SIZE }
+	/* FIXME: for now need to update get_metadata_precache_fields() in
+	   index-status.c when adding more fields. those fields should probably
+	   just be moved here to the same struct. */
+};
+
+static void index_mail_init_data(struct index_mail *mail);
+static int index_mail_parse_body(struct index_mail *mail,
+				 enum index_cache_field field);
+static int index_mail_write_body_snippet(struct index_mail *mail);
+
+int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf,
+				  unsigned int field_idx)
+{
+	struct mail *_mail = &mail->mail.mail;
+	int ret;
+
+	ret = mail_cache_lookup_field(mail->mail.mail.transaction->cache_view,
+				      buf, mail->mail.mail.seq, field_idx);
+	if (ret > 0)
+		mail->mail.mail.transaction->stats.cache_hit_count++;
+
+	/* If the request was lazy mark the field as cache wanted. */
+	if (_mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE_START_CACHING &&
+	    mail_cache_field_get_decision(_mail->box->cache, field_idx) ==
+	    MAIL_CACHE_DECISION_NO) {
+		mail_cache_decision_add(_mail->transaction->cache_view,
+					_mail->seq, field_idx);
+	}
+
+	return ret;
+}
+
+static void index_mail_try_set_attachment_keywords(struct index_mail *mail)
+{
+	enum mail_lookup_abort orig_lookup_abort = mail->mail.mail.lookup_abort;
+	mail->mail.mail.lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+	(void)mail_set_attachment_keywords(&mail->mail.mail);
+	mail->mail.mail.lookup_abort = orig_lookup_abort;
+}
+
+static bool
+index_mail_want_attachment_keywords_on_fetch(struct index_mail *mail)
+{
+	const struct mail_storage_settings *mail_set =
+		mailbox_get_settings(mail->mail.mail.box);
+
+	return mail_set->parsed_mail_attachment_detection_add_flags &&
+		!mail_set->parsed_mail_attachment_detection_no_flags_on_fetch &&
+		!mail_has_attachment_keywords(&mail->mail.mail);
+}
+
+static int get_serialized_parts(struct index_mail *mail, buffer_t **part_buf_r)
+{
+	const unsigned int field_idx =
+		mail->ibox->cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx;
+
+	*part_buf_r = t_buffer_create(128);
+	return index_mail_cache_lookup_field(mail, *part_buf_r, field_idx);
+}
+
+static struct message_part *get_unserialized_parts(struct index_mail *mail)
+{
+	struct message_part *parts;
+	buffer_t *part_buf;
+	const char *error;
+
+	if (get_serialized_parts(mail, &part_buf) <= 0)
+		return NULL;
+
+	parts = message_part_deserialize(mail->mail.data_pool, part_buf->data,
+					 part_buf->used, &error);
+	if (parts == NULL) {
+		mail_set_mail_cache_corrupted(&mail->mail.mail,
+			"Corrupted cached mime.parts data: %s (parts=%s)",
+			error, binary_to_hex(part_buf->data, part_buf->used));
+	}
+	return parts;
+}
+
+static bool message_parts_have_nuls(const struct message_part *part)
+{
+	for (; part != NULL; part = part->next) {
+		if ((part->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0)
+			return TRUE;
+		if (part->children != NULL) {
+			if (message_parts_have_nuls(part->children))
+				return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static bool get_cached_parts(struct index_mail *mail)
+{
+	struct message_part *part;
+
+	if (mail->data.parts != NULL)
+		return TRUE;
+	if (mail->data.parser_ctx != NULL) {
+		/* Message is already being parsed. Get the message parts by
+		   finishing its parsing, so there won't be any confusion about
+		   whether e.g. data->parsed_bodystructure=TRUE match data->parts  */
+		return FALSE;
+	}
+
+	T_BEGIN {
+		part = get_unserialized_parts(mail);
+	} T_END;
+	if (part == NULL)
+		return FALSE;
+
+	/* we know the NULs now, update them */
+	if (message_parts_have_nuls(part)) {
+		mail->mail.mail.has_nuls = TRUE;
+		mail->mail.mail.has_no_nuls = FALSE;
+	} else {
+		mail->mail.mail.has_nuls = FALSE;
+		mail->mail.mail.has_no_nuls = TRUE;
+	}
+
+	mail->data.parts = part;
+	if (index_mail_want_attachment_keywords_on_fetch(mail))
+		index_mail_try_set_attachment_keywords(mail);
+	return TRUE;
+}
+
+void index_mail_set_message_parts_corrupted(struct mail *mail, const char *error)
+{
+	buffer_t *part_buf;
+	const char *parts_str;
+
+	if (get_serialized_parts(INDEX_MAIL(mail), &part_buf) <= 0)
+		parts_str = "";
+	else
+		parts_str = binary_to_hex(part_buf->data, part_buf->used);
+
+	mail_set_cache_corrupted(mail,
+		MAIL_FETCH_MESSAGE_PARTS, t_strdup_printf(
+		"Cached MIME parts don't match message during parsing: %s (parts=%s)",
+		error, parts_str));
+}
+
+static bool index_mail_get_fixed_field(struct index_mail *mail,
+				       enum index_cache_field field,
+				       void *data, size_t data_size)
+{
+	const unsigned int field_idx = mail->ibox->cache_fields[field].idx;
+	buffer_t buf;
+	bool ret;
+
+	buffer_create_from_data(&buf, data, data_size);
+	if (index_mail_cache_lookup_field(mail, &buf, field_idx) <= 0)
+		ret = FALSE;
+	else {
+		i_assert(buf.used == data_size);
+		ret = TRUE;
+	}
+	return ret;
+}
+
+bool index_mail_get_cached_uoff_t(struct index_mail *mail,
+				  enum index_cache_field field, uoff_t *size_r)
+{
+	return index_mail_get_fixed_field(mail, field,
+					  size_r, sizeof(*size_r));
+}
+
+static bool index_mail_get_pvt(struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+
+	if (mail->seq_pvt != 0)
+		return TRUE;
+	if (_mail->box->view_pvt == NULL) {
+		/* no private view (set by view syncing) -> no private flags */
+		return FALSE;
+	}
+	if (_mail->saving) {
+		/* mail is still being saved, it has no private flags yet */
+		return FALSE;
+	}
+	i_assert(_mail->uid != 0);
+
+	index_transaction_init_pvt(_mail->transaction);
+	if (!mail_index_lookup_seq(_mail->transaction->view_pvt, _mail->uid,
+				   &mail->seq_pvt))
+		mail->seq_pvt = 0;
+	return mail->seq_pvt != 0;
+}
+
+enum mail_flags index_mail_get_flags(struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	const struct mail_index_record *rec;
+	enum mail_flags flags, pvt_flags_mask;
+
+	rec = mail_index_lookup(_mail->transaction->view, _mail->seq);
+	flags = rec->flags & (MAIL_FLAGS_NONRECENT |
+			      MAIL_INDEX_MAIL_FLAG_BACKEND);
+
+	if (mailbox_recent_flags_have_uid(_mail->box, _mail->uid))
+		flags |= MAIL_RECENT;
+
+	if (index_mail_get_pvt(_mail)) {
+		/* mailbox has private flags */
+		pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box);
+		flags &= ENUM_NEGATE(pvt_flags_mask);
+		rec = mail_index_lookup(_mail->transaction->view_pvt,
+					mail->seq_pvt);
+		flags |= rec->flags & pvt_flags_mask;
+	}
+	return flags;
+}
+
+uint64_t index_mail_get_modseq(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	if (mail->data.modseq != 0)
+		return mail->data.modseq;
+
+	mail_index_modseq_enable(_mail->box->index);
+	mail->data.modseq =
+		mail_index_modseq_lookup(_mail->transaction->view, _mail->seq);
+	return mail->data.modseq;
+}
+
+uint64_t index_mail_get_pvt_modseq(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	if (mail->data.pvt_modseq != 0)
+		return mail->data.pvt_modseq;
+
+	if (mailbox_open_index_pvt(_mail->box) <= 0)
+		return 0;
+	index_transaction_init_pvt(_mail->transaction);
+
+	mail_index_modseq_enable(_mail->box->index_pvt);
+	mail->data.pvt_modseq =
+		mail_index_modseq_lookup(_mail->transaction->view_pvt,
+					 _mail->seq);
+	return mail->data.pvt_modseq;
+}
+
+const char *const *index_mail_get_keywords(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	const char *const *names;
+	const unsigned int *keyword_indexes;
+	unsigned int i, count, names_count;
+
+	if (array_is_created(&data->keywords))
+		return array_front(&data->keywords);
+
+	(void)index_mail_get_keyword_indexes(_mail);
+
+	keyword_indexes = array_get(&data->keyword_indexes, &count);
+	names = array_get(mail->ibox->keyword_names, &names_count);
+	p_array_init(&data->keywords, mail->mail.data_pool, count + 1);
+	for (i = 0; i < count; i++) {
+		const char *name;
+		i_assert(keyword_indexes[i] < names_count);
+
+		name = names[keyword_indexes[i]];
+		array_push_back(&data->keywords, &name);
+	}
+
+	/* end with NULL */
+	array_append_zero(&data->keywords);
+	return array_front(&data->keywords);
+}
+
+const ARRAY_TYPE(keyword_indexes) *
+index_mail_get_keyword_indexes(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	if (!array_is_created(&data->keyword_indexes)) {
+		p_array_init(&data->keyword_indexes, mail->mail.data_pool, 32);
+		mail_index_lookup_keywords(_mail->transaction->view,
+					   mail->mail.mail.seq,
+					   &data->keyword_indexes);
+	}
+	return &data->keyword_indexes;
+}
+
+int index_mail_get_parts(struct mail *_mail, struct message_part **parts_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	data->cache_fetch_fields |= MAIL_FETCH_MESSAGE_PARTS;
+	if (data->parts != NULL || get_cached_parts(mail)) {
+		*parts_r = data->parts;
+		return 0;
+	}
+
+	if (data->parser_ctx == NULL) {
+		const char *reason =
+			index_mail_cache_reason(_mail, "mime parts");
+		if (index_mail_parse_headers(mail, NULL, reason) < 0)
+			return -1;
+		/* parts may be set now as a result of some plugin */
+	}
+
+	if (data->parts == NULL) {
+		data->save_message_parts = TRUE;
+		if (index_mail_parse_body(mail, 0) < 0)
+			return -1;
+	}
+
+	*parts_r = data->parts;
+	return 0;
+}
+
+int index_mail_get_received_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	data->cache_fetch_fields |= MAIL_FETCH_RECEIVED_DATE;
+	if (data->received_date == (time_t)-1) {
+		uint32_t t;
+
+		if (index_mail_get_fixed_field(mail, MAIL_CACHE_RECEIVED_DATE,
+						&t, sizeof(t)))
+			data->received_date = t;
+	}
+
+	*date_r = data->received_date;
+	return *date_r == (time_t)-1 ? -1 : 0;
+}
+
+int index_mail_get_save_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	data->cache_fetch_fields |= MAIL_FETCH_SAVE_DATE;
+	if (data->save_date == (time_t)-1) {
+		uint32_t t;
+
+		if (index_mail_get_fixed_field(mail, MAIL_CACHE_SAVE_DATE,
+					       &t, sizeof(t)))
+			data->save_date = t;
+	}
+
+	*date_r = data->save_date;
+	return *date_r == (time_t)-1 ? -1 : 1;
+}
+
+static int index_mail_cache_sent_date(struct index_mail *mail)
+{
+	struct index_mail_data *data = &mail->data;
+	const char *str;
+	time_t t;
+	int ret, tz;
+
+	if (data->sent_date.time != (uint32_t)-1)
+		return 0;
+
+	if ((ret = mail_get_first_header(&mail->mail.mail, "Date", &str)) < 0)
+		return ret;
+
+	if (ret == 0 ||
+	    !message_date_parse((const unsigned char *)str,
+				strlen(str), &t, &tz)) {
+		/* 0 = not found / invalid */
+		t = 0;
+		tz = 0;
+	}
+	data->sent_date.time = t;
+	data->sent_date.timezone = tz;
+	index_mail_cache_add(mail, MAIL_CACHE_SENT_DATE,
+			     &data->sent_date, sizeof(data->sent_date));
+	return 0;
+}
+
+int index_mail_get_date(struct mail *_mail, time_t *date_r, int *timezone_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct mail_sent_date sentdate;
+
+	data->cache_fetch_fields |= MAIL_FETCH_DATE;
+	if (data->sent_date.time != (uint32_t)-1) {
+		*timezone_r = data->sent_date.timezone;
+		*date_r = data->sent_date.time;
+		return 0;
+	}
+
+	if (index_mail_get_fixed_field(mail, MAIL_CACHE_SENT_DATE,
+				       &sentdate, sizeof(sentdate)))
+		data->sent_date = sentdate;
+
+	if (index_mail_cache_sent_date(mail) < 0)
+		return -1;
+
+	*timezone_r = data->sent_date.timezone;
+	*date_r = data->sent_date.time;
+	return 0;
+}
+
+static bool get_cached_msgpart_sizes(struct index_mail *mail)
+{
+	struct index_mail_data *data = &mail->data;
+
+	if (data->parts == NULL)
+		(void)get_cached_parts(mail);
+
+	if (data->parts != NULL) {
+		data->hdr_size_set = TRUE;
+		data->hdr_size = data->parts->header_size;
+		data->body_size = data->parts->body_size;
+		data->body_size_set = TRUE;
+		data->virtual_size = data->parts->header_size.virtual_size +
+			data->body_size.virtual_size;
+		data->physical_size = data->parts->header_size.physical_size +
+			data->body_size.physical_size;
+	}
+
+	return data->parts != NULL;
+}
+
+const uint32_t *index_mail_get_vsize_extension(struct mail *_mail)
+{
+	const void *idata;
+	bool expunged ATTR_UNUSED;
+
+	mail_index_lookup_ext(_mail->transaction->view, _mail->seq,
+			      _mail->box->mail_vsize_ext_id, &idata, &expunged);
+	const uint32_t *vsize = idata;
+	return vsize;
+}
+
+static void index_mail_try_set_body_size(struct index_mail *mail)
+{
+	struct index_mail_data *data = &mail->data;
+
+	if (data->hdr_size_set && !data->inexact_total_sizes &&
+	    data->physical_size != UOFF_T_MAX &&
+	    data->virtual_size != UOFF_T_MAX) {
+		/* We know the total size of this mail and we know the
+		   header size, so we can calculate also the body size.
+		   However, don't do this if there's a possibility that
+		   physical_size or virtual_size don't actually match the
+		   mail stream's size (e.g. buggy imapc servers). */
+		if (data->physical_size < data->hdr_size.physical_size) {
+			mail_set_cache_corrupted(&mail->mail.mail,
+				MAIL_FETCH_PHYSICAL_SIZE, t_strdup_printf(
+				"Cached physical size smaller than header size "
+				"(%"PRIuUOFF_T" < %"PRIuUOFF_T")",
+				data->physical_size, data->hdr_size.physical_size));
+		} else if (data->virtual_size < data->hdr_size.virtual_size) {
+			mail_set_cache_corrupted(&mail->mail.mail,
+				MAIL_FETCH_VIRTUAL_SIZE, t_strdup_printf(
+				"Cached virtual size smaller than header size "
+				"(%"PRIuUOFF_T" < %"PRIuUOFF_T")",
+				data->virtual_size, data->hdr_size.virtual_size));
+		} else {
+			data->body_size.physical_size = data->physical_size -
+				data->hdr_size.physical_size;
+			data->body_size.virtual_size = data->virtual_size -
+				data->hdr_size.virtual_size;
+			data->body_size_set = TRUE;
+		}
+	}
+}
+
+bool index_mail_get_cached_virtual_size(struct index_mail *mail, uoff_t *size_r)
+{
+	struct index_mail_data *data = &mail->data;
+	struct mail *_mail = &mail->mail.mail;
+	uoff_t size;
+	unsigned int idx ATTR_UNUSED;
+
+	/* see if we can get it from index */
+	const uint32_t *vsize = index_mail_get_vsize_extension(_mail);
+
+	data->cache_fetch_fields |= MAIL_FETCH_VIRTUAL_SIZE;
+	if (data->virtual_size == UOFF_T_MAX && vsize != NULL && *vsize > 0)
+		data->virtual_size = (*vsize)-1;
+	if (data->virtual_size == UOFF_T_MAX) {
+		if (index_mail_get_cached_uoff_t(mail,
+						 MAIL_CACHE_VIRTUAL_FULL_SIZE,
+						 &size))
+			data->virtual_size = size;
+		else {
+			if (!get_cached_msgpart_sizes(mail))
+				return FALSE;
+		}
+	}
+	index_mail_try_set_body_size(mail);
+	*size_r = data->virtual_size;
+
+	/* if vsize is present and wanted for index, but missing from index
+	   add it to index. */
+	if (vsize != NULL && *vsize == 0 &&
+	    data->virtual_size < (uint32_t)-1) {
+		uint32_t vsize = data->virtual_size+1;
+		mail_index_update_ext(_mail->transaction->itrans, _mail->seq,
+				      _mail->box->mail_vsize_ext_id, &vsize, NULL);
+	}
+
+	return TRUE;
+}
+
+static void index_mail_get_cached_body_size(struct index_mail *mail)
+{
+	struct index_mail_data *data = &mail->data;
+	uoff_t tmp;
+
+	if (!data->hdr_size_set)
+		return;
+
+	/* we've already called get_cached_msgpart_sizes() and it didn't work.
+	   try to do this by using cached virtual size and a quick physical
+	   size lookup. */
+	if (!index_mail_get_cached_virtual_size(mail, &tmp))
+		return;
+
+	if (!data->body_size_set) {
+		enum mail_lookup_abort old_abort = mail->mail.mail.lookup_abort;
+
+		/* get the physical size, but not if it requires reading
+		   through the whole message */
+		if (mail->mail.mail.lookup_abort < MAIL_LOOKUP_ABORT_READ_MAIL)
+			mail->mail.mail.lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
+		if (mail_get_physical_size(&mail->mail.mail, &tmp) == 0) {
+			/* we should have everything now. try again. */
+			(void)index_mail_get_cached_virtual_size(mail, &tmp);
+		}
+		mail->mail.mail.lookup_abort = old_abort;
+	}
+}
+
+int index_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct message_size hdr_size, body_size;
+	struct istream *input;
+	uoff_t old_offset;
+
+	if (index_mail_get_cached_virtual_size(mail, size_r))
+		return 0;
+
+	old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
+	if (mail_get_stream_because(_mail, &hdr_size, &body_size,
+			index_mail_cache_reason(_mail, "virtual size"), &input) < 0)
+		return -1;
+	i_stream_seek(data->stream, old_offset);
+
+	i_assert(data->virtual_size != UOFF_T_MAX);
+	*size_r = data->virtual_size;
+	return 0;
+}
+
+int index_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	uoff_t size;
+
+	if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NOT_IN_CACHE &&
+	    _mail->lookup_abort != MAIL_LOOKUP_ABORT_READ_MAIL) {
+		/* If size.physical isn't in cache yet, add it. Do this only
+		   when the caller appears to actually want it to be cached.
+		   We don't want to cache the size when coming in here from
+		   i_stream_mail_try_get_cached_size() or
+		   index_mail_get_cached_body_size(). */
+		data->cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
+	}
+	if (data->physical_size == UOFF_T_MAX) {
+		if (index_mail_get_cached_uoff_t(mail,
+						 MAIL_CACHE_PHYSICAL_FULL_SIZE,
+						 &size))
+			data->physical_size = size;
+		else
+			(void)get_cached_msgpart_sizes(mail);
+	}
+	*size_r = data->physical_size;
+	return *size_r == UOFF_T_MAX ? -1 : 0;
+}
+
+void index_mail_cache_add(struct index_mail *mail, enum index_cache_field field,
+			  const void *data, size_t data_size)
+{
+	index_mail_cache_add_idx(mail, mail->ibox->cache_fields[field].idx,
+				 data, data_size);
+}
+
+void index_mail_cache_add_idx(struct index_mail *mail, unsigned int field_idx,
+			      const void *data, size_t data_size)
+{
+	struct mail *_mail = &mail->mail.mail;
+	const struct mail_storage_settings *set = _mail->box->storage->set;
+	const struct mail_index_header *hdr;
+
+	if (set->mail_cache_min_mail_count > 0) {
+		/* First check if we've configured caching not to be used with
+		   low enough message count. */
+		hdr = mail_index_get_header(_mail->transaction->view);
+		if (hdr->messages_count < set->mail_cache_min_mail_count)
+			return;
+	}
+
+	if (!mail->data.no_caching &&
+	    mail->data.dont_cache_field_idx != field_idx &&
+	    !_mail->box->mail_cache_disabled) {
+		mail_cache_add(_mail->transaction->cache_trans, _mail->seq,
+			       field_idx, data, data_size);
+	}
+}
+
+void index_mail_cache_pop3_data(struct mail *_mail,
+				const char *uidl, uint32_t order)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	if (uidl != NULL)
+		index_mail_cache_add(mail, MAIL_CACHE_POP3_UIDL,
+				     uidl, strlen(uidl));
+
+	if (order != 0)
+		index_mail_cache_add(mail, MAIL_CACHE_POP3_ORDER,
+				     &order, sizeof(order));
+}
+
+static void parse_bodystructure_part_header(struct message_part *part,
+					    struct message_header_line *hdr,
+					    pool_t pool)
+{
+	message_part_data_parse_from_header(pool, part, hdr);
+}
+
+static bool want_plain_bodystructure_cached(struct index_mail *mail)
+{
+	const unsigned int cache_field_body =
+		mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODY].idx;
+	const unsigned int cache_field_bodystructure =
+		mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
+	struct mail *_mail = &mail->mail.mail;
+
+	if ((mail->data.wanted_fields & (MAIL_FETCH_IMAP_BODY |
+					 MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0)
+		return TRUE;
+
+	if (mail_cache_field_want_add(_mail->transaction->cache_trans,
+				      _mail->seq, cache_field_body))
+		return TRUE;
+	if (mail_cache_field_want_add(_mail->transaction->cache_trans,
+				      _mail->seq, cache_field_bodystructure))
+		return TRUE;
+	return FALSE;
+}
+
+static void index_mail_body_parsed_cache_flags(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct index_mail_data *data = &mail->data;
+	unsigned int cache_flags_idx;
+	uint32_t cache_flags = data->cache_flags;
+	bool want_cached;
+
+	cache_flags_idx = mail->ibox->cache_fields[MAIL_CACHE_FLAGS].idx;
+	want_cached = mail_cache_field_want_add(_mail->transaction->cache_trans,
+						_mail->seq, cache_flags_idx);
+
+	if (data->parsed_bodystructure &&
+	    message_part_data_is_plain_7bit(data->parts) &&
+	    (want_cached || want_plain_bodystructure_cached(mail))) {
+		cache_flags |= MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII;
+		/* we need message_parts cached to be able to
+		   actually use it in BODY/BODYSTRUCTURE reply */
+		want_cached = TRUE;
+		data->save_message_parts = TRUE;
+	}
+
+	/* cache flags should never get unset as long as the message doesn't
+	   change, but try to handle it anyway */
+	cache_flags &= ENUM_NEGATE(MAIL_CACHE_FLAG_BINARY_HEADER |
+				   MAIL_CACHE_FLAG_BINARY_BODY |
+				   MAIL_CACHE_FLAG_HAS_NULS |
+				   MAIL_CACHE_FLAG_HAS_NO_NULS);
+	if (message_parts_have_nuls(data->parts)) {
+		_mail->has_nuls = TRUE;
+		_mail->has_no_nuls = FALSE;
+		cache_flags |= MAIL_CACHE_FLAG_HAS_NULS;
+	} else {
+		_mail->has_nuls = FALSE;
+		_mail->has_no_nuls = TRUE;
+		cache_flags |= MAIL_CACHE_FLAG_HAS_NO_NULS;
+	}
+
+	if (data->hdr_size.virtual_size == data->hdr_size.physical_size)
+		cache_flags |= MAIL_CACHE_FLAG_BINARY_HEADER;
+	if (data->body_size.virtual_size == data->body_size.physical_size)
+		cache_flags |= MAIL_CACHE_FLAG_BINARY_BODY;
+
+	if (cache_flags != data->cache_flags && want_cached) {
+		index_mail_cache_add_idx(mail, cache_flags_idx,
+					 &cache_flags, sizeof(cache_flags));
+	}
+	data->cache_flags = cache_flags;
+}
+
+static void index_mail_body_parsed_cache_message_parts(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct index_mail_data *data = &mail->data;
+	const unsigned int cache_field =
+		mail->ibox->cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx;
+	enum mail_cache_decision_type decision;
+	buffer_t *buffer;
+
+	if (data->messageparts_saved_to_cache ||
+	    mail_cache_field_exists(_mail->transaction->cache_view, _mail->seq,
+				    cache_field) != 0) {
+		/* already cached */
+		return;
+	}
+
+	decision = mail_cache_field_get_decision(_mail->box->cache,
+						 cache_field);
+	if (decision == (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED)) {
+		/* we never want it cached */
+		return;
+	}
+	if (decision == MAIL_CACHE_DECISION_NO &&
+	    !data->save_message_parts &&
+	    (data->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) == 0) {
+		/* we didn't really care about the message parts themselves,
+		   just wanted to use something that depended on it */
+		return;
+	}
+
+	T_BEGIN {
+		buffer = t_buffer_create(1024);
+		message_part_serialize(mail->data.parts, buffer);
+		index_mail_cache_add_idx(mail, cache_field,
+					 buffer->data, buffer->used);
+	} T_END;
+
+	data->messageparts_saved_to_cache = TRUE;
+}
+
+static int
+index_mail_write_bodystructure(struct index_mail *mail, string_t *str,
+			       bool extended)
+{
+	const char *error;
+
+	if (imap_bodystructure_write(mail->data.parts, str, extended,
+				     &error) < 0) {
+		mail_set_cache_corrupted(&mail->mail.mail,
+			MAIL_FETCH_MESSAGE_PARTS, error);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+index_mail_body_parsed_cache_bodystructure(struct index_mail *mail,
+					   enum index_cache_field field)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct index_mail_data *data = &mail->data;
+	const unsigned int cache_field_parts =
+		mail->ibox->cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx;
+	const unsigned int cache_field_body =
+		mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODY].idx;
+	const unsigned int cache_field_bodystructure =
+		mail->ibox->cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
+	enum mail_cache_decision_type dec;
+	string_t *str;
+	bool bodystructure_cached = FALSE;
+	bool plain_bodystructure = FALSE;
+	bool cache_bodystructure, cache_body;
+
+	if ((data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) != 0) {
+		if (data->messageparts_saved_to_cache ||
+		    mail_cache_field_exists(_mail->transaction->cache_view,
+					    _mail->seq, cache_field_parts) > 0) {
+			/* cached it as flag + message_parts */
+			plain_bodystructure = TRUE;
+		}
+	}
+
+	if (!data->parsed_bodystructure)
+		return;
+	i_assert(data->parts != NULL);
+
+	/* If BODY is fetched first but BODYSTRUCTURE is also wanted, we don't
+	   normally want to first cache BODY and then BODYSTRUCTURE. So check
+	   the wanted_fields also in here. */
+	if (plain_bodystructure)
+		cache_bodystructure = FALSE;
+	else if (field == MAIL_CACHE_IMAP_BODYSTRUCTURE ||
+		 (data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0) {
+		cache_bodystructure =
+			mail_cache_field_can_add(_mail->transaction->cache_trans,
+				_mail->seq, cache_field_bodystructure);
+	} else {
+		cache_bodystructure =
+			mail_cache_field_want_add(_mail->transaction->cache_trans,
+				_mail->seq, cache_field_bodystructure);
+	}
+	if (cache_bodystructure) {
+		str = str_new(mail->mail.data_pool, 128);
+		if (index_mail_write_bodystructure(mail, str, TRUE) == 0) {
+			data->bodystructure = str_c(str);
+			index_mail_cache_add(mail, MAIL_CACHE_IMAP_BODYSTRUCTURE,
+					     str_c(str), str_len(str));
+			bodystructure_cached = TRUE;
+		}
+	} else {
+		bodystructure_cached =
+			mail_cache_field_exists(_mail->transaction->cache_view,
+				_mail->seq, cache_field_bodystructure) > 0;
+	}
+
+	/* normally don't cache both BODY and BODYSTRUCTURE, but do it
+	   if BODY is forced to be cached */
+	dec = mail_cache_field_get_decision(_mail->box->cache,
+					    cache_field_body);
+	if (plain_bodystructure ||
+	    (bodystructure_cached &&
+	     (dec != (MAIL_CACHE_DECISION_FORCED | MAIL_CACHE_DECISION_YES))))
+		cache_body = FALSE;
+	else if (field == MAIL_CACHE_IMAP_BODY) {
+		cache_body =
+			mail_cache_field_can_add(_mail->transaction->cache_trans,
+				_mail->seq, cache_field_body);
+	} else {
+		cache_body =
+			mail_cache_field_want_add(_mail->transaction->cache_trans,
+				_mail->seq, cache_field_body);
+	}
+
+	if (cache_body) {
+		str = str_new(mail->mail.data_pool, 128);
+		if (index_mail_write_bodystructure(mail, str, FALSE) == 0) {
+			data->body = str_c(str);
+			index_mail_cache_add(mail, MAIL_CACHE_IMAP_BODY,
+					     str_c(str), str_len(str));
+		}
+	}
+}
+
+bool index_mail_want_cache(struct index_mail *mail, enum index_cache_field field)
+{
+	struct mail *_mail = &mail->mail.mail;
+	enum mail_fetch_field fetch_field;
+	unsigned int cache_field;
+
+	switch (field) {
+	case MAIL_CACHE_SENT_DATE:
+		fetch_field = MAIL_FETCH_DATE;
+		break;
+	case MAIL_CACHE_RECEIVED_DATE:
+		fetch_field = MAIL_FETCH_RECEIVED_DATE;
+		break;
+	case MAIL_CACHE_SAVE_DATE:
+		fetch_field = MAIL_FETCH_SAVE_DATE;
+		break;
+	case MAIL_CACHE_VIRTUAL_FULL_SIZE:
+		fetch_field = MAIL_FETCH_VIRTUAL_SIZE;
+		break;
+	case MAIL_CACHE_PHYSICAL_FULL_SIZE:
+		fetch_field = MAIL_FETCH_PHYSICAL_SIZE;
+		break;
+	case MAIL_CACHE_BODY_SNIPPET:
+		fetch_field = MAIL_FETCH_BODY_SNIPPET;
+		break;
+	default:
+		i_unreached();
+	}
+
+	if ((mail->data.dont_cache_fetch_fields & fetch_field) != 0)
+		return FALSE;
+
+	/* If a field has been explicitly requested to be fetched, it's
+	   included in data.cache_fetch_fields. In that case use _can_add() to
+	   add it to the cache file if at all possible. Otherwise, use
+	   _want_add() to use previous caching decisions. */
+	cache_field = mail->ibox->cache_fields[field].idx;
+	if ((mail->data.cache_fetch_fields & fetch_field) != 0) {
+		return mail_cache_field_can_add(_mail->transaction->cache_trans,
+						_mail->seq, cache_field);
+	} else {
+		return mail_cache_field_want_add(_mail->transaction->cache_trans,
+						 _mail->seq, cache_field);
+	}
+}
+
+static void index_mail_save_finish_make_snippet(struct index_mail *mail)
+{
+	if (mail->data.save_body_snippet) {
+		if (index_mail_write_body_snippet(mail) < 0)
+			return;
+		mail->data.save_body_snippet = FALSE;
+	}
+
+	if (mail->data.body_snippet != NULL &&
+	    index_mail_want_cache(mail, MAIL_CACHE_BODY_SNIPPET)) {
+		index_mail_cache_add(mail, MAIL_CACHE_BODY_SNIPPET,
+				     mail->data.body_snippet,
+				     strlen(mail->data.body_snippet));
+	}
+}
+
+static void index_mail_cache_sizes(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct mail_index_view *view = _mail->transaction->view;
+
+	static enum index_cache_field size_fields[] = {
+		MAIL_CACHE_VIRTUAL_FULL_SIZE,
+		MAIL_CACHE_PHYSICAL_FULL_SIZE
+	};
+	uoff_t sizes[N_ELEMENTS(size_fields)];
+	unsigned int i;
+	uint32_t vsize;
+	uint32_t idx ATTR_UNUSED;
+
+	sizes[0] = mail->data.virtual_size;
+	sizes[1] = mail->data.physical_size;
+
+	/* store the virtual size in index if
+		extension for it exists or
+		extension for box virtual size exists and
+		size fits and is present and
+		size is not cached or
+		cached size differs
+	*/
+	if ((mail_index_map_get_ext_idx(view->index->map, _mail->box->mail_vsize_ext_id, &idx) ||
+	     mail_index_map_get_ext_idx(view->index->map, _mail->box->vsize_hdr_ext_id, &idx)) &&
+	    (sizes[0] != UOFF_T_MAX &&
+	     sizes[0] < (uint32_t)-1)) {
+		const uint32_t *vsize_ext =
+			index_mail_get_vsize_extension(_mail);
+		/* vsize = 0 means it's not present in index, consult cache.
+		   we store vsize for every +4GB-1 mail to cache because
+		   index can only hold 2^32-1 size. Cache will not be used
+		   when vsize is stored in index. */
+		vsize = sizes[0] + 1;
+		if (vsize_ext == NULL || vsize != *vsize_ext) {
+			mail_index_update_ext(_mail->transaction->itrans, _mail->seq,
+					      _mail->box->mail_vsize_ext_id, &vsize, NULL);
+		}
+		/* it's already in index, so don't update cache */
+		sizes[0] = UOFF_T_MAX;
+	}
+
+	for (i = 0; i < N_ELEMENTS(size_fields); i++) {
+		if (sizes[i] != UOFF_T_MAX &&
+		    index_mail_want_cache(mail, size_fields[i])) {
+			index_mail_cache_add(mail, size_fields[i],
+					     &sizes[i], sizeof(sizes[i]));
+		}
+	}
+}
+
+static void index_mail_cache_dates(struct index_mail *mail)
+{
+	static enum index_cache_field date_fields[] = {
+		MAIL_CACHE_RECEIVED_DATE,
+		MAIL_CACHE_SAVE_DATE
+	};
+	time_t dates[N_ELEMENTS(date_fields)];
+	unsigned int i;
+	uint32_t t;
+
+	dates[0] = mail->data.received_date;
+	dates[1] = mail->mail.mail.saving ? ioloop_time :
+		mail->data.save_date;
+
+	for (i = 0; i < N_ELEMENTS(date_fields); i++) {
+		if (dates[i] != (time_t)-1 &&
+		    index_mail_want_cache(mail, date_fields[i])) {
+			t = dates[i];
+			index_mail_cache_add(mail, date_fields[i],
+					     &t, sizeof(t));
+		}
+	}
+
+	if (mail->data.sent_date_parsed &&
+	    index_mail_want_cache(mail, MAIL_CACHE_SENT_DATE))
+		(void)index_mail_cache_sent_date(mail);
+}
+
+static struct message_part *
+index_mail_find_first_text_mime_part(struct message_part *parts)
+{
+	struct message_part_data *body_data = parts->data;
+	struct message_part *part;
+
+	i_assert(body_data != NULL);
+
+	if (body_data->content_type == NULL ||
+	    strcasecmp(body_data->content_type, "text") == 0) {
+		/* use any text/ part, even if we don't know what exactly
+		   it is. */
+		return parts;
+	}
+	if (strcasecmp(body_data->content_type, "multipart") != 0) {
+		/* for now we support only text Content-Types */
+		return NULL;
+	}
+
+	if (strcasecmp(body_data->content_subtype, "alternative") == 0) {
+		/* text/plain > text/html > text/ */
+		struct message_part *html_part = NULL, *text_part = NULL;
+
+		for (part = parts->children; part != NULL; part = part->next) {
+			struct message_part_data *sub_body_data =
+				part->data;
+
+			i_assert(sub_body_data != NULL);
+
+			if (sub_body_data->content_type == NULL ||
+			    strcasecmp(sub_body_data->content_type, "text") == 0) {
+				if (sub_body_data->content_subtype == NULL ||
+				    strcasecmp(sub_body_data->content_subtype, "plain") == 0)
+					return part;
+				if (strcasecmp(sub_body_data->content_subtype, "html") == 0)
+					html_part = part;
+				else
+					text_part = part;
+			}
+		}
+		return html_part != NULL ? html_part : text_part;
+	}
+	/* find the first usable MIME part */
+	for (part = parts->children; part != NULL; part = part->next) {
+		struct message_part *subpart =
+			index_mail_find_first_text_mime_part(part);
+		if (subpart != NULL)
+			return subpart;
+	}
+	return NULL;
+}
+
+static int index_mail_write_body_snippet(struct index_mail *mail)
+{
+	struct message_part *part;
+	struct istream *input;
+	uoff_t old_offset;
+	string_t *str;
+	int ret;
+
+	i_assert(mail->data.parsed_bodystructure);
+
+	part = index_mail_find_first_text_mime_part(mail->data.parts);
+	if (part == NULL) {
+		mail->data.body_snippet = BODY_SNIPPET_ALGO_V1;
+		return 0;
+	}
+
+	old_offset = mail->data.stream == NULL ? 0 : mail->data.stream->v_offset;
+	const char *reason = index_mail_cache_reason(&mail->mail.mail, "snippet");
+	if (mail_get_stream_because(&mail->mail.mail, NULL, NULL, reason, &input) < 0)
+		return -1;
+	i_assert(mail->data.stream != NULL);
+
+	i_stream_seek(input, part->physical_pos);
+	input = i_stream_create_limit(input, part->header_size.physical_size +
+				      part->body_size.physical_size);
+
+	str = str_new(mail->mail.data_pool, 128);
+	str_append(str, BODY_SNIPPET_ALGO_V1);
+	ret = message_snippet_generate(input, BODY_SNIPPET_MAX_CHARS, str);
+	if (ret == 0)
+		mail->data.body_snippet = str_c(str);
+	i_stream_destroy(&input);
+
+	i_stream_seek(mail->data.stream, old_offset);
+	return ret;
+}
+
+void index_mail_parts_reset(struct index_mail *mail)
+{
+	mail->data.parts = NULL;
+	mail->data.parsed_bodystructure_header = FALSE;
+	mail->data.parsed_bodystructure = FALSE;
+}
+
+static int
+index_mail_parse_body_finish(struct index_mail *mail,
+			     enum index_cache_field field, bool success)
+{
+	struct istream *parser_input = mail->data.parser_input;
+	const struct mail_storage_settings *mail_set =
+		mailbox_get_settings(mail->mail.mail.box);
+	const char *error = NULL;
+	int ret;
+
+	if (parser_input == NULL) {
+		ret = message_parser_deinit_from_parts(&mail->data.parser_ctx,
+			&mail->data.parts, &error) < 0 ? 0 : 1;
+	} else {
+		mail->data.parser_input = NULL;
+		i_stream_ref(parser_input);
+		ret = message_parser_deinit_from_parts(&mail->data.parser_ctx,
+			&mail->data.parts, &error) < 0 ? 0 : 1;
+		if (success && (parser_input->stream_errno == 0 ||
+				parser_input->stream_errno == EPIPE)) {
+			/* do one final read, which verifies that the message
+			   size is correct. */
+			if (i_stream_read(parser_input) != -1 ||
+			    i_stream_have_bytes_left(parser_input))
+				i_unreached();
+		}
+		/* EPIPE = input already closed. allow the caller to
+		   decide if that is an error or not. (for example we
+		   could be coming here from IMAP APPEND when IMAP
+		   client has closed the connection too early. we
+		   don't want to log an error in that case.)
+		   Note that EPIPE may also come from istream-mail which
+		   detects a corrupted message size. Either way, the
+		   body wasn't successfully parsed. */
+		if (parser_input->stream_errno == 0)
+			;
+		else if (parser_input->stream_errno == EPIPE)
+			ret = -1;
+		else {
+			index_mail_stream_log_failure_for(mail, parser_input);
+			ret = -1;
+		}
+		i_stream_unref(&parser_input);
+	}
+	if (ret <= 0) {
+		if (ret == 0) {
+			i_assert(error != NULL);
+			index_mail_set_message_parts_corrupted(&mail->mail.mail, error);
+		}
+		index_mail_parts_reset(mail);
+		if (mail->data.save_bodystructure_body)
+			mail->data.save_bodystructure_header = TRUE;
+		if (mail->data.header_parser_initialized)
+			index_mail_parse_header_deinit(mail);
+		return -1;
+	}
+	if (mail->data.header_parser_initialized) {
+		i_assert(!success);
+		index_mail_parse_header_deinit(mail);
+	}
+
+	if (mail->data.save_bodystructure_body) {
+		mail->data.parsed_bodystructure = TRUE;
+		mail->data.save_bodystructure_header = FALSE;
+		mail->data.save_bodystructure_body = FALSE;
+		i_assert(mail->data.parts != NULL);
+	}
+
+	if (mail->data.no_caching) {
+		/* if we're here because we aborted parsing, don't get any
+		   further or we may crash while generating output from
+		   incomplete data */
+		return 0;
+	}
+
+	(void)get_cached_msgpart_sizes(mail);
+
+	index_mail_body_parsed_cache_flags(mail);
+	index_mail_body_parsed_cache_message_parts(mail);
+	index_mail_body_parsed_cache_bodystructure(mail, field);
+	index_mail_cache_sizes(mail);
+	index_mail_cache_dates(mail);
+	if (mail_set->parsed_mail_attachment_detection_add_flags &&
+	    !mail_has_attachment_keywords(&mail->mail.mail))
+		index_mail_try_set_attachment_keywords(mail);
+	return 0;
+}
+
+static void index_mail_stream_log_failure(struct index_mail *mail)
+{
+	index_mail_stream_log_failure_for(mail, mail->data.stream);
+}
+
+int index_mail_stream_check_failure(struct index_mail *mail)
+{
+	if (mail->data.stream->stream_errno == 0)
+		return 0;
+	index_mail_stream_log_failure(mail);
+	return -1;
+}
+
+void index_mail_refresh_expunged(struct mail *mail)
+{
+	mail_index_refresh(mail->box->index);
+	if (mail_index_is_expunged(mail->transaction->view, mail->seq))
+		mail_set_expunged(mail);
+}
+
+void index_mail_stream_log_failure_for(struct index_mail *mail,
+				       struct istream *input)
+{
+	struct mail *_mail = &mail->mail.mail;
+
+	i_assert(input->stream_errno != 0);
+
+	if (input->stream_errno == ENOENT) {
+		/* was the mail just expunged? we could get here especially if
+		   external attachments are used and the attachment is deleted
+		   before we've opened the file. */
+		index_mail_refresh_expunged(_mail);
+		if (_mail->expunged)
+			return;
+	}
+
+	const char *old_error =
+		mailbox_get_last_internal_error(_mail->box, NULL);
+	const char *new_error = t_strdup_printf("read(%s) failed: %s",
+		i_stream_get_name(input), i_stream_get_error(input));
+
+	if (mail->data.istream_error_logged &&
+	    strstr(old_error, new_error) != NULL) {
+		/* Avoid logging the same istream error multiple times
+		   (even if the read reason is different). The old_error begins
+		   with the UID=n prefix, which we can ignore since we know
+		   that this mail already logged a critical error, so it has
+		   to be about this same mail. */
+		return;
+	}
+	mail->data.istream_error_logged = TRUE;
+	mail_set_critical(_mail, "%s (read reason=%s)", new_error,
+		mail->mail.get_stream_reason == NULL ? "" :
+		mail->mail.get_stream_reason);
+}
+
+static int index_mail_parse_body(struct index_mail *mail,
+				 enum index_cache_field field)
+{
+	struct index_mail_data *data = &mail->data;
+	uoff_t old_offset;
+	int ret;
+
+	i_assert(data->parser_ctx != NULL);
+
+	old_offset = data->stream->v_offset;
+	i_stream_seek(data->stream, data->hdr_size.physical_size);
+
+	if (data->save_bodystructure_body) {
+		/* bodystructure header is parsed, we want the body's mime
+		   headers too */
+		i_assert(data->parsed_bodystructure_header);
+		message_parser_parse_body(data->parser_ctx,
+					  parse_bodystructure_part_header,
+					  mail->mail.data_pool);
+	} else {
+		message_parser_parse_body(data->parser_ctx,
+			*null_message_part_header_callback, NULL);
+	}
+	ret = index_mail_stream_check_failure(mail);
+	if (index_mail_parse_body_finish(mail, field, TRUE) < 0)
+		ret = -1;
+
+	i_stream_seek(data->stream, old_offset);
+	return ret;
+}
+
+static void index_mail_stream_destroy_callback(struct index_mail *mail)
+{
+	i_assert(mail->data.destroying_stream);
+
+	mail->data.destroying_stream = FALSE;
+}
+
+void index_mail_set_read_buffer_size(struct mail *_mail, struct istream *input)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	unsigned int block_size;
+
+	i_stream_set_max_buffer_size(input, MAIL_READ_FULL_BLOCK_SIZE);
+	block_size = (mail->data.access_part & (READ_BODY | PARSE_BODY)) != 0 ?
+		MAIL_READ_FULL_BLOCK_SIZE : MAIL_READ_HDR_BLOCK_SIZE;
+	i_stream_set_init_buffer_size(input, block_size);
+}
+
+int index_mail_init_stream(struct index_mail *mail,
+			   struct message_size *hdr_size,
+			   struct message_size *body_size,
+			   struct istream **stream_r)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct index_mail_data *data = &mail->data;
+	struct istream *input;
+	bool has_nuls, body_size_from_stream = FALSE;
+	int ret;
+
+	i_assert(_mail->mail_stream_accessed);
+
+	if (!data->initialized_wrapper_stream &&
+	    _mail->transaction->stats_track) {
+		input = i_stream_create_mail(_mail, data->stream,
+					     !data->stream_has_only_header);
+		i_stream_unref(&data->stream);
+		data->stream = input;
+		data->initialized_wrapper_stream = TRUE;
+	}
+
+	if (!data->destroy_callback_set) {
+		/* do this only once in case a plugin changes the stream.
+		   otherwise the check would break. */
+		data->destroy_callback_set = TRUE;
+		i_stream_add_destroy_callback(data->stream,
+			index_mail_stream_destroy_callback, mail);
+	}
+
+	bool want_attachment_kw =
+		index_mail_want_attachment_keywords_on_fetch(mail);
+	if (want_attachment_kw) {
+		data->access_part |= PARSE_HDR | PARSE_BODY;
+		data->access_reason_code = "mail:attachment_keywords";
+	}
+
+	if (hdr_size != NULL || body_size != NULL)
+		(void)get_cached_msgpart_sizes(mail);
+
+	bool want_body_parsing = want_attachment_kw ||
+		(body_size != NULL && !data->body_size_set &&
+		 (data->access_part & PARSE_BODY) != 0);
+
+	if (hdr_size != NULL || body_size != NULL || want_body_parsing) {
+		i_stream_seek(data->stream, 0);
+		if (!data->hdr_size_set || want_body_parsing) {
+			if ((data->access_part & (PARSE_HDR | PARSE_BODY)) != 0) {
+				(void)get_cached_parts(mail);
+				if (index_mail_parse_headers_internal(mail, NULL) < 0)
+					return -1;
+			} else {
+				if (message_get_header_size(data->stream,
+							    &data->hdr_size,
+							    &has_nuls) < 0) {
+					index_mail_stream_log_failure(mail);
+					return -1;
+				}
+				data->hdr_size_set = TRUE;
+			}
+		}
+
+		if (hdr_size != NULL)
+			*hdr_size = data->hdr_size;
+	}
+
+	if (body_size != NULL || want_body_parsing) {
+		if (!data->body_size_set && body_size != NULL)
+			index_mail_get_cached_body_size(mail);
+		if (!data->body_size_set || want_body_parsing) {
+			i_stream_seek(data->stream,
+				      data->hdr_size.physical_size);
+			if ((data->access_part & PARSE_BODY) != 0) {
+				if (index_mail_parse_body(mail, 0) < 0)
+					return -1;
+			} else {
+				if (message_get_body_size(data->stream,
+							  &data->body_size,
+							  &has_nuls) < 0) {
+					index_mail_stream_log_failure(mail);
+					return -1;
+				}
+				data->body_size_set = TRUE;
+			}
+			body_size_from_stream = TRUE;
+		}
+
+		if (body_size != NULL)
+			*body_size = data->body_size;
+	}
+
+	if (data->hdr_size_set && data->body_size_set) {
+		data->virtual_size = data->hdr_size.virtual_size +
+			data->body_size.virtual_size;
+		data->physical_size = data->hdr_size.physical_size +
+			data->body_size.physical_size;
+		if (body_size_from_stream) {
+			/* the sizes were just calculated */
+			data->inexact_total_sizes = FALSE;
+		}
+	} else {
+		/* If body_size==NULL, the caller doesn't care about it.
+		   However, try to set it anyway if it can be calculated. */
+		index_mail_try_set_body_size(mail);
+	}
+	ret = index_mail_stream_check_failure(mail);
+
+	i_stream_seek(data->stream, 0);
+	if (ret < 0)
+		return -1;
+	*stream_r = data->stream;
+	return 0;
+}
+
+static int
+index_mail_parse_bodystructure_full(struct index_mail *mail,
+				    enum index_cache_field field)
+{
+	struct index_mail_data *data = &mail->data;
+
+	if ((data->save_bodystructure_header &&
+	     !data->parsed_bodystructure_header) ||
+	    !data->save_bodystructure_body ||
+	    field == MAIL_CACHE_BODY_SNIPPET) {
+		/* we haven't parsed the header yet */
+		const char *reason =
+			index_mail_cache_reason(&mail->mail.mail, "bodystructure");
+		bool orig_bodystructure_header =
+			data->save_bodystructure_header;
+		bool orig_bodystructure_body =
+			data->save_bodystructure_body;
+		data->save_bodystructure_header = TRUE;
+		data->save_bodystructure_body = TRUE;
+		(void)get_cached_parts(mail);
+		if (index_mail_parse_headers(mail, NULL, reason) < 0) {
+			data->save_bodystructure_header =
+				orig_bodystructure_header;
+			data->save_bodystructure_body =
+				orig_bodystructure_body;
+			return -1;
+		}
+		i_assert(data->parser_ctx != NULL);
+	}
+
+	return index_mail_parse_body(mail, field);
+}
+
+static int index_mail_parse_bodystructure(struct index_mail *mail,
+					  enum index_cache_field field)
+{
+	struct index_mail_data *data = &mail->data;
+	string_t *str;
+
+	if (data->parsed_bodystructure && field != MAIL_CACHE_BODY_SNIPPET) {
+		/* we have everything parsed already, but just not written to
+		   a string */
+		index_mail_body_parsed_cache_bodystructure(mail, field);
+	} else {
+		if (index_mail_parse_bodystructure_full(mail, field) < 0)
+			return -1;
+		if (data->parts == NULL) {
+			/* Corrupted mime.parts detected. Retry by parsing
+			   the mail. */
+			data->parsed_bodystructure = FALSE;
+			data->parsed_bodystructure_header = FALSE;
+			data->save_bodystructure_header = TRUE;
+			data->save_bodystructure_body = TRUE;
+			if (index_mail_parse_bodystructure_full(mail, field) < 0)
+				return -1;
+		}
+	}
+	i_assert(data->parts != NULL);
+
+	/* if we didn't want to have the body(structure) cached,
+	   it's still not written. */
+	switch (field) {
+	case MAIL_CACHE_IMAP_BODY:
+		if (data->body == NULL) {
+			str = str_new(mail->mail.data_pool, 128);
+			if (index_mail_write_bodystructure(mail, str, FALSE) < 0)
+				return -1;
+			data->body = str_c(str);
+		}
+		break;
+	case MAIL_CACHE_IMAP_BODYSTRUCTURE:
+		if (data->bodystructure == NULL) {
+			str = str_new(mail->mail.data_pool, 128);
+			if (index_mail_write_bodystructure(mail, str, TRUE) < 0)
+				return -1;
+			data->bodystructure = str_c(str);
+		}
+		break;
+	case MAIL_CACHE_BODY_SNIPPET:
+		if (data->body_snippet == NULL) {
+			if (index_mail_write_body_snippet(mail) < 0)
+				return -1;
+
+			if (index_mail_want_cache(mail, MAIL_CACHE_BODY_SNIPPET))
+				index_mail_cache_add(mail, MAIL_CACHE_BODY_SNIPPET,
+						     mail->data.body_snippet,
+						     strlen(mail->data.body_snippet));
+		}
+		i_assert(data->body_snippet != NULL &&
+			 data->body_snippet[0] != '\0');
+		break;
+	default:
+		i_unreached();
+	}
+	return 0;
+}
+
+static void
+index_mail_get_plain_bodystructure(struct index_mail *mail, string_t *str,
+				   bool extended)
+{
+	str_printfa(str, IMAP_BODY_PLAIN_7BIT_ASCII" %"PRIuUOFF_T" %u",
+		    mail->data.parts->body_size.virtual_size,
+		    mail->data.parts->body_size.lines);
+	if (extended)
+		str_append(str, " NIL NIL NIL NIL");
+}
+
+static int
+index_mail_fetch_body_snippet(struct index_mail *mail, const char **value_r)
+{
+	const struct mail_cache_field *cache_fields = mail->ibox->cache_fields;
+	const unsigned int cache_field =
+		cache_fields[MAIL_CACHE_BODY_SNIPPET].idx;
+	string_t *str;
+
+	mail->data.cache_fetch_fields |= MAIL_FETCH_BODY_SNIPPET;
+	if (mail->data.body_snippet == NULL) {
+		str = str_new(mail->mail.data_pool, 128);
+		if (index_mail_cache_lookup_field(mail, str, cache_field) > 0 &&
+		    str_len(str) > 0)
+			mail->data.body_snippet = str_c(str);
+	}
+	if (mail->data.body_snippet != NULL) {
+		*value_r = mail->data.body_snippet;
+		return 0;
+	}
+
+	/* reuse the IMAP bodystructure parsing code to get all the useful
+	   headers that we need. */
+	mail->data.save_body_snippet = TRUE;
+	if (index_mail_parse_bodystructure(mail, MAIL_CACHE_BODY_SNIPPET) < 0)
+		return -1;
+	i_assert(mail->data.body_snippet != NULL);
+	*value_r = mail->data.body_snippet;
+	return 0;
+}
+
+bool index_mail_get_cached_body(struct index_mail *mail, const char **value_r)
+{
+	const struct mail_cache_field *cache_fields = mail->ibox->cache_fields;
+	const unsigned int body_cache_field =
+		cache_fields[MAIL_CACHE_IMAP_BODY].idx;
+	const unsigned int bodystructure_cache_field =
+		cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
+	struct index_mail_data *data = &mail->data;
+	string_t *str;
+	const char *error;
+
+	if (data->body != NULL) {
+		*value_r = data->body;
+		return TRUE;
+	}
+
+	str = str_new(mail->mail.data_pool, 128);
+	if ((data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) != 0 &&
+	    get_cached_parts(mail)) {
+		index_mail_get_plain_bodystructure(mail, str, FALSE);
+		*value_r = data->body = str_c(str);
+		return TRUE;
+	}
+
+	/* 2) get BODY if it exists */
+	if (index_mail_cache_lookup_field(mail, str, body_cache_field) > 0) {
+		*value_r = data->body = str_c(str);
+		return TRUE;
+	}
+	/* 3) get it using BODYSTRUCTURE if it exists */
+	if (index_mail_cache_lookup_field(mail, str, bodystructure_cache_field) > 0) {
+		data->bodystructure =
+			p_strdup(mail->mail.data_pool, str_c(str));
+		str_truncate(str, 0);
+
+		if (imap_body_parse_from_bodystructure(data->bodystructure,
+						       str, &error) < 0) {
+			/* broken, continue.. */
+			mail_set_cache_corrupted(&mail->mail.mail,
+				MAIL_FETCH_IMAP_BODYSTRUCTURE, t_strdup_printf(
+				"Invalid BODYSTRUCTURE %s: %s",
+				data->bodystructure, error));
+		} else {
+			*value_r = data->body = str_c(str);
+			return TRUE;
+		}
+	}
+
+	str_free(&str);
+	return FALSE;
+}
+
+bool index_mail_get_cached_bodystructure(struct index_mail *mail,
+					 const char **value_r)
+{
+	const struct mail_cache_field *cache_fields = mail->ibox->cache_fields;
+	const unsigned int bodystructure_cache_field =
+		cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
+	struct index_mail_data *data = &mail->data;
+	string_t *str;
+
+	if (data->bodystructure != NULL) {
+		*value_r = data->bodystructure;
+		return TRUE;
+	}
+
+	str = str_new(mail->mail.data_pool, 128);
+	if ((data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) != 0 &&
+	    get_cached_parts(mail))
+		index_mail_get_plain_bodystructure(mail, str, TRUE);
+	else if (index_mail_cache_lookup_field(mail, str,
+			bodystructure_cache_field) <= 0) {
+		str_free(&str);
+		return FALSE;
+	}
+
+	*value_r = data->bodystructure = str_c(str);
+	if (index_mail_want_attachment_keywords_on_fetch(mail))
+		index_mail_try_set_attachment_keywords(mail);
+	return TRUE;
+}
+
+int index_mail_get_special(struct mail *_mail,
+			   enum mail_fetch_field field, const char **value_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	switch (field) {
+	case MAIL_FETCH_IMAP_BODY:
+		if (index_mail_get_cached_body(mail, value_r))
+			return 0;
+
+		/* parse body structure, and save BODY/BODYSTRUCTURE
+		   depending on what we want cached */
+		if (index_mail_parse_bodystructure(mail, MAIL_CACHE_IMAP_BODY) < 0)
+			return -1;
+		i_assert(data->body != NULL);
+		*value_r = data->body;
+		return 0;
+	case MAIL_FETCH_IMAP_BODYSTRUCTURE:
+		if (index_mail_get_cached_bodystructure(mail, value_r))
+			return 0;
+
+		if (index_mail_parse_bodystructure(mail, MAIL_CACHE_IMAP_BODYSTRUCTURE) < 0)
+			return -1;
+		i_assert(data->bodystructure != NULL);
+		*value_r = data->bodystructure;
+		return 0;
+	case MAIL_FETCH_IMAP_ENVELOPE:
+		if (data->envelope == NULL) {
+			if (index_mail_headers_get_envelope(mail) < 0)
+				return -1;
+		}
+		*value_r = data->envelope;
+		return 0;
+	case MAIL_FETCH_FROM_ENVELOPE:
+		*value_r = data->from_envelope != NULL ?
+			data->from_envelope : "";
+		return 0;
+	case MAIL_FETCH_BODY_SNIPPET:
+		return index_mail_fetch_body_snippet(mail, value_r);
+	case MAIL_FETCH_STORAGE_ID:
+	case MAIL_FETCH_UIDL_BACKEND:
+	case MAIL_FETCH_SEARCH_RELEVANCY:
+	case MAIL_FETCH_GUID:
+	case MAIL_FETCH_HEADER_MD5:
+	case MAIL_FETCH_POP3_ORDER:
+	case MAIL_FETCH_REFCOUNT:
+	case MAIL_FETCH_REFCOUNT_ID:
+		*value_r = "";
+		return 0;
+	case MAIL_FETCH_MAILBOX_NAME:
+		*value_r = _mail->box->vname;
+		return 0;
+	default:
+		i_unreached();
+	}
+}
+
+int index_mail_get_backend_mail(struct mail *mail,
+				struct mail **real_mail_r)
+{
+	*real_mail_r = mail;
+	return 0;
+}
+
+struct mail *
+index_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers)
+{
+	struct index_mail *mail;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mail", 2048);
+	mail = p_new(pool, struct index_mail, 1);
+
+	index_mail_init(mail, t, wanted_fields, wanted_headers, pool, NULL);
+	return &mail->mail.mail;
+}
+
+void index_mail_init(struct index_mail *mail,
+		     struct mailbox_transaction_context *t,
+		     enum mail_fetch_field wanted_fields,
+		     struct mailbox_header_lookup_ctx *wanted_headers,
+		     struct pool *mail_pool,
+		     struct pool *data_pool)
+{
+	mail->mail.pool = mail_pool;
+	array_create(&mail->mail.module_contexts, mail->mail.pool,
+		     sizeof(void *), 5);
+
+	mail->mail.v = *t->box->mail_vfuncs;
+	mail->mail.mail.box = t->box;
+	mail->mail.mail.transaction = t;
+	t->mail_ref_count++;
+	if (data_pool != NULL)
+		mail->mail.data_pool = data_pool;
+	else
+		mail->mail.data_pool = pool_alloconly_create("index_mail", 16384);
+	mail->ibox = INDEX_STORAGE_CONTEXT(t->box);
+	mail->mail.wanted_fields = wanted_fields;
+	if (wanted_headers != NULL) {
+		mail->mail.wanted_headers = wanted_headers;
+		mailbox_header_lookup_ref(wanted_headers);
+	}
+	index_mail_init_data(mail);
+}
+
+static void index_mail_close_streams_full(struct index_mail *mail, bool closing)
+{
+	struct index_mail_data *data = &mail->data;
+	struct message_part *parts;
+	const char *error;
+
+	if (data->parser_ctx != NULL) {
+		if (message_parser_deinit_from_parts(&data->parser_ctx, &parts, &error) < 0)
+			index_mail_set_message_parts_corrupted(&mail->mail.mail, error);
+		mail->data.parser_input = NULL;
+		if (mail->data.save_bodystructure_body)
+			mail->data.save_bodystructure_header = TRUE;
+	}
+	i_stream_unref(&data->filter_stream);
+	if (data->stream != NULL) {
+		struct istream *orig_stream = data->stream;
+
+		data->destroying_stream = TRUE;
+		if (!closing && data->destroy_callback_set) {
+			/* we're replacing the stream with a new one. it's
+			   allowed to have references until the mail is closed
+			   (but we can't really check that) */
+			i_stream_remove_destroy_callback(data->stream,
+				index_mail_stream_destroy_callback);
+		}
+		i_stream_unref(&data->stream);
+		/* there must be no references to the mail when the
+		   mail is being closed. */
+		if (!closing)
+			data->destroying_stream = FALSE;
+		else if (mail->data.destroying_stream) {
+			i_panic("Input stream %s unexpectedly has references",
+				i_stream_get_name(orig_stream));
+		}
+
+		data->initialized_wrapper_stream = FALSE;
+		data->destroy_callback_set = FALSE;
+	}
+}
+
+void index_mail_close_streams(struct index_mail *mail)
+{
+	index_mail_close_streams_full(mail, FALSE);
+}
+
+static void index_mail_init_data(struct index_mail *mail)
+{
+	struct index_mail_data *data = &mail->data;
+
+	data->virtual_size = UOFF_T_MAX;
+	data->physical_size = UOFF_T_MAX;
+	data->save_date = (time_t)-1;
+	data->received_date = (time_t)-1;
+	data->sent_date.time = (uint32_t)-1;
+	data->dont_cache_field_idx = UINT_MAX;
+
+	data->wanted_fields = mail->mail.wanted_fields;
+	if (mail->mail.wanted_headers != NULL) {
+		data->wanted_headers = mail->mail.wanted_headers;
+		mailbox_header_lookup_ref(data->wanted_headers);
+	}
+}
+
+static void index_mail_reset_data(struct index_mail *mail)
+{
+	i_zero(&mail->data);
+	p_clear(mail->mail.data_pool);
+
+	index_mail_init_data(mail);
+
+	mail->mail.mail.seq = 0;
+	mail->mail.mail.uid = 0;
+	mail->mail.seq_pvt = 0;
+	mail->mail.mail.expunged = FALSE;
+	mail->mail.mail.has_nuls = FALSE;
+	mail->mail.mail.has_no_nuls = FALSE;
+	mail->mail.mail.saving = FALSE;
+	mail->mail.mail.mail_stream_accessed = FALSE;
+	mail->mail.mail.mail_metadata_accessed = FALSE;
+}
+
+void index_mail_close(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	if (mail->mail.mail.seq == 0) {
+		/* mail_set_seq*() hasn't been called yet, or is being called
+		   right now. Don't reset anything yet. We especially don't
+		   want to reset wanted_fields or wanted_headers so that
+		   mail_add_temp_wanted_fields() can be called by plugins
+		   before mail_set_seq_saving() for
+		   mail_save_context.dest_mail. */
+		return;
+	}
+
+	/* make sure old mail isn't visible in the event anymore even if it's
+	   attempted to be used. */
+	event_unref(&mail->mail._event);
+
+	/* If uid == 0 but seq != 0, we came here from saving a (non-mbox)
+	   message. If that happens, don't bother checking if anything should
+	   be cached since it was already checked. Also by now the transaction
+	   may have already been rollbacked and seq point to a nonexistent
+	   message. */
+	if (mail->mail.mail.uid != 0) {
+		index_mail_cache_sizes(mail);
+		index_mail_cache_dates(mail);
+	}
+
+	index_mail_close_streams_full(mail, TRUE);
+	/* Notify cache that the mail is no longer open. This mainly helps
+	   with INDEX=MEMORY to keep all data added with mail_cache_add() in
+	   memory until this point. */
+	mail_cache_close_mail(_mail->transaction->cache_trans, _mail->seq);
+
+	mailbox_header_lookup_unref(&mail->data.wanted_headers);
+	if (!mail->freeing)
+		index_mail_reset_data(mail);
+}
+
+static void check_envelope(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	const unsigned int cache_field_envelope =
+		mail->ibox->cache_fields[MAIL_CACHE_IMAP_ENVELOPE].idx;
+	unsigned int cache_field_hdr;
+
+	if ((mail->data.access_part & PARSE_HDR) != 0) {
+		mail->data.save_envelope = TRUE;
+		return;
+	}
+
+	/* if "imap.envelope" is cached, that's all we need */
+	if (mail_cache_field_exists(_mail->transaction->cache_view,
+				    _mail->seq, cache_field_envelope) > 0)
+		return;
+
+	/* don't waste time doing full checks for all required
+	   headers. assume that if we have "hdr.message-id" cached,
+	   we don't need to parse the header. */
+	cache_field_hdr = mail_cache_register_lookup(_mail->box->cache,
+						     "hdr.message-id");
+	if (cache_field_hdr == UINT_MAX ||
+	    mail_cache_field_exists(_mail->transaction->cache_view,
+				    _mail->seq, cache_field_hdr) <= 0) {
+		mail->data.access_reason_code = "mail:imap_envelope";
+		mail->data.access_part |= PARSE_HDR;
+	}
+	mail->data.save_envelope = TRUE;
+}
+
+void index_mail_update_access_parts_pre(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct mail_storage *storage = _mail->box->storage;
+	const struct mail_cache_field *cache_fields = mail->ibox->cache_fields;
+	struct mail_cache_view *cache_view = _mail->transaction->cache_view;
+	const struct mail_storage_settings *mail_set = _mail->box->storage->set;
+
+	if (_mail->seq == 0) {
+		/* mail_add_temp_wanted_fields() called before mail_set_seq*().
+		   We'll allow this, since it can be useful for plugins to
+		   call it for mail_save_context.dest_mail. This function
+		   is called again in mail_set_seq*(). */
+		return;
+	}
+
+	if ((data->wanted_fields & (MAIL_FETCH_NUL_STATE |
+				    MAIL_FETCH_IMAP_BODY |
+				    MAIL_FETCH_IMAP_BODYSTRUCTURE)) != 0 &&
+	    !_mail->has_nuls && !_mail->has_no_nuls) {
+		(void)index_mail_get_fixed_field(mail, MAIL_CACHE_FLAGS,
+						 &data->cache_flags,
+						 sizeof(data->cache_flags));
+		_mail->has_nuls =
+			(data->cache_flags & MAIL_CACHE_FLAG_HAS_NULS) != 0;
+		_mail->has_no_nuls =
+			(data->cache_flags & MAIL_CACHE_FLAG_HAS_NO_NULS) != 0;
+		/* we currently don't forcibly set the nul state. if it's not
+		   already cached, the caller can figure out itself what to
+		   do when neither is set */
+	}
+
+	/* see if wanted_fields can tell us if we need to read/parse
+	   header/body */
+	if ((data->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) != 0 &&
+	    (storage->nonbody_access_fields & MAIL_FETCH_MESSAGE_PARTS) == 0 &&
+	    data->parts == NULL) {
+		const unsigned int cache_field =
+			cache_fields[MAIL_CACHE_MESSAGE_PARTS].idx;
+
+		if (mail_cache_field_exists(cache_view, _mail->seq,
+					    cache_field) <= 0) {
+			data->access_reason_code = "mail:mime_parts";
+			data->access_part |= PARSE_HDR | PARSE_BODY;
+			data->save_message_parts = TRUE;
+		}
+	}
+
+	if ((data->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0 &&
+	    (storage->nonbody_access_fields & MAIL_FETCH_IMAP_ENVELOPE) == 0 &&
+	    data->envelope == NULL)
+		check_envelope(mail);
+
+	if ((data->wanted_fields & MAIL_FETCH_IMAP_BODY) != 0 &&
+	    (data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) == 0 &&
+	    (storage->nonbody_access_fields & MAIL_FETCH_IMAP_BODY) == 0 &&
+	    data->body == NULL) {
+		/* we need either imap.body or imap.bodystructure */
+		const unsigned int cache_field1 =
+			cache_fields[MAIL_CACHE_IMAP_BODY].idx;
+		const unsigned int cache_field2 =
+			cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
+
+		if (mail_cache_field_exists(cache_view, _mail->seq,
+					    cache_field1) <= 0 &&
+		    mail_cache_field_exists(cache_view, _mail->seq,
+					    cache_field2) <= 0) {
+			data->access_reason_code = "mail:imap_bodystructure";
+			data->access_part |= PARSE_HDR | PARSE_BODY;
+			data->save_bodystructure_header = TRUE;
+			data->save_bodystructure_body = TRUE;
+		}
+	}
+
+	if ((data->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) != 0 &&
+	    (data->cache_flags & MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII) == 0 &&
+	    (storage->nonbody_access_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) == 0 &&
+	    data->bodystructure == NULL) {
+		const unsigned int cache_field =
+			cache_fields[MAIL_CACHE_IMAP_BODYSTRUCTURE].idx;
+
+                if (mail_cache_field_exists(cache_view, _mail->seq,
+                                            cache_field) <= 0) {
+			data->access_reason_code = "mail:imap_bodystructure";
+			data->access_part |= PARSE_HDR | PARSE_BODY;
+			data->save_bodystructure_header = TRUE;
+			data->save_bodystructure_body = TRUE;
+		}
+	}
+
+	if ((data->wanted_fields & MAIL_FETCH_DATE) != 0 &&
+	    (storage->nonbody_access_fields & MAIL_FETCH_DATE) == 0 &&
+	    data->sent_date.time == (uint32_t)-1) {
+		const unsigned int cache_field =
+			cache_fields[MAIL_CACHE_SENT_DATE].idx;
+
+		if (mail_cache_field_exists(cache_view, _mail->seq,
+					    cache_field) <= 0) {
+			data->access_reason_code = "mail:date";
+			data->access_part |= PARSE_HDR;
+			data->save_sent_date = TRUE;
+		}
+	}
+	if ((data->wanted_fields & MAIL_FETCH_BODY_SNIPPET) != 0 &&
+	    (storage->nonbody_access_fields & MAIL_FETCH_BODY_SNIPPET) == 0) {
+		const unsigned int cache_field =
+			cache_fields[MAIL_CACHE_BODY_SNIPPET].idx;
+
+		if (mail_cache_field_exists(cache_view, _mail->seq,
+					    cache_field) <= 0) {
+			data->access_reason_code = "mail:snippet";
+			data->access_part |= PARSE_HDR | PARSE_BODY;
+			data->save_body_snippet = TRUE;
+		}
+	}
+	if ((data->wanted_fields & (MAIL_FETCH_STREAM_HEADER |
+				    MAIL_FETCH_STREAM_BODY)) != 0) {
+		/* Clear reason_code if set. The mail is going to be read
+		   in any case, so the previous reason for deciding to open
+		   the mail won't matter. */
+		data->access_reason_code = NULL;
+		if ((data->wanted_fields & MAIL_FETCH_STREAM_HEADER) != 0)
+			data->access_part |= READ_HDR;
+		if ((data->wanted_fields & MAIL_FETCH_STREAM_BODY) != 0)
+			data->access_part |= READ_BODY;
+	}
+
+	/* NOTE: Keep this attachment detection the last, so that the
+	   access_part check works correctly.
+
+	   The attachment flag detection is done while parsing BODYSTRUCTURE.
+	   We want to do this for mails that are being saved, but also when
+	   we need to open the mail body anyway. */
+	if (mail_set->parsed_mail_attachment_detection_add_flags &&
+	    (_mail->saving || data->access_part != 0) &&
+	    !mail_has_attachment_keywords(&mail->mail.mail)) {
+		data->save_bodystructure_header = TRUE;
+		data->save_bodystructure_body = TRUE;
+	}
+}
+
+void index_mail_update_access_parts_post(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	const struct mail_index_header *hdr;
+	struct istream *input;
+
+	if (_mail->seq == 0) {
+		/* see index_mail_update_access_parts_pre() */
+		return;
+	}
+
+	/* when mail_prefetch_count>1, at this point we've started the
+	   prefetching to all the mails and we're now starting to access the
+	   first mail. */
+
+	if (data->access_part != 0) {
+		/* open stream immediately to set expunged flag if
+		   it's already lost */
+
+		/* open the stream only if we didn't get here from
+		   mailbox_save_init() */
+		hdr = mail_index_get_header(_mail->transaction->view);
+		if (!_mail->saving && _mail->uid < hdr->next_uid) {
+			if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0)
+				(void)mail_get_stream_because(_mail, NULL, NULL, "access", &input);
+			else
+				(void)mail_get_hdr_stream(_mail, NULL, &input);
+		}
+	}
+}
+
+void index_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	const struct mail_index_record *rec;
+	struct mail_index_map *map;
+	bool expunged;
+
+	if (mail->mail.mail.seq == seq) {
+		if (!saving)
+			return;
+		/* we started saving a mail, aborted it, and now we're saving
+		   another mail with the same sequence. make sure the mail
+		   gets reset. */
+	}
+
+	mail->mail.v.close(&mail->mail.mail);
+
+	mail->mail.mail.seq = seq;
+	mail->mail.mail.saving = saving;
+
+	rec = mail_index_lookup_full(_mail->transaction->view, seq,
+				     &map, &expunged);
+	mail->mail.mail.uid = rec->uid;
+
+	/* Recreate the mail event when changing mails. Even though the same
+	   mail struct is reused, they are practically different mails. The
+	   event should have already been freed by close(). */
+	i_assert(mail->mail._event == NULL);
+
+	if (mail_index_view_is_inconsistent(_mail->transaction->view)) {
+		mail_set_expunged(&mail->mail.mail);
+		return;
+	}
+	/* Allow callers to easily find out if this mail was already expunged
+	   by another session. It's possible that it could still be
+	   successfully accessed. */
+	if (expunged)
+		mail_set_expunged(&mail->mail.mail);
+
+	if (!mail->mail.search_mail) {
+		index_mail_update_access_parts_pre(_mail);
+		index_mail_update_access_parts_post(_mail);
+	} else {
+		/* searching code will call the
+		   index_mail_update_access_parts_*() after we know the mail is
+		   actually wanted to be fetched. */
+	}
+	mail->data.initialized = TRUE;
+}
+
+bool index_mail_prefetch(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */
+#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
+	struct mail_storage *storage = _mail->box->storage;
+	struct istream *input;
+	off_t len;
+	int fd;
+
+	if ((storage->class_flags & MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG) == 0) {
+		/* we're handling only file-per-msg storages for now. */
+		return TRUE;
+	}
+	if (mail->data.access_part == 0) {
+		/* everything we need is cached */
+		return TRUE;
+	}
+
+	if (mail->data.stream == NULL) {
+		(void)mail_get_stream_because(_mail, NULL, NULL, "prefetch", &input);
+		if (mail->data.stream == NULL)
+			return TRUE;
+	}
+
+	/* tell OS to start reading the file into memory */
+	fd = i_stream_get_fd(mail->data.stream);
+	if (fd != -1) {
+		if ((mail->data.access_part & (READ_BODY | PARSE_BODY)) != 0)
+			len = 0;
+		else
+			len = MAIL_READ_HDR_BLOCK_SIZE;
+		if (posix_fadvise(fd, 0, len, POSIX_FADV_WILLNEED) < 0) {
+			i_error("posix_fadvise(%s) failed: %m",
+				i_stream_get_name(mail->data.stream));
+		}
+		mail->data.prefetch_sent = TRUE;
+	}
+#endif
+	return !mail->data.prefetch_sent;
+}
+
+bool index_mail_set_uid(struct mail *_mail, uint32_t uid)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	uint32_t seq;
+
+	if (mail_index_lookup_seq(_mail->transaction->view, uid, &seq)) {
+		index_mail_set_seq(_mail, seq, FALSE);
+		return TRUE;
+	} else {
+		mail->mail.v.close(&mail->mail.mail);
+		mail->mail.mail.uid = uid;
+		mail_set_expunged(&mail->mail.mail);
+		return FALSE;
+	}
+}
+
+void index_mail_add_temp_wanted_fields(struct mail *_mail,
+				       enum mail_fetch_field fields,
+				       struct mailbox_header_lookup_ctx *headers)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct mailbox_header_lookup_ctx *new_wanted_headers;
+
+	data->wanted_fields |= fields;
+	if (headers == NULL) {
+		/* keep old ones */
+	} else if (data->wanted_headers == NULL) {
+		data->wanted_headers = headers;
+		mailbox_header_lookup_ref(headers);
+	} else {
+		/* merge headers */
+		new_wanted_headers = mailbox_header_lookup_merge(data->wanted_headers,
+								 headers);
+		mailbox_header_lookup_unref(&data->wanted_headers);
+		data->wanted_headers = new_wanted_headers;
+	}
+	index_mail_update_access_parts_pre(_mail);
+	/* Don't call _post(), which would try to open the stream. It should be
+	   enough to delay the opening until it happens anyway.
+
+	   Otherwise there's not really any good place to call this in the
+	   plugins: set_seq() call get_stream() internally, which can already
+	   start parsing the headers, so it's too late. If we use get_stream()
+	   and there's a _post() call here, it gets into infinite loop. The
+	   loop could probably be prevented in some way, but it's probably
+	   better to eventually try to remove the _post() call entirely
+	   everywhere. */
+}
+
+void index_mail_set_uid_cache_updates(struct mail *_mail, bool set)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	mail->data.no_caching = set || mail->data.forced_no_caching;
+}
+
+void index_mail_free(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	mail->freeing = TRUE;
+	mail->mail.v.close(_mail);
+
+	i_assert(_mail->transaction->mail_ref_count > 0);
+	_mail->transaction->mail_ref_count--;
+
+	buffer_free(&mail->header_data);
+	if (array_is_created(&mail->header_lines))
+		array_free(&mail->header_lines);
+	if (array_is_created(&mail->header_match))
+		array_free(&mail->header_match);
+	if (array_is_created(&mail->header_match_lines))
+		array_free(&mail->header_match_lines);
+
+	mailbox_header_lookup_unref(&mail->data.wanted_headers);
+	mailbox_header_lookup_unref(&mail->mail.wanted_headers);
+	event_unref(&mail->mail._event);
+	pool_unref(&mail->mail.data_pool);
+	pool_unref(&mail->mail.pool);
+}
+
+void index_mail_cache_parse_continue(struct mail *_mail)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct message_block block;
+
+	while (message_parser_parse_next_block(mail->data.parser_ctx,
+					       &block) > 0) {
+		if (block.size != 0)
+			continue;
+
+		if (!mail->data.header_parsed) {
+			index_mail_parse_header(block.part, block.hdr, mail);
+			if (block.hdr == NULL)
+				mail->data.header_parsed = TRUE;
+		} else {
+			message_part_data_parse_from_header(mail->mail.data_pool,
+							block.part, block.hdr);
+		}
+	}
+}
+
+void index_mail_cache_parse_deinit(struct mail *_mail, time_t received_date,
+				   bool success)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	if (!success) {
+		/* we're going to delete this mail anyway,
+		   don't bother trying to update cache file */
+		mail->data.no_caching = TRUE;
+		mail->data.forced_no_caching = TRUE;
+
+		if (mail->data.parser_ctx == NULL) {
+			/* we didn't even start cache parsing */
+			i_assert(!mail->data.header_parser_initialized);
+			return;
+		}
+	}
+
+	/* This is needed with 0 byte mails to get hdr=NULL call done. */
+	index_mail_cache_parse_continue(_mail);
+
+	if (mail->data.received_date == (time_t)-1)
+		mail->data.received_date = received_date;
+	if (mail->data.save_date == (time_t)-1) {
+		/* this save_date may not be exactly the same as what we get
+		   in future, but then again neither mbox nor maildir
+		   guarantees it anyway. */
+		mail->data.save_date = ioloop_time;
+	}
+
+	(void)index_mail_parse_body_finish(mail, 0, success);
+}
+
+static bool
+index_mail_update_pvt_flags(struct mail *_mail, enum modify_type modify_type,
+			    enum mail_flags pvt_flags)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	const struct mail_index_record *rec;
+	enum mail_flags old_pvt_flags;
+
+	if (!index_mail_get_pvt(_mail))
+		return FALSE;
+	if (pvt_flags == 0 && modify_type != MODIFY_REPLACE)
+		return FALSE;
+
+	/* see if the flags actually change anything */
+	rec = mail_index_lookup(_mail->transaction->view_pvt, mail->seq_pvt);
+	old_pvt_flags = rec->flags & mailbox_get_private_flags_mask(_mail->box);
+
+	switch (modify_type) {
+	case MODIFY_ADD:
+		return (old_pvt_flags & pvt_flags) != pvt_flags;
+	case MODIFY_REPLACE:
+		return old_pvt_flags != pvt_flags;
+	case MODIFY_REMOVE:
+		return (old_pvt_flags & pvt_flags) != 0;
+	}
+	i_unreached();
+}
+
+void index_mail_update_flags(struct mail *_mail, enum modify_type modify_type,
+			     enum mail_flags flags)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	enum mail_flags pvt_flags_mask, pvt_flags = 0;
+	bool update_modseq = FALSE;
+
+	flags &= MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND;
+
+	if (_mail->box->view_pvt != NULL) {
+		/* mailbox has private flags */
+		pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box);
+		pvt_flags = flags & pvt_flags_mask;
+		flags &= ENUM_NEGATE(pvt_flags_mask);
+		if (index_mail_update_pvt_flags(_mail, modify_type, pvt_flags)) {
+			mail_index_update_flags(_mail->transaction->itrans_pvt,
+						mail->seq_pvt,
+						modify_type, pvt_flags);
+			update_modseq = TRUE;
+		}
+	}
+
+	if (!update_modseq) {
+		/* no forced modseq update */
+	} else if (modify_type == MODIFY_REMOVE) {
+		/* add the modseq update separately */
+		mail_index_update_flags(_mail->transaction->itrans, _mail->seq,
+			MODIFY_ADD, (enum mail_flags )MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ);
+	} else {
+		/* add as part of the flag updates */
+		flags |= MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ;
+	}
+	mail_index_update_flags(_mail->transaction->itrans, _mail->seq,
+				modify_type, flags);
+}
+
+void index_mail_update_keywords(struct mail *mail, enum modify_type modify_type,
+				struct mail_keywords *keywords)
+{
+	struct index_mail *imail = INDEX_MAIL(mail);
+
+	if (array_is_created(&imail->data.keyword_indexes))
+		array_free(&imail->data.keyword_indexes);
+	if (array_is_created(&imail->data.keywords)) {
+		/* clear the keywords array so the next mail_get_keywords()
+		   returns the updated keywords. don't free the array, because
+		   then any existing mail_get_keywords() return values would
+		   point to broken data. this won't leak memory because the
+		   array is allocated from mail's memory pool. */
+		memset(&imail->data.keywords, 0,
+		       sizeof(imail->data.keywords));
+	}
+
+	mail_index_update_keywords(mail->transaction->itrans, mail->seq,
+				   modify_type, keywords);
+}
+
+void index_mail_update_modseq(struct mail *mail, uint64_t min_modseq)
+{
+	mail_index_update_modseq(mail->transaction->itrans, mail->seq,
+				 min_modseq);
+}
+
+void index_mail_update_pvt_modseq(struct mail *mail, uint64_t min_pvt_modseq)
+{
+	if (mail->box->view_pvt == NULL)
+		return;
+	index_transaction_init_pvt(mail->transaction);
+	mail_index_update_modseq(mail->transaction->itrans_pvt, mail->seq,
+				 min_pvt_modseq);
+}
+
+void index_mail_expunge(struct mail *mail)
+{
+	enum mail_lookup_abort old_abort = mail->lookup_abort;
+	const char *value;
+	guid_128_t guid_128;
+
+	mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+	if (mail_get_special(mail, MAIL_FETCH_GUID, &value) < 0)
+		mail_index_expunge(mail->transaction->itrans, mail->seq);
+	else {
+		mail_generate_guid_128_hash(value, guid_128);
+		mail_index_expunge_guid(mail->transaction->itrans,
+					mail->seq, guid_128);
+	}
+	mail->lookup_abort = old_abort;
+}
+
+static void index_mail_parse(struct mail *mail, bool parse_body)
+{
+	struct index_mail *imail = INDEX_MAIL(mail);
+
+	imail->data.access_part |= PARSE_HDR;
+	if (index_mail_parse_headers(imail, NULL, "precache") == 0) {
+		if (parse_body) {
+			imail->data.access_part |= PARSE_BODY;
+			(void)index_mail_parse_body(imail, 0);
+		}
+	}
+}
+
+int index_mail_precache(struct mail *mail)
+{
+	struct index_mail *imail = INDEX_MAIL(mail);
+	enum mail_fetch_field cache;
+	time_t date;
+	uoff_t size;
+	const char *str;
+
+	if (mail_cache_field_exists_any(mail->transaction->cache_view,
+					mail->seq)) {
+		/* already cached this mail (we should get here only if FTS
+		   plugin decreased the first precached seq) */
+		return 0;
+	}
+
+	cache = imail->data.wanted_fields;
+	if ((cache & (MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY)) != 0)
+		index_mail_parse(mail, (cache & MAIL_FETCH_STREAM_BODY) != 0);
+	if ((cache & MAIL_FETCH_RECEIVED_DATE) != 0)
+		(void)mail_get_received_date(mail, &date);
+	if ((cache & MAIL_FETCH_SAVE_DATE) != 0)
+		(void)mail_get_save_date(mail, &date);
+	if ((cache & MAIL_FETCH_VIRTUAL_SIZE) != 0)
+		(void)mail_get_virtual_size(mail, &size);
+	if ((cache & MAIL_FETCH_PHYSICAL_SIZE) != 0)
+		(void)mail_get_physical_size(mail, &size);
+	if ((cache & MAIL_FETCH_UIDL_BACKEND) != 0)
+		(void)mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &str);
+	if ((cache & MAIL_FETCH_POP3_ORDER) != 0)
+		(void)mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str);
+	if ((cache & MAIL_FETCH_GUID) != 0)
+		(void)mail_get_special(mail, MAIL_FETCH_GUID, &str);
+	return 0;
+}
+
+static void
+index_mail_reset_vsize_ext(struct mail *mail)
+{
+	unsigned int idx;
+	uint32_t vsize = 0;
+	struct mail_index_view *view = mail->transaction->view;
+	if (mail_index_map_get_ext_idx(view->map, mail->box->mail_vsize_ext_id,
+				       &idx)) {
+		mail_index_update_ext(mail->transaction->itrans, mail->seq,
+				      mail->box->mail_vsize_ext_id, &vsize, NULL);
+	}
+}
+
+void index_mail_set_cache_corrupted(struct mail *mail,
+				    enum mail_fetch_field field,
+				    const char *reason)
+{
+	struct index_mail *imail = INDEX_MAIL(mail);
+	const char *field_name;
+
+	switch ((int)field) {
+	case 0:
+		field_name = "fields";
+		break;
+	case MAIL_FETCH_PHYSICAL_SIZE:
+		field_name = "physical size";
+		imail->data.physical_size = UOFF_T_MAX;
+		imail->data.virtual_size = UOFF_T_MAX;
+		index_mail_parts_reset(imail);
+		index_mail_reset_vsize_ext(mail);
+		break;
+	case MAIL_FETCH_VIRTUAL_SIZE:
+		field_name = "virtual size";
+		imail->data.physical_size = UOFF_T_MAX;
+		imail->data.virtual_size = UOFF_T_MAX;
+		index_mail_parts_reset(imail);
+		index_mail_reset_vsize_ext(mail);
+		break;
+	case MAIL_FETCH_MESSAGE_PARTS:
+		field_name = "MIME parts";
+		index_mail_parts_reset(imail);
+		break;
+	case MAIL_FETCH_IMAP_BODY:
+		field_name = "IMAP BODY";
+		imail->data.body = NULL;
+		imail->data.bodystructure = NULL;
+		break;
+	case MAIL_FETCH_IMAP_BODYSTRUCTURE:
+		field_name = "IMAP BODYSTRUCTURE";
+		imail->data.body = NULL;
+		imail->data.bodystructure = NULL;
+		break;
+	default:
+		field_name = t_strdup_printf("#%x", field);
+	}
+
+	/* make sure we don't cache invalid values */
+	mail_cache_transaction_reset(mail->transaction->cache_trans);
+	imail->data.no_caching = TRUE;
+	imail->data.forced_no_caching = TRUE;
+
+	if (mail->saving) {
+		mail_set_critical(mail,
+			"BUG: Broken %s found while saving a new mail: %s",
+			field_name, reason);
+	} else if (reason[0] == '\0') {
+		mail_set_mail_cache_corrupted(mail,
+			"Broken %s in mailbox %s",
+			field_name, mail->box->vname);
+	} else {
+		mail_set_mail_cache_corrupted(mail,
+			"Broken %s in mailbox %s: %s",
+			field_name, mail->box->vname, reason);
+	}
+}
+
+int index_mail_opened(struct mail *mail,
+		      struct istream **stream ATTR_UNUSED)
+{
+	struct index_mail *imail =
+		container_of(mail, struct index_mail, mail.mail);
+	struct event_reason *reason = NULL;
+
+	if (imail->data.access_reason_code != NULL)
+		reason = event_reason_begin(imail->data.access_reason_code);
+	mail_opened_event(mail);
+	event_reason_end(&reason);
+	return 0;
+}
+
+void index_mail_save_finish(struct mail_save_context *ctx)
+{
+	struct index_mail *imail = INDEX_MAIL(ctx->dest_mail);
+
+	index_mail_save_finish_make_snippet(imail);
+
+	if (ctx->data.from_envelope != NULL &&
+	    imail->data.from_envelope == NULL) {
+		imail->data.from_envelope =
+			p_strdup(imail->mail.data_pool, ctx->data.from_envelope);
+	}
+}
+
+const char *index_mail_cache_reason(struct mail *mail, const char *reason)
+{
+	const char *cache_reason =
+		mail_cache_get_missing_reason(mail->transaction->cache_view, mail->seq);
+	return t_strdup_printf("%s (%s)", reason, cache_reason);
+}
diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
new file mode 100644
index 0000000..74107d2
--- /dev/null
+++ b/src/lib-storage/index/index-mail.h
@@ -0,0 +1,292 @@
+#ifndef INDEX_MAIL_H
+#define INDEX_MAIL_H
+
+#include "message-size.h"
+#include "mail-cache.h"
+#include "mail-storage-private.h"
+
+enum index_cache_field {
+	/* fixed size fields */
+	MAIL_CACHE_FLAGS = 0,
+	MAIL_CACHE_SENT_DATE,
+	MAIL_CACHE_RECEIVED_DATE,
+	MAIL_CACHE_SAVE_DATE,
+	MAIL_CACHE_VIRTUAL_FULL_SIZE,
+	MAIL_CACHE_PHYSICAL_FULL_SIZE,
+
+	/* variable sized field */
+	MAIL_CACHE_IMAP_BODY,
+	MAIL_CACHE_IMAP_BODYSTRUCTURE,
+	MAIL_CACHE_IMAP_ENVELOPE,
+	MAIL_CACHE_POP3_UIDL,
+	MAIL_CACHE_POP3_ORDER,
+	MAIL_CACHE_GUID,
+	MAIL_CACHE_MESSAGE_PARTS,
+	MAIL_CACHE_BINARY_PARTS,
+	MAIL_CACHE_BODY_SNIPPET,
+
+	MAIL_INDEX_CACHE_FIELD_COUNT
+};
+extern struct mail_cache_field
+	global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT];
+
+#define IMAP_BODY_PLAIN_7BIT_ASCII \
+	"\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\""
+
+enum mail_cache_record_flag {
+	/* If binary flags are set, it's not checked whether mail is
+	   missing CRs. So this flag may be set as an optimization for
+	   regular non-binary mails as well if it's known that it contains
+	   valid CR+LF line breaks. */
+	MAIL_CACHE_FLAG_BINARY_HEADER		= 0x0001,
+	MAIL_CACHE_FLAG_BINARY_BODY		= 0x0002,
+
+	/* Mail header or body is known to contain NUL characters. */
+	MAIL_CACHE_FLAG_HAS_NULS		= 0x0004,
+	/* Mail header or body is known to not contain NUL characters. */
+	MAIL_CACHE_FLAG_HAS_NO_NULS		= 0x0020,
+	/* obsolete _HAS_NO_NULS flag, which was being set incorrectly */
+	MAIL_CACHE_FLAG_HAS_NO_NULS_BROKEN	= 0x0008,
+
+	/* BODY is IMAP_BODY_PLAIN_7BIT_ASCII and rest of BODYSTRUCTURE
+	   fields are NIL */
+	MAIL_CACHE_FLAG_TEXT_PLAIN_7BIT_ASCII	= 0x0010
+};
+
+enum index_mail_access_part {
+	READ_HDR	= 0x01,
+	READ_BODY	= 0x02,
+	PARSE_HDR	= 0x04,
+	PARSE_BODY	= 0x08
+};
+
+struct mail_sent_date {
+	uint32_t time;
+	int32_t timezone;
+};
+
+struct index_mail_line {
+	unsigned int field_idx;
+	uint32_t start_pos, end_pos;
+	uint32_t line_num;
+};
+
+struct message_header_line;
+
+struct index_mail_data {
+	time_t date, received_date, save_date;
+	uoff_t virtual_size, physical_size;
+
+	struct mail_sent_date sent_date;
+	struct index_mail_line parse_line;
+	uint32_t parse_line_num;
+
+	struct message_part *parts;
+	struct message_binary_part *bin_parts;
+	const char *envelope, *body, *bodystructure, *guid, *filename;
+	const char *from_envelope, *body_snippet;
+	struct message_part_envelope *envelope_data;
+
+	uint32_t cache_flags;
+	uint64_t modseq, pvt_modseq;
+	enum index_mail_access_part access_part;
+	const char *access_reason_code;
+	/* dont_cache_fields overrides cache_fields */
+	enum mail_fetch_field cache_fetch_fields, dont_cache_fetch_fields;
+	unsigned int dont_cache_field_idx;
+	enum mail_fetch_field wanted_fields;
+	struct mailbox_header_lookup_ctx *wanted_headers;
+
+	buffer_t *search_results;
+
+	struct istream *stream, *filter_stream;
+	struct tee_istream *tee_stream;
+	struct message_size hdr_size, body_size;
+	struct istream *parser_input;
+	struct message_parser_ctx *parser_ctx;
+	int parsing_count;
+	ARRAY_TYPE(keywords) keywords;
+	ARRAY_TYPE(keyword_indexes) keyword_indexes;
+
+	bool initialized:1;
+	bool save_sent_date:1;
+	bool sent_date_parsed:1;
+	bool save_envelope:1;
+	bool save_bodystructure_header:1;
+	bool save_bodystructure_body:1;
+	bool save_message_parts:1;
+	bool save_body_snippet:1;
+	bool stream_has_only_header:1;
+	bool parsed_bodystructure:1;
+	bool parsed_bodystructure_header:1;
+	bool hdr_size_set:1;
+	bool body_size_set:1;
+	bool messageparts_saved_to_cache:1;
+	bool header_parsed:1;
+	bool no_caching:1;
+	bool forced_no_caching:1;
+	bool istream_error_logged:1;
+	bool destroying_stream:1;
+	bool initialized_wrapper_stream:1;
+	bool destroy_callback_set:1;
+	bool prefetch_sent:1;
+	bool header_parser_initialized:1;
+	/* virtual_size and physical_size may not match the stream size.
+	   Try to avoid trusting them too much. */
+	bool inexact_total_sizes:1;
+};
+
+struct index_mail {
+        struct mail_private mail;
+	struct index_mail_data data;
+	struct index_mailbox_context *ibox;
+
+	int pop3_state;
+
+	/* per-mail variables, here for performance reasons: */
+	uint32_t header_seq;
+	string_t *header_data;
+	ARRAY(struct index_mail_line) header_lines;
+#define HEADER_MATCH_FLAG_FOUND 1
+#define HEADER_MATCH_SKIP_COUNT 2
+#define HEADER_MATCH_USABLE(mail, num) \
+	((num & ~1U) == (mail)->header_match_value)
+	ARRAY(uint8_t) header_match;
+	ARRAY(unsigned int) header_match_lines;
+	uint8_t header_match_value;
+
+	bool pop3_state_set:1;
+	/* close() is being called from mail_free() */
+	bool freeing:1;
+};
+
+#define INDEX_MAIL(s)	container_of(s, struct index_mail, mail.mail)
+
+struct mail *
+index_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers);
+void index_mail_init(struct index_mail *mail,
+		     struct mailbox_transaction_context *_t,
+		     enum mail_fetch_field wanted_fields,
+		     struct mailbox_header_lookup_ctx *_wanted_headers,
+		     struct pool *mail_pool,
+		     struct pool *data_pool);
+
+void index_mail_set_seq(struct mail *mail, uint32_t seq, bool saving);
+bool index_mail_set_uid(struct mail *mail, uint32_t uid);
+void index_mail_set_uid_cache_updates(struct mail *mail, bool set);
+bool index_mail_prefetch(struct mail *mail);
+void index_mail_add_temp_wanted_fields(struct mail *mail,
+				       enum mail_fetch_field fields,
+				       struct mailbox_header_lookup_ctx *headers);
+void index_mail_update_access_parts_pre(struct mail *mail);
+void index_mail_update_access_parts_post(struct mail *_mail);
+void index_mail_close(struct mail *mail);
+void index_mail_close_streams(struct index_mail *mail);
+void index_mail_free(struct mail *mail);
+void index_mail_set_message_parts_corrupted(struct mail *mail, const char *error);
+
+bool index_mail_want_parse_headers(struct index_mail *mail);
+void index_mail_parse_header_init(struct index_mail *mail,
+				  struct mailbox_header_lookup_ctx *headers)
+	ATTR_NULL(2);
+void index_mail_parse_header(struct message_part *part,
+			     struct message_header_line *hdr,
+			     struct index_mail *mail) ATTR_NULL(1);
+int index_mail_parse_headers(struct index_mail *mail,
+			     struct mailbox_header_lookup_ctx *headers,
+			     const char *reason)
+	ATTR_NULL(2);
+void index_mail_parse_header_deinit(struct index_mail *mail);
+/* Same as index_mail_parse_headers(), but assume that the stream is
+   already opened. */
+int index_mail_parse_headers_internal(struct index_mail *mail,
+				      struct mailbox_header_lookup_ctx *headers)
+	ATTR_NULL(2);
+int index_mail_headers_get_envelope(struct index_mail *mail);
+void index_mail_parts_reset(struct index_mail *mail);
+
+int index_mail_get_first_header(struct mail *_mail, const char *field,
+				bool decode_to_utf8, const char **value_r);
+int index_mail_get_headers(struct mail *_mail, const char *field,
+			   bool decode_to_utf8, const char *const **value_r);
+int index_mail_get_header_stream(struct mail *_mail,
+				 struct mailbox_header_lookup_ctx *headers,
+				 struct istream **stream_r);
+void index_mail_set_read_buffer_size(struct mail *mail, struct istream *input);
+
+enum mail_flags index_mail_get_flags(struct mail *_mail);
+uint64_t index_mail_get_modseq(struct mail *_mail);
+uint64_t index_mail_get_pvt_modseq(struct mail *_mail);
+const char *const *index_mail_get_keywords(struct mail *_mail);
+const ARRAY_TYPE(keyword_indexes) *
+index_mail_get_keyword_indexes(struct mail *_mail);
+int index_mail_get_parts(struct mail *_mail, struct message_part **parts_r);
+int index_mail_get_received_date(struct mail *_mail, time_t *date_r);
+int index_mail_get_save_date(struct mail *_mail, time_t *date_r);
+int index_mail_get_date(struct mail *_mail, time_t *date_r, int *timezone_r);
+int index_mail_get_virtual_size(struct mail *mail, uoff_t *size_r);
+int index_mail_get_physical_size(struct mail *mail, uoff_t *size_r);
+int index_mail_init_stream(struct index_mail *mail,
+			   struct message_size *hdr_size,
+			   struct message_size *body_size,
+			   struct istream **stream_r) ATTR_NULL(2, 3);
+int index_mail_get_binary_stream(struct mail *_mail,
+				 const struct message_part *part,
+				 bool include_hdr, uoff_t *size_r,
+				 unsigned int *body_lines_r, bool *binary_r,
+				 struct istream **stream_r);
+int index_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+			   const char **value_r);
+int index_mail_get_backend_mail(struct mail *mail, struct mail **real_mail_r);
+
+void index_mail_update_flags(struct mail *mail, enum modify_type modify_type,
+			     enum mail_flags flags);
+void index_mail_update_keywords(struct mail *mail, enum modify_type modify_type,
+				struct mail_keywords *keywords);
+void index_mail_update_modseq(struct mail *mail, uint64_t min_modseq);
+void index_mail_update_pvt_modseq(struct mail *mail, uint64_t min_pvt_modseq);
+void index_mail_expunge(struct mail *mail);
+int index_mail_precache(struct mail *mail);
+void index_mail_set_cache_corrupted(struct mail *mail,
+				    enum mail_fetch_field field,
+				    const char *reason);
+int index_mail_opened(struct mail *mail, struct istream **stream);
+int index_mail_stream_check_failure(struct index_mail *mail);
+void index_mail_stream_log_failure_for(struct index_mail *mail,
+				       struct istream *input);
+void index_mail_refresh_expunged(struct mail *mail);
+struct index_mail *index_mail_get_index_mail(struct mail *mail);
+
+bool index_mail_get_cached_uoff_t(struct index_mail *mail,
+				  enum index_cache_field field, uoff_t *size_r);
+bool index_mail_get_cached_virtual_size(struct index_mail *mail,
+					uoff_t *size_r);
+bool index_mail_get_cached_body(struct index_mail *mail, const char **value_r);
+bool index_mail_get_cached_bodystructure(struct index_mail *mail,
+					 const char **value_r);
+const uint32_t *index_mail_get_vsize_extension(struct mail *_mail);
+
+bool index_mail_want_cache(struct index_mail *mail, enum index_cache_field field);
+void index_mail_cache_add(struct index_mail *mail, enum index_cache_field field,
+			  const void *data, size_t data_size);
+void index_mail_cache_add_idx(struct index_mail *mail, unsigned int field_idx,
+			      const void *data, size_t data_size);
+
+void index_mail_cache_pop3_data(struct mail *_mail,
+				const char *uidl, uint32_t order);
+
+struct istream *index_mail_cache_parse_init(struct mail *mail,
+					    struct istream *input);
+void index_mail_cache_parse_continue(struct mail *mail);
+void index_mail_cache_parse_deinit(struct mail *mail, time_t received_date,
+				   bool success);
+
+int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf,
+				  unsigned int field_idx);
+void index_mail_save_finish(struct mail_save_context *ctx);
+
+const char *index_mail_cache_reason(struct mail *mail, const char *reason);
+
+#endif
diff --git a/src/lib-storage/index/index-mailbox-size.c b/src/lib-storage/index/index-mailbox-size.c
new file mode 100644
index 0000000..35b1081
--- /dev/null
+++ b/src/lib-storage/index/index-mailbox-size.c
@@ -0,0 +1,502 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "net.h"
+#include "write-full.h"
+#include "mail-search-build.h"
+#include "index-storage.h"
+#include "index-mailbox-size.h"
+
+/*
+   Saving new mails: After transaction is committed and synced, trigger
+   vsize updating. Lock vsize updates. Check if the message count +
+   last-indexed-uid are still valid. If they are, add all the missing new
+   mails. Unlock.
+
+   Fetching vsize: Lock vsize updates. Check if the message count +
+   last-indexed-uid are still valid. If not, set them to zero. Add all
+   the missing mails. Unlock.
+
+   Expunging mails: Check if syncing would expunge any mails. If so, lock the
+   vsize updates before locking syncing (to avoid deadlocks). Check if the
+   message count + last-indexed-uid are still valid. If not, unlock vsize and
+   do nothing else. Otherwise, for each expunged mail whose UID <=
+   last-indexed-uid, decrease the message count and the vsize in memory. After
+   syncing is successfully committed, write the changes to header. Unlock.
+
+   Note that the final expunge handling with some mailbox formats is done while
+   syncing is no longer locked. Because of this we need to have the vsize
+   locking. The final vsize header update requires committing a transaction,
+   which internally is the same as a sync lock. So to avoid deadlocks we always
+   need to lock vsize updates before sync.
+*/
+
+#define VSIZE_LOCK_SUFFIX "dovecot-vsize.lock"
+#define VSIZE_UPDATE_MAX_LOCK_SECS 10
+
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
+struct mailbox_vsize_update {
+	struct mailbox *box;
+	struct mail_index_view *view;
+	struct mailbox_index_vsize vsize_hdr, orig_vsize_hdr;
+
+	struct file_lock *lock;
+	bool lock_failed;
+	bool skip_write;
+	bool rebuild;
+	bool written;
+	bool finish_in_background;
+};
+
+static void vsize_header_refresh(struct mailbox_vsize_update *update)
+{
+	const void *data;
+	size_t size;
+
+	if (update->view != NULL)
+		mail_index_view_close(&update->view);
+	(void)mail_index_refresh(update->box->index);
+	update->view = mail_index_view_open(update->box->index);
+
+	mail_index_get_header_ext(update->view, update->box->vsize_hdr_ext_id,
+				  &data, &size);
+	if (size > 0) {
+		memcpy(&update->orig_vsize_hdr, data,
+		       I_MIN(size, sizeof(update->orig_vsize_hdr)));
+	}
+	if (size == sizeof(update->vsize_hdr))
+		memcpy(&update->vsize_hdr, data, sizeof(update->vsize_hdr));
+	else {
+		if (size != 0) {
+			mailbox_set_critical(update->box,
+				"vsize-hdr has invalid size: %zu",
+				size);
+		}
+		update->rebuild = TRUE;
+		i_zero(&update->vsize_hdr);
+	}
+}
+
+static void
+index_mailbox_vsize_check_rebuild(struct mailbox_vsize_update *update)
+{
+	uint32_t seq1, seq2;
+
+	if (update->vsize_hdr.highest_uid == 0)
+		return;
+	if (!mail_index_lookup_seq_range(update->view, 1,
+					 update->vsize_hdr.highest_uid,
+					 &seq1, &seq2))
+		seq2 = 0;
+
+	if (update->vsize_hdr.message_count != seq2) {
+		if (update->vsize_hdr.message_count < seq2) {
+			mailbox_set_critical(update->box,
+				"vsize-hdr has invalid message-count (%u < %u)",
+				update->vsize_hdr.message_count, seq2);
+		} else {
+			/* some messages have been expunged, rescan */
+		}
+		i_zero(&update->vsize_hdr);
+		update->rebuild = TRUE;
+	}
+}
+
+struct mailbox_vsize_update *
+index_mailbox_vsize_update_init(struct mailbox *box)
+{
+	struct mailbox_vsize_update *update;
+
+	i_assert(box->opened);
+
+	update = i_new(struct mailbox_vsize_update, 1);
+	update->box = box;
+
+	vsize_header_refresh(update);
+	return update;
+}
+
+static bool vsize_update_lock_full(struct mailbox_vsize_update *update,
+				   unsigned int lock_secs)
+{
+	struct mailbox *box = update->box;
+	const char *error;
+	int ret;
+
+	if (update->lock != NULL)
+		return TRUE;
+	if (update->lock_failed)
+		return FALSE;
+	if (MAIL_INDEX_IS_IN_MEMORY(box->index))
+		return FALSE;
+
+	ret = mailbox_lock_file_create(box, VSIZE_LOCK_SUFFIX, lock_secs,
+				       &update->lock, &error);
+	if (ret <= 0) {
+		/* don't log lock timeouts, because we're somewhat expecting
+		   them. Especially when lock_secs is 0. */
+		if (ret < 0)
+			mailbox_set_critical(box, "%s", error);
+		update->lock_failed = TRUE;
+		return FALSE;
+	}
+	update->rebuild = FALSE;
+	vsize_header_refresh(update);
+	index_mailbox_vsize_check_rebuild(update);
+	return TRUE;
+}
+
+bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update)
+{
+	return vsize_update_lock_full(update, 0);
+}
+
+bool index_mailbox_vsize_update_wait_lock(struct mailbox_vsize_update *update)
+{
+	return vsize_update_lock_full(update, VSIZE_UPDATE_MAX_LOCK_SECS);
+}
+
+bool index_mailbox_vsize_want_updates(struct mailbox_vsize_update *update)
+{
+	return update->vsize_hdr.highest_uid > 0;
+}
+
+static void
+index_mailbox_vsize_update_write_to_index(struct mailbox_vsize_update *update)
+{
+	struct mail_index_transaction *trans;
+
+	trans = mail_index_transaction_begin(update->view,
+				MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+	mail_index_update_header_ext(trans, update->box->vsize_hdr_ext_id,
+				     0, &update->vsize_hdr,
+				     sizeof(update->vsize_hdr));
+	(void)mail_index_transaction_commit(&trans);
+}
+
+static void
+index_mailbox_vsize_update_write(struct mailbox_vsize_update *update)
+{
+	if (update->written)
+		return;
+	update->written = TRUE;
+
+	if (update->rebuild == FALSE &&
+	    memcmp(&update->orig_vsize_hdr, &update->vsize_hdr,
+		   sizeof(update->vsize_hdr)) == 0) {
+		/* no changes */
+		return;
+	}
+	index_mailbox_vsize_update_write_to_index(update);
+}
+
+static void index_mailbox_vsize_notify_indexer(struct mailbox *box)
+{
+	string_t *str = t_str_new(256);
+	const char *path;
+	int fd;
+
+	path = t_strconcat(box->storage->user->set->base_dir,
+			   "/"INDEXER_SOCKET_NAME, NULL);
+	fd = net_connect_unix(path);
+	if (fd == -1) {
+		mailbox_set_critical(box,
+			"Can't start vsize building on background: "
+			"net_connect_unix(%s) failed: %m", path);
+		return;
+	}
+	str_append(str, INDEXER_HANDSHAKE);
+	str_append(str, "APPEND\t0\t");
+	str_append_tabescaped(str, box->storage->user->username);
+	str_append_c(str, '\t');
+	str_append_tabescaped(str, box->vname);
+	str_append_c(str, '\n');
+
+	if (write_full(fd, str_data(str), str_len(str)) < 0) {
+		mailbox_set_critical(box,
+			"Can't start vsize building on background: "
+			"write(%s) failed: %m", path);
+	}
+	i_close_fd(&fd);
+}
+
+void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
+{
+	struct mailbox_vsize_update *update = *_update;
+
+	*_update = NULL;
+
+	if ((update->lock != NULL || update->rebuild) && !update->skip_write)
+		index_mailbox_vsize_update_write(update);
+	file_lock_free(&update->lock);
+	if (update->finish_in_background)
+		index_mailbox_vsize_notify_indexer(update->box);
+
+	mail_index_view_close(&update->view);
+	i_free(update);
+}
+
+void index_mailbox_vsize_hdr_expunge(struct mailbox_vsize_update *update,
+				     uint32_t uid, uoff_t vsize)
+{
+	i_assert(update->lock != NULL);
+
+	if (uid > update->vsize_hdr.highest_uid)
+		return;
+	if (update->vsize_hdr.message_count == 0) {
+		mailbox_set_critical(update->box,
+			"vsize-hdr's message_count shrank below 0");
+		i_zero(&update->vsize_hdr);
+		return;
+	}
+	update->vsize_hdr.message_count--;
+	if (update->vsize_hdr.vsize < vsize) {
+		mailbox_set_critical(update->box,
+			"vsize-hdr's vsize shrank below 0");
+		i_zero(&update->vsize_hdr);
+		return;
+	}
+	update->vsize_hdr.vsize -= vsize;
+}
+
+static void
+index_mailbox_vsize_finish_bg(struct mailbox_vsize_update *update,
+			      bool require_result)
+{
+	mail_storage_set_error(update->box->storage, MAIL_ERROR_INUSE,
+			       "Finishing vsize calculation on background");
+	if (require_result)
+		update->finish_in_background = TRUE;
+}
+
+static int
+index_mailbox_vsize_hdr_add_missing(struct mailbox_vsize_update *update,
+				    bool require_result)
+{
+	struct mailbox_index_vsize *vsize_hdr = &update->vsize_hdr;
+	struct mailbox_transaction_context *trans;
+	struct mail_search_context *search_ctx;
+	struct mail_search_args *search_args;
+	struct mailbox_status status;
+	struct mail *mail;
+	unsigned int idx, mails_left;
+	uint32_t seq1, seq2;
+	uoff_t vsize;
+	int ret = 0;
+
+	mailbox_get_open_status(update->box, STATUS_UIDNEXT, &status);
+	if (vsize_hdr->highest_uid + 1 >= status.uidnext) {
+		/* nothing to do - we should have usually caught this already
+		   before locking */
+		return 0;
+	}
+
+	/* note that update->view may be more up-to-date than box->view.
+	   we'll just add whatever new mails are in box->view. if we'll notice
+	   that some of the new mails are missing, we'll need to stop there
+	   since that expunge will be applied later on to the vsize header. */
+	search_args = mail_search_build_init();
+	if (!mail_index_lookup_seq_range(update->box->view,
+					 vsize_hdr->highest_uid + 1,
+					 status.uidnext-1, &seq1, &seq2)) {
+		/* nothing existed, but update uidnext */
+		vsize_hdr->highest_uid = status.uidnext - 1;
+		mail_search_args_unref(&search_args);
+		return 0;
+	}
+	mail_search_build_add_seqset(search_args, seq1, seq2);
+
+	if (!mail_index_map_get_ext_idx(update->box->view->map,
+					update->box->vsize_hdr_ext_id, &idx)) {
+		/* vsize header doesn't exist yet. Create it here early so
+		   that vsize mail records get created (instead of adding
+		   size.virtuals to cache). */
+		index_mailbox_vsize_update_write_to_index(update);
+	}
+
+	trans = mailbox_transaction_begin(update->box, 0, "vsize update");
+	search_ctx = mailbox_search_init(trans, search_args, NULL,
+					 MAIL_FETCH_VIRTUAL_SIZE, NULL);
+	if (!require_result)
+		mails_left = 0;
+	else if (update->box->storage->set->mail_vsize_bg_after_count == 0)
+		mails_left = UINT_MAX;
+	else
+		mails_left = update->box->storage->set->mail_vsize_bg_after_count;
+
+	while (mailbox_search_next(search_ctx, &mail)) {
+		if (mails_left == 0) {
+			if (mail->mail_stream_accessed) {
+				/* Seems stream is opened by mailbox search, so we
+				   will stop here, and finish it on background. */
+				index_mailbox_vsize_finish_bg(update,
+							      require_result);
+				ret = -1;
+				break;
+			}
+			/* if there are any more mails whose vsize can't be
+			   looked up from cache, abort and finish on
+			   background. */
+			mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+		}
+		ret = mail_get_virtual_size(mail, &vsize);
+		mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+
+		if (ret < 0 &&
+		    mailbox_get_last_mail_error(update->box) == MAIL_ERROR_LOOKUP_ABORTED) {
+			/* abort and finish on background */
+			i_assert(mails_left == 0);
+			index_mailbox_vsize_finish_bg(update, require_result);
+			break;
+		}
+		if (mail->mail_stream_accessed ||
+		    mail->mail_metadata_accessed) {
+			/* slow vsize lookup */
+			i_assert(mails_left > 0);
+			mails_left--;
+		}
+
+		if (ret < 0) {
+			if (mail->expunged)
+				continue;
+			ret = -1;
+			break;
+		}
+		vsize_hdr->vsize += vsize;
+		vsize_hdr->highest_uid = mail->uid;
+		vsize_hdr->message_count++;
+	}
+	if (mailbox_search_deinit(&search_ctx) < 0)
+		ret = -1;
+	mail_search_args_unref(&search_args);
+
+	if (ret == 0) {
+		/* success, cache all */
+		vsize_hdr->highest_uid = status.uidnext - 1;
+	} else {
+		/* search failed, cache only up to highest seen uid */
+	}
+	(void)mailbox_transaction_commit(&trans);
+	return ret;
+}
+
+int index_mailbox_get_virtual_size(struct mailbox *box,
+				   struct mailbox_metadata *metadata_r)
+{
+	struct mailbox_vsize_update *update;
+	struct mailbox_status status;
+	int ret;
+
+	mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, &status);
+	update = index_mailbox_vsize_update_init(box);
+	if (update->vsize_hdr.highest_uid + 1 == status.uidnext &&
+	    update->vsize_hdr.message_count == status.messages) {
+		/* up to date */
+		metadata_r->virtual_size = update->vsize_hdr.vsize;
+		index_mailbox_vsize_update_deinit(&update);
+		return 0;
+	}
+
+	/* we need to update it - lock it if possible. if not, update it
+	   anyway internally even though we won't be saving the result. */
+	(void)index_mailbox_vsize_update_wait_lock(update);
+
+	struct event_reason *reason = event_reason_begin("mailbox:vsize");
+	ret = index_mailbox_vsize_hdr_add_missing(update, TRUE);
+	event_reason_end(&reason);
+	metadata_r->virtual_size = update->vsize_hdr.vsize;
+	index_mailbox_vsize_update_deinit(&update);
+	return ret;
+}
+
+int index_mailbox_get_physical_size(struct mailbox *box,
+				    struct mailbox_metadata *metadata_r)
+{
+	struct mailbox_transaction_context *trans;
+	struct mail_search_context *ctx;
+	struct mail *mail;
+	struct mail_search_args *search_args;
+	uoff_t size;
+	int ret = 0;
+
+	/* if physical size = virtual size always for the storage, we can
+	   use the optimized vsize code for this */
+	if (box->mail_vfuncs->get_physical_size ==
+	    box->mail_vfuncs->get_virtual_size) {
+		if (index_mailbox_get_virtual_size(box, metadata_r) < 0)
+			return -1;
+		metadata_r->physical_size = metadata_r->virtual_size;
+		return 0;
+	}
+	/* do it the slow way (we could implement similar logic as for vsize,
+	   but for now it's not really needed) */
+	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0)
+		return -1;
+
+	trans = mailbox_transaction_begin(box, 0, "mailbox physical size");
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_all(search_args);
+	ctx = mailbox_search_init(trans, search_args, NULL,
+				  MAIL_FETCH_PHYSICAL_SIZE, NULL);
+	mail_search_args_unref(&search_args);
+
+	metadata_r->physical_size = 0;
+	while (mailbox_search_next(ctx, &mail)) {
+		if (mail_get_physical_size(mail, &size) == 0)
+			metadata_r->physical_size += size;
+		else {
+			const char *errstr;
+			enum mail_error error;
+
+			errstr = mailbox_get_last_internal_error(box, &error);
+			if (error != MAIL_ERROR_EXPUNGED) {
+				i_error("Couldn't get size of mail UID %u in %s: %s",
+					mail->uid, box->vname, errstr);
+				ret = -1;
+				break;
+			}
+		}
+	}
+	if (mailbox_search_deinit(&ctx) < 0) {
+		i_error("Listing mails in %s failed: %s",
+			box->vname, mailbox_get_last_internal_error(box, NULL));
+		ret = -1;
+	}
+	(void)mailbox_transaction_commit(&trans);
+	return ret;
+}
+
+void index_mailbox_vsize_update_appends(struct mailbox *box)
+{
+	struct mailbox_vsize_update *update;
+	struct mailbox_status status;
+
+	update = index_mailbox_vsize_update_init(box);
+	if (update->rebuild) {
+		/* The vsize header doesn't exist. Don't create it. */
+		update->skip_write = TRUE;
+	}
+
+	/* update here only if we don't need to rebuild the whole vsize. */
+	index_mailbox_vsize_check_rebuild(update);
+	if (index_mailbox_vsize_want_updates(update)) {
+		/* Get the UIDNEXT only after checking that vsize updating is
+		   even potentially wanted for this mailbox. We especially
+		   don't want to do this with imapc, because it could trigger
+		   a remote STATUS (UIDNEXT) call. */
+		mailbox_get_open_status(update->box, STATUS_UIDNEXT, &status);
+		if (update->vsize_hdr.highest_uid + 1 != status.uidnext &&
+		    index_mailbox_vsize_update_try_lock(update)) {
+			struct event_reason *reason =
+				event_reason_begin("mailbox:vsize");
+			(void)index_mailbox_vsize_hdr_add_missing(update, FALSE);
+			event_reason_end(&reason);
+		}
+	}
+	index_mailbox_vsize_update_deinit(&update);
+}
diff --git a/src/lib-storage/index/index-mailbox-size.h b/src/lib-storage/index/index-mailbox-size.h
new file mode 100644
index 0000000..002da22
--- /dev/null
+++ b/src/lib-storage/index/index-mailbox-size.h
@@ -0,0 +1,20 @@
+#ifndef INDEX_MAILBOX_SIZE_H
+#define INDEX_MAILBOX_SIZE_H
+
+struct mailbox;
+
+struct mailbox_vsize_update *
+index_mailbox_vsize_update_init(struct mailbox *box);
+void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **update);
+
+void index_mailbox_vsize_hdr_expunge(struct mailbox_vsize_update *update,
+				     uint32_t uid, uoff_t vsize);
+
+bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update);
+bool index_mailbox_vsize_update_wait_lock(struct mailbox_vsize_update *update);
+/* Returns TRUE if expunges & appends should be updating the header. */
+bool index_mailbox_vsize_want_updates(struct mailbox_vsize_update *update);
+
+void index_mailbox_vsize_update_appends(struct mailbox *box);
+
+#endif
diff --git a/src/lib-storage/index/index-pop3-uidl.c b/src/lib-storage/index/index-pop3-uidl.c
new file mode 100644
index 0000000..e537e9f
--- /dev/null
+++ b/src/lib-storage/index/index-pop3-uidl.c
@@ -0,0 +1,104 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "index-storage.h"
+#include "index-mail.h"
+#include "index-pop3-uidl.h"
+
+void index_pop3_uidl_set_max_uid(struct mailbox *box,
+				 struct mail_index_transaction *trans,
+				 uint32_t uid)
+{
+	struct mailbox_index_pop3_uidl uidl;
+
+	i_zero(&uidl);
+	uidl.max_uid_with_pop3_uidl = uid;
+
+	mail_index_update_header_ext(trans, box->pop3_uidl_hdr_ext_id,
+				     0, &uidl, sizeof(uidl));
+}
+
+bool index_pop3_uidl_can_exist(struct mail *mail)
+{
+	struct mailbox_index_pop3_uidl uidl;
+	const void *data;
+	size_t size;
+
+	/* We'll assume that if the header exists, it's up-to-date. normally
+	   UIDLs are set only during migration, so this value never changes.
+	   Also even if it does, it becomes out-of-date only when the mailbox
+	   is modified with old Dovecot versions. To fix that we'd have to
+	   add and keep updating "max tracked uid" in this header for every
+	   saved mail, which isn't worth it. */
+	mail_index_get_header_ext(mail->transaction->view,
+				  mail->box->pop3_uidl_hdr_ext_id,
+				  &data, &size);
+	if (size < sizeof(uidl)) {
+		/* this header isn't set yet */
+		return TRUE;
+	}
+	memcpy(&uidl, data, sizeof(uidl));
+	return mail->uid <= uidl.max_uid_with_pop3_uidl;
+}
+
+void index_pop3_uidl_update_exists(struct mail *mail, bool exists)
+{
+	struct mailbox_transaction_context *trans = mail->transaction;
+
+	if (exists) {
+		if (trans->highest_pop3_uidl_uid < mail->uid) {
+			trans->highest_pop3_uidl_uid = mail->uid;
+			trans->prev_pop3_uidl_tracking_seq = mail->seq;
+		}
+	} else if (mail->seq == trans->prev_pop3_uidl_tracking_seq+1) {
+		trans->prev_pop3_uidl_tracking_seq++;
+	} else {
+		/* skipping mails. we don't know the state. */
+	}
+}
+
+void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans)
+{
+	struct mail_index_view *view;
+	struct mailbox_index_pop3_uidl uidl;
+	const void *data;
+	size_t size;
+	bool seen_all_msgs;
+
+	mail_index_get_header_ext(trans->view, trans->box->pop3_uidl_hdr_ext_id,
+				  &data, &size);
+
+	if (trans->highest_pop3_uidl_uid == 0 && size >= sizeof(uidl)) {
+		/* header already set and nothing to change */
+		return;
+	}
+
+	/* First check that we actually looked at UIDL for all messages.
+	   Otherwise we can't say for sure if the newest messages had UIDLs. */
+	if (trans->prev_pop3_uidl_tracking_seq !=
+	    mail_index_view_get_messages_count(trans->view))
+		return;
+
+	/* Just to be sure: Refresh the index and check again. POP3 keeps
+	   transactions open for duration of the entire session. Maybe another
+	   process already added new mails (and already updated this header).
+	   This check is racy, but normally UIDLs aren't added after migration
+	   so it's a bit questionable if it's even worth having this check in
+	   there. */
+	view = mail_index_view_open(trans->box->index);
+	seen_all_msgs = mail_index_refresh(trans->box->index) == 0 &&
+		trans->prev_pop3_uidl_tracking_seq ==
+		mail_index_view_get_messages_count(view);
+	mail_index_view_close(&view);
+	if (!seen_all_msgs)
+		return;
+
+	/* check if we have already the same header */
+	if (size >= sizeof(uidl)) {
+		memcpy(&uidl, data, sizeof(uidl));
+		if (trans->highest_pop3_uidl_uid == uidl.max_uid_with_pop3_uidl)
+			return;
+	}
+	index_pop3_uidl_set_max_uid(trans->box, trans->itrans,
+				    trans->highest_pop3_uidl_uid);
+}
diff --git a/src/lib-storage/index/index-pop3-uidl.h b/src/lib-storage/index/index-pop3-uidl.h
new file mode 100644
index 0000000..956ab7d
--- /dev/null
+++ b/src/lib-storage/index/index-pop3-uidl.h
@@ -0,0 +1,16 @@
+#ifndef INDEX_POP3_H
+#define INDEX_POP3_H
+
+struct mail_index_transaction;
+struct mail;
+struct mailbox;
+struct mailbox_transaction_context;
+
+void index_pop3_uidl_set_max_uid(struct mailbox *box,
+				 struct mail_index_transaction *trans,
+				 uint32_t uid);
+bool index_pop3_uidl_can_exist(struct mail *mail);
+void index_pop3_uidl_update_exists(struct mail *mail, bool exists);
+void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans);
+
+#endif
diff --git a/src/lib-storage/index/index-rebuild.c b/src/lib-storage/index/index-rebuild.c
new file mode 100644
index 0000000..0eaaab5
--- /dev/null
+++ b/src/lib-storage/index/index-rebuild.c
@@ -0,0 +1,257 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-cache.h"
+#include "mail-index-modseq.h"
+#include "mailbox-list-private.h"
+#include "mailbox-recent-flags.h"
+#include "index-storage.h"
+#include "index-rebuild.h"
+
+static void
+index_index_copy_vsize(struct index_rebuild_context *ctx,
+		       struct mail_index_view *view,
+		       uint32_t old_seq, uint32_t new_seq)
+{
+	const void *data;
+	bool expunged;
+
+	mail_index_lookup_ext(view, old_seq, ctx->box->mail_vsize_ext_id,
+			      &data, &expunged);
+	if (data != NULL && !expunged) {
+		mail_index_update_ext(ctx->trans, new_seq,
+				      ctx->box->mail_vsize_ext_id, data, NULL);
+	}
+}
+
+static void
+index_index_copy_cache(struct index_rebuild_context *ctx,
+		       struct mail_index_view *view,
+		       uint32_t old_seq, uint32_t new_seq)
+{
+	struct mail_index_map *map;
+	const void *data;
+	uint32_t reset_id = 0;
+	bool expunged;
+
+	if (ctx->cache_ext_id == (uint32_t)-1)
+		return;
+
+	mail_index_lookup_ext_full(view, old_seq, ctx->cache_ext_id,
+				   &map, &data, &expunged);
+	if (expunged)
+		return;
+
+	if (!mail_index_ext_get_reset_id(view, map, ctx->cache_ext_id,
+					 &reset_id) || reset_id == 0)
+		return;
+
+	if (!ctx->cache_used) {
+		/* set reset id */
+		ctx->cache_used = TRUE;
+		ctx->cache_reset_id = reset_id;
+		mail_index_ext_reset(ctx->trans, ctx->cache_ext_id,
+				     ctx->cache_reset_id, TRUE);
+	}
+	if (ctx->cache_reset_id == reset_id) {
+		mail_index_update_ext(ctx->trans, new_seq,
+				      ctx->cache_ext_id, data, NULL);
+	}
+}
+
+static void
+index_index_copy_from_old(struct index_rebuild_context *ctx,
+			  struct mail_index_view *view,
+			  uint32_t old_seq, uint32_t new_seq)
+{
+	struct mail_index *index = mail_index_view_get_index(view);
+	const struct mail_index_record *rec;
+	ARRAY_TYPE(keyword_indexes) old_keywords;
+	struct mail_keywords *kw;
+	uint64_t modseq;
+
+	/* copy flags */
+	rec = mail_index_lookup(view, old_seq);
+	mail_index_update_flags(ctx->trans, new_seq,
+				MODIFY_REPLACE, rec->flags);
+
+	/* copy keywords */
+	t_array_init(&old_keywords, 32);
+	mail_index_lookup_keywords(view, old_seq, &old_keywords);
+	kw = mail_index_keywords_create_from_indexes(index, &old_keywords);
+	mail_index_update_keywords(ctx->trans, new_seq, MODIFY_REPLACE, kw);
+	mail_index_keywords_unref(&kw);
+
+	/* copy modseq */
+	modseq = mail_index_modseq_lookup(view, old_seq);
+	mail_index_update_modseq(ctx->trans, new_seq, modseq);
+
+	index_index_copy_vsize(ctx, view, old_seq, new_seq);
+	index_index_copy_cache(ctx, view, old_seq, new_seq);
+}
+
+void index_rebuild_index_metadata(struct index_rebuild_context *ctx,
+				  uint32_t new_seq, uint32_t uid)
+{
+	uint32_t old_seq;
+
+	if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) {
+		/* the message exists in the old index.
+		   copy the metadata from it. */
+		index_index_copy_from_old(ctx, ctx->view, old_seq, new_seq);
+	} else if (ctx->backup_view != NULL &&
+		   mail_index_lookup_seq(ctx->backup_view, uid, &old_seq)) {
+		/* copy the metadata from backup index. */
+		index_index_copy_from_old(ctx, ctx->backup_view,
+					  old_seq, new_seq);
+	}
+}
+
+static void
+index_rebuild_header(struct index_rebuild_context *ctx,
+		     index_rebuild_generate_uidvalidity_t *gen_uidvalidity)
+{
+	const struct mail_index_header *hdr, *backup_hdr, *trans_hdr;
+	struct mail_index *index = mail_index_view_get_index(ctx->view);
+	struct mail_index_modseq_header modseq_hdr;
+	struct mail_index_view *trans_view;
+	uint32_t uid_validity, next_uid, first_recent_uid;
+	uint64_t modseq;
+
+	hdr = mail_index_get_header(ctx->view);
+	backup_hdr = ctx->backup_view == NULL ? NULL :
+		mail_index_get_header(ctx->backup_view);
+	trans_view = mail_index_transaction_open_updated_view(ctx->trans);
+	trans_hdr = mail_index_get_header(trans_view);
+
+	/* set uidvalidity */
+	if (hdr->uid_validity != 0)
+		uid_validity = hdr->uid_validity;
+	else if (backup_hdr != NULL && backup_hdr->uid_validity != 0)
+		uid_validity = backup_hdr->uid_validity;
+	else
+		uid_validity = gen_uidvalidity(ctx->box->list);
+	mail_index_update_header(ctx->trans,
+		offsetof(struct mail_index_header, uid_validity),
+		&uid_validity, sizeof(uid_validity), TRUE);
+
+	/* set next-uid */
+	if (hdr->next_uid != 0)
+		next_uid = hdr->next_uid;
+	else if (backup_hdr != NULL && backup_hdr->next_uid != 0)
+		next_uid = backup_hdr->next_uid;
+	else
+		next_uid = 1;
+	if (next_uid > trans_hdr->next_uid) {
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, next_uid),
+			&next_uid, sizeof(next_uid), FALSE);
+	}
+
+	/* set first_recent_uid */
+	first_recent_uid = hdr->first_recent_uid;
+	if (backup_hdr != NULL &&
+	    backup_hdr->first_recent_uid > first_recent_uid &&
+	    backup_hdr->first_recent_uid <= next_uid)
+		first_recent_uid = backup_hdr->first_recent_uid;
+	first_recent_uid = I_MIN(first_recent_uid, next_uid);
+	mail_index_update_header(ctx->trans,
+		offsetof(struct mail_index_header, first_recent_uid),
+		&first_recent_uid, sizeof(first_recent_uid), FALSE);
+
+	/* set highest-modseq */
+	i_zero(&modseq_hdr);
+	modseq_hdr.highest_modseq = mail_index_modseq_get_highest(ctx->view);
+	if (ctx->backup_view != NULL) {
+		modseq = mail_index_modseq_get_highest(ctx->backup_view);
+		if (modseq_hdr.highest_modseq < modseq)
+			modseq_hdr.highest_modseq = modseq;
+	}
+	mail_index_update_header_ext(ctx->trans, index->modseq_ext_id,
+				     0, &modseq_hdr, sizeof(modseq_hdr));
+	mail_index_view_close(&trans_view);
+}
+
+static void
+index_rebuild_box_preserve_header(struct index_rebuild_context *ctx,
+				  uint32_t ext_id)
+{
+	const void *hdr;
+	size_t hdr_size;
+
+	mail_index_get_header_ext(ctx->view, ext_id, &hdr, &hdr_size);
+	if (hdr_size == 0 && ctx->backup_view != NULL) {
+		mail_index_get_header_ext(ctx->backup_view, ext_id,
+					  &hdr, &hdr_size);
+	}
+	if (hdr_size == 0)
+		return;
+	mail_index_update_header_ext(ctx->trans, ext_id, 0, hdr, hdr_size);
+}
+
+struct index_rebuild_context *
+index_index_rebuild_init(struct mailbox *box, struct mail_index_view *view,
+			 struct mail_index_transaction *trans)
+{
+	struct index_rebuild_context *ctx;
+	const char *index_dir, *backup_path;
+	enum mail_index_open_flags open_flags = MAIL_INDEX_OPEN_FLAG_READONLY;
+
+	/* Rebuilding really should be done locked so multiple processes won't
+	   try to rebuild concurrently. Also at the end of rebiuld cache
+	   purging requires this lock. */
+	i_assert(mail_index_is_locked(view->index));
+
+	ctx = i_new(struct index_rebuild_context, 1);
+	ctx->box = box;
+	ctx->view = view;
+	ctx->trans = trans;
+	mail_index_reset(ctx->trans);
+	mailbox_recent_flags_reset(box);
+	(void)mail_index_ext_lookup(box->index, "cache", &ctx->cache_ext_id);
+
+	/* open cache and read the caching decisions. */
+	(void)mail_cache_open_and_verify(ctx->box->cache);
+
+	/* if backup index file exists, try to use it */
+	index_dir = mailbox_get_index_path(box);
+	backup_path = t_strconcat(box->index_prefix, ".backup", NULL);
+	ctx->backup_index = mail_index_alloc(box->event,
+					     index_dir, backup_path);
+
+#ifndef MMAP_CONFLICTS_WRITE
+	if (box->storage->set->mmap_disable)
+#endif
+		open_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
+	mail_index_set_lock_method(ctx->backup_index,
+				   box->storage->set->parsed_lock_method,
+				   UINT_MAX);
+	if (mail_index_open(ctx->backup_index, open_flags) <= 0)
+		mail_index_free(&ctx->backup_index);
+	else
+		ctx->backup_view = mail_index_view_open(ctx->backup_index);
+	return ctx;
+}
+
+void index_index_rebuild_deinit(struct index_rebuild_context **_ctx,
+				index_rebuild_generate_uidvalidity_t *cb)
+{
+	struct index_rebuild_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+
+	/* initialize cache file with the old field decisions */
+	(void)mail_cache_purge_with_trans(ctx->box->cache, ctx->trans,
+					  (uint32_t)-1, "rebuilding index");
+	index_rebuild_header(ctx, cb);
+	index_rebuild_box_preserve_header(ctx, ctx->box->box_name_hdr_ext_id);
+	index_rebuild_box_preserve_header(ctx, ctx->box->box_last_rename_stamp_ext_id);
+	index_rebuild_box_preserve_header(ctx, ctx->box->pop3_uidl_hdr_ext_id);
+	if (ctx->backup_index != NULL) {
+		mail_index_view_close(&ctx->backup_view);
+		mail_index_close(ctx->backup_index);
+		mail_index_free(&ctx->backup_index);
+	}
+	i_free(ctx);
+}
diff --git a/src/lib-storage/index/index-rebuild.h b/src/lib-storage/index/index-rebuild.h
new file mode 100644
index 0000000..183da9d
--- /dev/null
+++ b/src/lib-storage/index/index-rebuild.h
@@ -0,0 +1,32 @@
+#ifndef INDEX_REBUILD_H
+#define INDEX_REBUILD_H
+
+struct mailbox_list;
+
+struct index_rebuild_context {
+	struct mailbox *box;
+
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	uint32_t cache_ext_id;
+	uint32_t cache_reset_id;
+
+	struct mail_index *backup_index;
+	struct mail_index_view *backup_view;
+
+	bool cache_used:1;
+};
+
+typedef unsigned int
+index_rebuild_generate_uidvalidity_t(struct mailbox_list *list);
+
+struct index_rebuild_context *
+index_index_rebuild_init(struct mailbox *box, struct mail_index_view *view,
+			 struct mail_index_transaction *trans);
+void index_index_rebuild_deinit(struct index_rebuild_context **ctx,
+				index_rebuild_generate_uidvalidity_t *cb);
+
+void index_rebuild_index_metadata(struct index_rebuild_context *ctx,
+				  uint32_t new_seq, uint32_t uid);
+
+#endif
diff --git a/src/lib-storage/index/index-search-mime.c b/src/lib-storage/index/index-search-mime.c
new file mode 100644
index 0000000..da7e5e1
--- /dev/null
+++ b/src/lib-storage/index/index-search-mime.c
@@ -0,0 +1,624 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "message-date.h"
+#include "message-address.h"
+#include "message-part-data.h"
+#include "imap-bodystructure.h"
+#include "mail-search.h"
+#include "mail-search-mime.h"
+#include "index-search-private.h"
+
+struct search_mimepart_stack {
+	unsigned int index;
+};
+
+struct search_mimepart_context {
+	pool_t pool;
+	struct index_search_context *index_ctx;
+
+	/* message parts parsed from BODYSTRUCTURE */
+	struct message_part *mime_parts, *mime_part;
+
+	string_t *buf;
+
+	unsigned int depth, index;
+	ARRAY(struct search_mimepart_stack) stack;
+};
+
+static void search_mime_arg(struct mail_search_mime_arg *arg,
+			      struct search_mimepart_context *mpctx);
+
+static int seach_arg_mime_parent_match(struct search_mimepart_context *mpctx,
+				   struct mail_search_mime_arg *args)
+{
+	struct message_part *part = mpctx->mime_part;
+	unsigned int prev_depth, prev_index;
+	struct search_mimepart_stack *level;
+	int ret;
+
+	if (args->value.subargs == NULL) {
+		/* PARENT EXISTS: matches if this part has a parent.
+		 */
+		return (part->parent != NULL ? 1 : 0);
+	}
+
+	/* PARENT <mpart-key>: matches if this part's parent matches the
+	   mpart-key (subargs).
+	 */
+
+	prev_depth = mpctx->depth;
+	prev_index = mpctx->index;
+
+	level = array_idx_modifiable
+		(&mpctx->stack, mpctx->depth-1);
+
+	mpctx->mime_part = part->parent;
+	mail_search_mime_args_reset(args->value.subargs, TRUE);
+
+	mpctx->index = level->index;
+	mpctx->depth = mpctx->depth-1;
+	ret = mail_search_mime_args_foreach
+		(args->value.subargs, search_mime_arg, mpctx);
+
+	mpctx->mime_part = part;
+	mpctx->index = prev_index;
+	mpctx->depth = prev_depth;
+	return ret;
+}
+
+static int seach_arg_mime_child_match(struct search_mimepart_context *mpctx,
+				   struct mail_search_mime_arg *args)
+{
+	struct message_part *part, *prev_part;
+	unsigned int prev_depth, prev_index, depth;
+	struct search_mimepart_stack *level;
+	int ret = 0;
+
+	part = mpctx->mime_part;
+	if (args->value.subargs == NULL) {
+		/* CHILD EXISTS: matches if this part has any children; i.e., it is
+		   multipart.
+		 */
+		return (part->children != NULL ? 1 : 0);
+	}
+
+	/* CHILD <mpart-key>: matches if this part has any child that mathes
+	   the mpart-key (subargs).
+	 */
+
+	prev_part = part;
+	prev_depth = mpctx->depth;
+	prev_index = mpctx->index;
+
+	depth = mpctx->depth;
+	T_BEGIN {
+		ARRAY(struct search_mimepart_stack) prev_stack;
+
+		/* preserve current stack for any nested CHILD PARENT nastiness */
+		t_array_init(&prev_stack, 16);
+		array_copy(&prev_stack.arr, 0, &mpctx->stack.arr, 0,
+			array_count(&mpctx->stack));
+
+		depth++;
+		if (depth < array_count(&mpctx->stack))
+			level = array_idx_modifiable(&mpctx->stack, depth);
+		else {
+			i_assert(depth == array_count(&mpctx->stack));
+			level = array_append_space(&mpctx->stack);
+		}
+		level->index = 1;
+
+		part = part->children;
+		while (part != NULL) {
+			mpctx->mime_part = part;
+			mail_search_mime_args_reset(args->value.subargs, TRUE);
+
+			mpctx->depth = depth - prev_depth;
+			mpctx->index = level->index;
+			if ((ret=mail_search_mime_args_foreach
+				(args->value.subargs, search_mime_arg, mpctx)) != 0)
+				break;
+			if (part->children != NULL) {
+				depth++;
+				if (depth < array_count(&mpctx->stack))
+					level = array_idx_modifiable(&mpctx->stack, depth);
+				else {
+					i_assert(depth == array_count(&mpctx->stack));
+					level = array_append_space(&mpctx->stack);
+				}
+				level->index = 1;
+				part = part->children;
+			} else {
+				while (part->next == NULL) {
+					if (part->parent == NULL || part->parent == prev_part)
+						break;
+					depth--;
+					level = array_idx_modifiable(&mpctx->stack, depth);
+					part = part->parent;
+				}
+				level->index++;
+				part = part->next;
+			}
+		}
+
+		array_clear(&mpctx->stack);
+		array_copy(&mpctx->stack.arr, 0, &prev_stack.arr, 0,
+			array_count(&prev_stack));
+	} T_END;
+
+	mpctx->mime_part = prev_part;
+	mpctx->index = prev_index;
+	mpctx->depth = prev_depth;
+	return ret;
+}
+
+static int
+seach_arg_mime_substring_match(
+	struct search_mimepart_context *mpctx ATTR_UNUSED,
+	const char *key, const char *value)
+{
+	if (value == NULL)
+		return 0;
+
+	/* FIXME: Normalization is required */
+	return (strstr(value, key) != NULL ? 1 : 0);
+}
+
+static int
+seach_arg_mime_envelope_time_match(
+	struct search_mimepart_context *mpctx ATTR_UNUSED,
+	enum mail_search_mime_arg_type type, time_t search_time,
+	const struct message_part_envelope *envelope)
+{
+	time_t sent_time;
+	int timezone_offset;
+
+	if (envelope == NULL)
+		return 0;
+
+	/* NOTE: RFC-3501 specifies that timezone is ignored
+	   in searches. sent_time is returned as UTC, so change it. */
+	// FIXME: adjust comment
+	if (!message_date_parse((const unsigned char *)envelope->date,
+			strlen(envelope->date), &sent_time, &timezone_offset))
+		return 0;
+	sent_time += timezone_offset * 60;
+
+	switch (type) {
+	case SEARCH_MIME_SENTBEFORE:
+		return sent_time < search_time ? 1 : 0;
+	case SEARCH_MIME_SENTON:
+		return (sent_time >= search_time &&
+			sent_time < search_time + 3600*24) ? 1 : 0;
+	case SEARCH_MIME_SENTSINCE:
+		return sent_time >= search_time ? 1 : 0;
+	default:
+		i_unreached();
+	}
+}
+
+static int
+seach_arg_mime_envelope_address_match(
+	struct search_mimepart_context *mpctx ATTR_UNUSED,
+	enum mail_search_mime_arg_type type, const char *key,
+	const struct message_part_envelope *envelope)
+{
+	const struct message_address *addrs;
+	string_t *addrs_enc;
+
+	if (envelope == NULL)
+		return 0;
+
+	switch (type) {
+	case SEARCH_MIME_CC:
+		addrs = envelope->cc;
+		break;
+	case SEARCH_MIME_BCC:
+		addrs = envelope->bcc;
+		break;
+	case SEARCH_MIME_FROM:
+		addrs = envelope->from;
+		break;
+	case SEARCH_MIME_SENDER:
+		addrs = envelope->sender;
+		break;
+	case SEARCH_MIME_REPLY_TO:
+		addrs = envelope->reply_to;
+		break;
+	case SEARCH_MIME_TO:
+		addrs = envelope->to;
+		break;
+	default:
+		i_unreached();
+	}
+
+	/* FIXME: do we need to normalize anything? at least case insensitivity.
+	   MIME header encoding will make this a bit difficult, so it should
+	   probably be normalized directly in the struct message_address. */
+
+	addrs_enc = t_str_new(128);
+	message_address_write(addrs_enc, addrs);
+	return (strstr(str_c(addrs_enc), key) != NULL ? 1 : 0);
+}
+
+static int
+seach_arg_mime_filename_match(struct search_mimepart_context *mpctx,
+				   struct mail_search_mime_arg *arg)
+{
+	struct index_search_context *ictx = mpctx->index_ctx;
+	struct message_part *part = mpctx->mime_part;
+	char *key;
+	const char *value;
+	size_t vlen, alen;
+
+	if (!message_part_data_get_filename(part, &value))
+		return 0;
+
+	if (mpctx->buf == NULL)
+		mpctx->buf = str_new(default_pool, 256);
+
+	if (arg->context == NULL) {
+		str_truncate(mpctx->buf, 0);
+
+		if (ictx->mail_ctx.normalizer(arg->value.str,
+			strlen(arg->value.str), mpctx->buf) < 0)
+			i_panic("search key not utf8: %s", arg->value.str);
+		key = i_strdup(str_c(mpctx->buf));
+		arg->context = (void *)key;
+	} else {
+		key = (char *)arg->context;
+	}
+
+	str_truncate(mpctx->buf, 0);
+	if (ictx->mail_ctx.normalizer(value,
+		strlen(value), mpctx->buf) >= 0)
+		value = str_c(mpctx->buf);
+
+	switch (arg->type) {
+	case SEARCH_MIME_FILENAME_IS:
+		return (strcmp(value, key) == 0 ? 1 : 0);
+	case SEARCH_MIME_FILENAME_CONTAINS:
+		return (strstr(value, key) != NULL ? 1 : 0);
+	case SEARCH_MIME_FILENAME_BEGINS:
+		return (str_begins(value, key) ? 1 : 0);
+	case SEARCH_MIME_FILENAME_ENDS:
+		vlen = strlen(value);
+		alen = strlen(key);
+		return (str_begins(value + (vlen - alen), key) ? 1 : 0);
+	default:
+		break;
+	}
+	i_unreached();
+}
+static void
+search_arg_mime_filename_deinit(
+	struct search_mimepart_context *mpctx ATTR_UNUSED,
+	struct mail_search_mime_arg *arg)
+{
+	char *key = (char *)arg->context;
+
+	i_free(key);
+}
+
+static int
+seach_arg_mime_param_match(const struct message_part_param *params,
+				   unsigned int params_count,
+				   const char *name, const char *key)
+{
+	unsigned int i;
+
+	/* FIXME: Is normalization required? */
+
+	for (i = 0; i < params_count; i++) {
+		if (strcasecmp(params[i].name, name) == 0) {
+			if (key == NULL || *key == '\0')
+				return 1;
+			return (strstr(params[i].value, key) != NULL ? 1 : 0);
+		}
+	}
+	return 0;
+}
+
+static int
+seach_arg_mime_language_match(struct search_mimepart_context *mpctx,
+				   const char *key)
+{
+	struct message_part_data *data = mpctx->mime_part->data;
+	const char *const *lang;
+
+	i_assert(data != NULL);
+
+	lang = data->content_language;
+	if (lang != NULL) {
+		while (*lang != NULL) {
+			/* FIXME: Should use RFC 4647 matching rules */
+			if (strcasecmp(*lang, key) == 0)
+				return 1;
+			lang++;
+		}
+	}
+	return 0;
+}
+
+/* Returns >0 = matched, 0 = not matched (unused), -1 = unknown */
+static int search_mime_arg_match(struct search_mimepart_context *mpctx,
+				   struct mail_search_mime_arg *arg)
+{
+	struct message_part *part = mpctx->mime_part;
+	const struct message_part_data *data = part->data;
+
+	i_assert(data != NULL);
+
+	switch (arg->type) {
+	case SEARCH_MIME_OR:
+	case SEARCH_MIME_SUB:
+		i_unreached();
+
+	case SEARCH_MIME_SIZE_EQUAL:
+		return (part->body_size.virtual_size == arg->value.size ? 1 : 0);
+	case SEARCH_MIME_SIZE_LARGER:
+		return (part->body_size.virtual_size > arg->value.size ? 1 : 0);
+	case SEARCH_MIME_SIZE_SMALLER:
+		return (part->body_size.virtual_size < arg->value.size ? 1 : 0);
+
+	case SEARCH_MIME_DESCRIPTION:
+		return seach_arg_mime_substring_match(mpctx,
+			arg->value.str, data->content_description);
+	case SEARCH_MIME_DISPOSITION_TYPE:
+		return (data->content_disposition != NULL &&
+			strcasecmp(data->content_disposition,
+				arg->value.str) == 0 ? 1 : 0);
+	case SEARCH_MIME_DISPOSITION_PARAM:
+		return seach_arg_mime_param_match
+			(data->content_disposition_params,
+				data->content_disposition_params_count,
+				arg->field_name, arg->value.str);
+	case SEARCH_MIME_ENCODING:
+		return (data->content_transfer_encoding != NULL &&
+			strcasecmp(data->content_transfer_encoding,
+				arg->value.str) == 0 ? 1 : 0);
+	case SEARCH_MIME_ID:
+		return (data->content_id != NULL &&
+			strcasecmp(data->content_id,
+				arg->value.str) == 0 ? 1 : 0);
+	case SEARCH_MIME_LANGUAGE:
+		return seach_arg_mime_language_match(mpctx, arg->value.str);
+	case SEARCH_MIME_LOCATION:
+		return (data->content_location != NULL &&
+			strcasecmp(data->content_location,
+				arg->value.str) == 0 ? 1 : 0);
+	case SEARCH_MIME_MD5:
+		return (data->content_md5 != NULL &&
+			strcmp(data->content_md5,
+				arg->value.str) == 0 ? 1 : 0);
+
+	case SEARCH_MIME_TYPE:
+		return (data->content_type != NULL &&
+			strcasecmp(data->content_type,
+				arg->value.str) == 0 ? 1 : 0);
+	case SEARCH_MIME_SUBTYPE:
+		return (data->content_subtype != NULL &&
+			strcasecmp(data->content_subtype,
+				arg->value.str) == 0 ? 1 : 0);
+	case SEARCH_MIME_PARAM:
+		return seach_arg_mime_param_match
+			(data->content_type_params,
+				data->content_type_params_count,
+				arg->field_name, arg->value.str);
+
+	case SEARCH_MIME_SENTBEFORE:
+	case SEARCH_MIME_SENTON:
+	case SEARCH_MIME_SENTSINCE:
+		return seach_arg_mime_envelope_time_match
+			(mpctx, arg->type, arg->value.time, data->envelope);
+
+	case SEARCH_MIME_CC:
+	case SEARCH_MIME_BCC:
+	case SEARCH_MIME_FROM:
+	case SEARCH_MIME_REPLY_TO:
+	case SEARCH_MIME_SENDER:
+	case SEARCH_MIME_TO:
+		return seach_arg_mime_envelope_address_match
+			(mpctx, arg->type, arg->value.str, data->envelope);
+
+	case SEARCH_MIME_SUBJECT:
+		if (data->envelope == NULL)
+			return 0;
+		return seach_arg_mime_substring_match(mpctx,
+			arg->value.str, data->envelope->subject);
+	case SEARCH_MIME_IN_REPLY_TO:
+		if (data->envelope == NULL)
+			return 0;
+		return seach_arg_mime_substring_match(mpctx,
+			arg->value.str, data->envelope->in_reply_to);
+	case SEARCH_MIME_MESSAGE_ID:
+		if (data->envelope == NULL)
+			return 0;
+		return seach_arg_mime_substring_match(mpctx,
+			arg->value.str, data->envelope->message_id);
+
+	case SEARCH_MIME_DEPTH_EQUAL:
+		return (mpctx->depth == arg->value.number ? 1 : 0);
+	case SEARCH_MIME_DEPTH_MIN:
+		return (mpctx->depth >= arg->value.number ? 1 : 0);
+	case SEARCH_MIME_DEPTH_MAX:
+		return (mpctx->depth <= arg->value.number ? 1 : 0);
+	case SEARCH_MIME_INDEX:
+		return (mpctx->index == arg->value.number ? 1 : 0);
+
+	case SEARCH_MIME_PARENT:
+		return seach_arg_mime_parent_match(mpctx, arg);
+	case SEARCH_MIME_CHILD:
+		return seach_arg_mime_child_match(mpctx, arg);
+
+	case SEARCH_MIME_FILENAME_IS:
+	case SEARCH_MIME_FILENAME_CONTAINS:
+	case SEARCH_MIME_FILENAME_BEGINS:
+	case SEARCH_MIME_FILENAME_ENDS:
+		return seach_arg_mime_filename_match(mpctx, arg);
+
+	case SEARCH_MIME_HEADER:
+	case SEARCH_MIME_BODY:
+	case SEARCH_MIME_TEXT:
+		break;
+	}
+	return -1;
+}
+
+static void search_mime_arg(struct mail_search_mime_arg *arg,
+			      struct search_mimepart_context *mpctx)
+{
+	switch (search_mime_arg_match(mpctx, arg)) {
+	case -1:
+		/* unknown */
+		break;
+	case 0:
+		ARG_SET_RESULT(arg, 0);
+		break;
+	default:
+		ARG_SET_RESULT(arg, 1);
+		break;
+	}
+}
+
+static int seach_arg_mime_parts_match(struct search_mimepart_context *mpctx,
+				   struct mail_search_mime_arg *args,
+				   struct message_part *parts)
+{
+	struct message_part *part;
+	struct search_mimepart_stack *level;
+	int ret;
+
+	level = array_append_space(&mpctx->stack);
+	level->index = 1;
+
+	part = parts;
+	while (part != NULL) {
+		mpctx->mime_part = part;
+		mail_search_mime_args_reset(args, TRUE);
+
+		mpctx->index = level->index;
+		mpctx->depth = array_count(&mpctx->stack)-1;
+
+		if ((ret=mail_search_mime_args_foreach
+			(args, search_mime_arg, mpctx)) != 0)
+			return ret;
+		if (part->children != NULL) {
+			level = array_append_space(&mpctx->stack);
+			level->index = 1;
+			part = part->children;
+		} else {
+			while (part->next == NULL) {
+				if (part->parent == NULL)
+					break;
+				array_pop_back(&mpctx->stack);
+				level = array_back_modifiable(&mpctx->stack);
+				part = part->parent;
+			}
+			level->index++;
+			part = part->next;
+		}
+	}
+
+	return 0;
+}
+
+/* Returns >0 = matched, 0 = not matched, -1 = unknown */
+static int search_arg_match_mimepart(struct search_mimepart_context *mpctx,
+				   struct mail_search_arg *arg)
+{
+	struct index_search_context *ctx = mpctx->index_ctx;
+	const char *bodystructure, *error;
+
+	if (arg->type != SEARCH_MIMEPART)
+		return -1;
+
+	if (mpctx->pool == NULL) {
+		mpctx->pool = pool_alloconly_create
+			(MEMPOOL_GROWING"search mime parts", 4096);
+		p_array_init(&mpctx->stack, mpctx->pool, 16);
+	}
+	if (mpctx->mime_parts == NULL) {
+		/* FIXME: could the mail object already have message_part tree with
+		   data? */
+		if (mail_get_special(ctx->cur_mail,
+			MAIL_FETCH_IMAP_BODYSTRUCTURE, &bodystructure) < 0)
+			return -1;
+		if (imap_bodystructure_parse_full(bodystructure, mpctx->pool,
+			&mpctx->mime_parts, &error) < 0)
+			return -1;
+	}
+
+	/* FIXME: implement HEADER, BODY and TEXT (not from BODYSTRUCTURE)
+	   Needs to support FTS */
+	return seach_arg_mime_parts_match
+		(mpctx, arg->value.mime_part->args, mpctx->mime_parts);
+}
+
+static void search_mimepart_arg(struct mail_search_arg *arg,
+			      struct search_mimepart_context *mpctx)
+{
+	switch (search_arg_match_mimepart(mpctx, arg)) {
+	case -1:
+		/* unknown */
+		break;
+	case 0:
+		ARG_SET_RESULT(arg, 0);
+		break;
+	default:
+		ARG_SET_RESULT(arg, 1);
+		break;
+	}
+}
+
+int index_search_mime_arg_match(struct mail_search_arg *args,
+	struct index_search_context *ctx)
+{
+	struct search_mimepart_context mpctx;
+	int ret;
+
+	i_zero(&mpctx);
+	mpctx.index_ctx = ctx;
+
+	ret = mail_search_args_foreach(args,
+				       search_mimepart_arg, &mpctx);
+
+	pool_unref(&mpctx.pool);
+	str_free(&mpctx.buf);
+	return ret;
+}
+
+static void
+search_mime_arg_deinit(struct mail_search_mime_arg *arg,
+			      struct search_mimepart_context *mpctx ATTR_UNUSED)
+{
+	switch (arg->type) {
+	case SEARCH_MIME_FILENAME_IS:
+	case SEARCH_MIME_FILENAME_CONTAINS:
+	case SEARCH_MIME_FILENAME_BEGINS:
+	case SEARCH_MIME_FILENAME_ENDS:
+		search_arg_mime_filename_deinit(mpctx, arg);
+		break;
+	default:
+		break;
+	}
+}
+
+void index_search_mime_arg_deinit(struct mail_search_arg *arg,
+	struct index_search_context *ctx)
+{
+	struct search_mimepart_context mpctx;
+	struct mail_search_mime_arg *args;
+
+	i_assert(arg->type == SEARCH_MIMEPART);
+	args = arg->value.mime_part->args;
+
+	i_zero(&mpctx);
+	mpctx.index_ctx = ctx;
+
+	mail_search_mime_args_reset(args, TRUE);
+	(void)mail_search_mime_args_foreach(args,
+		search_mime_arg_deinit, &mpctx);
+}
diff --git a/src/lib-storage/index/index-search-private.h b/src/lib-storage/index/index-search-private.h
new file mode 100644
index 0000000..f451b56
--- /dev/null
+++ b/src/lib-storage/index/index-search-private.h
@@ -0,0 +1,45 @@
+#ifndef INDEX_SEARCH_PRIVATE_H
+#define INDEX_SEARCH_PRIVATE_H
+
+#include "mail-storage-private.h"
+
+#include <sys/time.h>
+
+struct mail_search_mime_part;
+struct imap_message_part;
+
+struct index_search_context {
+        struct mail_search_context mail_ctx;
+	struct mail_index_view *view;
+	struct mailbox *box;
+
+	uint32_t pvt_uid, pvt_seq;
+
+	enum mail_fetch_field extra_wanted_fields;
+	struct mailbox_header_lookup_ctx *extra_wanted_headers;
+
+	uint32_t seq1, seq2;
+	struct mail *cur_mail;
+	struct index_mail *cur_imail;
+	struct mail_thread_context *thread_ctx;
+
+	struct timeval search_start_time, last_notify;
+	struct timeval last_nonblock_timeval;
+	unsigned long long cost, next_time_check_cost;
+
+	bool failed:1;
+	bool sorted:1;
+	bool have_seqsets:1;
+	bool have_index_args:1;
+	bool have_mailbox_args:1;
+	bool have_nonmatch_always:1;
+};
+
+struct mail *index_search_get_mail(struct index_search_context *ctx);
+
+int index_search_mime_arg_match(struct mail_search_arg *args,
+	struct index_search_context *ctx);
+void index_search_mime_arg_deinit(struct mail_search_arg *arg,
+	struct index_search_context *ctx);
+
+#endif
diff --git a/src/lib-storage/index/index-search-result.c b/src/lib-storage/index/index-search-result.c
new file mode 100644
index 0000000..6bce84f
--- /dev/null
+++ b/src/lib-storage/index/index-search-result.c
@@ -0,0 +1,194 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "seq-range-array.h"
+#include "mail-search.h"
+#include "mailbox-search-result-private.h"
+#include "index-storage.h"
+#include "index-search-result.h"
+
+static void
+search_result_range_remove(struct mail_search_result *result,
+			   const ARRAY_TYPE(seq_range) *changed_uids_arr,
+			   unsigned int *idx,
+			   uint32_t *next_uid, uint32_t last_uid)
+{
+	const struct seq_range *uids;
+	unsigned int i, count;
+	uint32_t uid;
+
+	/* remove full seq_ranges */
+	uid = *next_uid;
+	uids = array_get(changed_uids_arr, &count);
+	for (i = *idx; uids[i].seq2 < last_uid;) {
+		i_assert(uids[i].seq1 <= uid);
+		for (; uid <= uids[i].seq2; uid++)
+			mailbox_search_result_remove(result, uid);
+		i++;
+		i_assert(i < count);
+		uid = uids[i].seq1;
+	}
+
+	/* remove the last seq_range */
+	i_assert(uids[i].seq1 <= uid && uids[i].seq2 >= last_uid);
+	for (; uid <= last_uid; uid++)
+		mailbox_search_result_remove(result, uid);
+
+	if (uid > uids[i].seq2) {
+		/* finished this range */
+		if (++i < count)
+			uid = uids[i].seq1;
+		else {
+			/* this was the last searched message */
+			uid = 0;
+		}
+	}
+
+	*next_uid = uid;
+	*idx = i;
+}
+
+static int
+search_result_update_search(struct mail_search_result *result,
+			    const ARRAY_TYPE(seq_range) *changed_uids_arr)
+{
+	struct mailbox_transaction_context *t;
+	struct mail_search_context *search_ctx;
+	struct mail *mail;
+	const struct seq_range *changed_uids;
+	unsigned int changed_count, changed_idx;
+	uint32_t next_uid;
+	int ret;
+
+	changed_uids = array_get(changed_uids_arr, &changed_count);
+	i_assert(changed_count > 0);
+	next_uid = changed_uids[0].seq1;
+	changed_idx = 0;
+
+	mail_search_args_init(result->search_args, result->box, FALSE, NULL);
+
+	t = mailbox_transaction_begin(result->box, 0, __func__);
+	search_ctx = mailbox_search_init(t, result->search_args, NULL, 0, NULL);
+	/* tell search that we're updating an existing search result,
+	   so it can do some optimizations based on it */
+	search_ctx->update_result = result;
+
+	while (mailbox_search_next(search_ctx, &mail)) {
+		i_assert(next_uid != 0);
+
+		if (next_uid != mail->uid) {
+			/* some messages in changed_uids didn't match.
+			   make sure they don't exist in the search result. */
+			search_result_range_remove(result, changed_uids_arr,
+						   &changed_idx, &next_uid,
+						   mail->uid-1);
+			i_assert(next_uid == mail->uid);
+		}
+		if (changed_uids[changed_idx].seq2 > next_uid) {
+			next_uid++;
+		} else if (++changed_idx < changed_count) {
+			next_uid = changed_uids[changed_idx].seq1;
+		} else {
+			/* this was the last searched message */
+			next_uid = 0;
+		}
+		/* match - make sure it exists in search result */
+		mailbox_search_result_add(result, mail->uid);
+	}
+	mail_search_args_deinit(result->search_args);
+	ret = mailbox_search_deinit(&search_ctx);
+
+	if (next_uid != 0 && ret == 0) {
+		/* last message(s) didn't match. make sure they don't exist
+		   in the search result. */
+		search_result_range_remove(result, changed_uids_arr,
+					   &changed_idx, &next_uid,
+					   changed_uids[changed_count-1].seq2);
+	}
+
+	if (mailbox_transaction_commit(&t) < 0)
+		ret = -1;
+	return ret;
+}
+
+int index_search_result_update_flags(struct mail_search_result *result,
+				     const ARRAY_TYPE(seq_range) *uids)
+{
+	struct mail_search_arg search_arg;
+	int ret;
+
+	if (array_count(uids) == 0)
+		return 0;
+
+	/* add a temporary search parameter to limit the search only to
+	   the changed messages */
+	i_zero(&search_arg);
+	search_arg.type = SEARCH_UIDSET;
+	search_arg.value.seqset = *uids;
+	search_arg.next = result->search_args->args;
+	result->search_args->args = &search_arg;
+	ret = search_result_update_search(result, uids);
+	i_assert(result->search_args->args == &search_arg);
+	result->search_args->args = search_arg.next;
+	return ret;
+}
+
+int index_search_result_update_appends(struct mail_search_result *result,
+				       unsigned int old_messages_count)
+{
+	struct mailbox_transaction_context *t;
+	struct mail_search_context *search_ctx;
+	struct mail *mail;
+	struct mail_search_arg search_arg;
+	uint32_t message_count;
+	int ret;
+
+	message_count = mail_index_view_get_messages_count(result->box->view);
+	if (old_messages_count == message_count) {
+		/* no new messages */
+		return 0;
+	}
+
+	/* add a temporary search parameter to limit the search only to
+	   the new messages */
+	i_zero(&search_arg);
+	search_arg.type = SEARCH_SEQSET;
+	t_array_init(&search_arg.value.seqset, 1);
+	seq_range_array_add_range(&search_arg.value.seqset,
+				  old_messages_count + 1, message_count);
+	search_arg.next = result->search_args->args;
+	result->search_args->args = &search_arg;
+
+	/* add all messages matching the search to search result */
+	t = mailbox_transaction_begin(result->box, 0, __func__);
+	search_ctx = mailbox_search_init(t, result->search_args, NULL, 0, NULL);
+
+	while (mailbox_search_next(search_ctx, &mail))
+		mailbox_search_result_add(result, mail->uid);
+
+	ret = mailbox_search_deinit(&search_ctx);
+	if (mailbox_transaction_commit(&t) < 0)
+		ret = -1;
+
+	i_assert(result->search_args->args == &search_arg);
+	result->search_args->args = search_arg.next;
+	return ret;
+}
+
+void index_search_results_update_expunges(struct mailbox *box,
+					  const ARRAY_TYPE(seq_range) *expunges)
+{
+	const struct seq_range *seqs;
+	uint32_t seq, uid;
+
+	if (array_count(&box->search_results) == 0)
+		return;
+
+	array_foreach(expunges, seqs) {
+		for (seq = seqs->seq1; seq <= seqs->seq2; seq++) {
+			mail_index_lookup_uid(box->view, seq, &uid);
+			mailbox_search_results_remove(box, uid);
+		}
+	}
+}
diff --git a/src/lib-storage/index/index-search-result.h b/src/lib-storage/index/index-search-result.h
new file mode 100644
index 0000000..a22d368
--- /dev/null
+++ b/src/lib-storage/index/index-search-result.h
@@ -0,0 +1,11 @@
+#ifndef INDEX_SEARCH_RESULT_H
+#define INDEX_SEARCH_RESULT_H
+
+int index_search_result_update_flags(struct mail_search_result *result,
+				     const ARRAY_TYPE(seq_range) *uids);
+int index_search_result_update_appends(struct mail_search_result *result,
+				       unsigned int old_messages_count);
+void index_search_results_update_expunges(struct mailbox *box,
+					  const ARRAY_TYPE(seq_range) *expunges);
+
+#endif
diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c
new file mode 100644
index 0000000..dec52f3
--- /dev/null
+++ b/src/lib-storage/index/index-search.c
@@ -0,0 +1,1922 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "istream.h"
+#include "utc-offset.h"
+#include "str.h"
+#include "time-util.h"
+#include "unichar.h"
+#include "imap-match.h"
+#include "message-address.h"
+#include "message-date.h"
+#include "message-search.h"
+#include "message-parser.h"
+#include "mail-index-modseq.h"
+#include "index-storage.h"
+#include "index-mail.h"
+#include "index-sort.h"
+#include "mail-search.h"
+#include "mailbox-search-result-private.h"
+#include "mailbox-recent-flags.h"
+#include "index-search-private.h"
+
+#include <ctype.h>
+
+#define SEARCH_NOTIFY_INTERVAL_SECS 10
+
+#define SEARCH_COST_DENTRY 3ULL
+#define SEARCH_COST_ATTR 1ULL
+#define SEARCH_COST_FILES_READ 25ULL
+#define SEARCH_COST_KBYTE 15ULL
+#define SEARCH_COST_CACHE 1ULL
+
+#define SEARCH_MIN_NONBLOCK_USECS 200000
+#define SEARCH_MAX_NONBLOCK_USECS 250000
+#define SEARCH_INITIAL_MAX_COST 30000
+#define SEARCH_RECALC_MIN_USECS 50000
+
+struct search_header_context {
+        struct index_search_context *index_ctx;
+        struct index_mail *imail;
+	struct mail_search_arg *args;
+
+	struct message_block decoded_block;
+	bool decoded_block_set;
+
+        struct message_header_line *hdr;
+
+	bool parse_headers:1;
+	bool custom_header:1;
+	bool threading:1;
+};
+
+struct search_body_context {
+        struct index_search_context *index_ctx;
+	struct istream *input;
+	struct message_part *part;
+};
+
+static void search_parse_msgset_args(unsigned int messages_count,
+				     struct mail_search_arg *args,
+				     uint32_t *seq1_r, uint32_t *seq2_r);
+
+static void ATTR_NULL(2)
+search_none(struct mail_search_arg *arg ATTR_UNUSED, void *ctx ATTR_UNUSED)
+{
+}
+
+static void search_set_failed(struct index_search_context *ctx)
+{
+	if (ctx->failed)
+		return;
+
+	/* remember the first failure */
+	mail_storage_last_error_push(ctx->box->storage);
+	ctx->failed = TRUE;
+}
+
+static void search_cur_mail_failed(struct index_search_context *ctx)
+{
+	switch (mailbox_get_last_mail_error(ctx->cur_mail->box)) {
+	case MAIL_ERROR_EXPUNGED:
+		ctx->mail_ctx.seen_lost_data = TRUE;
+		break;
+	case MAIL_ERROR_LOOKUP_ABORTED:
+		/* expected failure */
+		break;
+	default:
+		search_set_failed(ctx);
+		break;
+	}
+}
+
+static void search_init_arg(struct mail_search_arg *arg,
+			    struct index_search_context *ctx)
+{
+	struct mailbox_metadata metadata;
+	bool match;
+
+	switch (arg->type) {
+	case SEARCH_SEQSET:
+		ctx->have_seqsets = TRUE;
+		break;
+	case SEARCH_UIDSET:
+	case SEARCH_INTHREAD:
+	case SEARCH_FLAGS:
+	case SEARCH_KEYWORDS:
+	case SEARCH_MODSEQ:
+		if (arg->type == SEARCH_MODSEQ)
+			mail_index_modseq_enable(ctx->box->index);
+		ctx->have_index_args = TRUE;
+		break;
+	case SEARCH_MAILBOX_GUID:
+		if (mailbox_get_metadata(ctx->box, MAILBOX_METADATA_GUID,
+					 &metadata) < 0) {
+			/* result will be unknown */
+			break;
+		}
+
+		match = strcmp(guid_128_to_string(metadata.guid),
+			       arg->value.str) == 0;
+		if (match != arg->match_not)
+			arg->match_always = TRUE;
+		else {
+			arg->nonmatch_always = TRUE;
+			ctx->have_nonmatch_always = TRUE;
+		}
+		break;
+	case SEARCH_MAILBOX:
+	case SEARCH_MAILBOX_GLOB:
+		ctx->have_mailbox_args = TRUE;
+		break;
+	case SEARCH_ALL:
+		if (!arg->match_not)
+			arg->match_always = TRUE;
+		else {
+			arg->nonmatch_always = TRUE;
+			ctx->have_nonmatch_always = TRUE;
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+static void search_seqset_arg(struct mail_search_arg *arg,
+			      struct index_search_context *ctx)
+{
+	if (arg->type == SEARCH_SEQSET) {
+		if (seq_range_exists(&arg->value.seqset, ctx->mail_ctx.seq))
+			ARG_SET_RESULT(arg, 1);
+		else
+			ARG_SET_RESULT(arg, 0);
+	}
+}
+
+static int search_arg_match_keywords(struct index_search_context *ctx,
+				     struct mail_search_arg *arg)
+{
+	ARRAY_TYPE(keyword_indexes) keyword_indexes_arr;
+	const struct mail_keywords *search_kws = arg->initialized.keywords;
+	const unsigned int *keyword_indexes;
+	unsigned int i, j, count;
+
+	if (search_kws->count == 0) {
+		/* invalid keyword - never matches */
+		return 0;
+	}
+
+	t_array_init(&keyword_indexes_arr, 128);
+	mail_index_lookup_keywords(ctx->view, ctx->mail_ctx.seq,
+				   &keyword_indexes_arr);
+	keyword_indexes = array_get(&keyword_indexes_arr, &count);
+
+	/* there probably aren't many keywords, so O(n*m) for now */
+	for (i = 0; i < search_kws->count; i++) {
+		for (j = 0; j < count; j++) {
+			if (search_kws->idx[i] == keyword_indexes[j])
+				break;
+		}
+		if (j == count)
+			return 0;
+	}
+	return 1;
+}
+
+static bool
+index_search_get_pvt(struct index_search_context *ctx, uint32_t uid)
+{
+	index_transaction_init_pvt(ctx->mail_ctx.transaction);
+
+	if (ctx->pvt_uid == uid)
+		return ctx->pvt_seq != 0;
+	ctx->pvt_uid = uid;
+	return mail_index_lookup_seq(ctx->mail_ctx.transaction->view_pvt,
+				     uid, &ctx->pvt_seq);
+}
+
+/* Returns >0 = matched, 0 = not matched, -1 = unknown */
+static int search_arg_match_index(struct index_search_context *ctx,
+				  struct mail_search_arg *arg,
+				  const struct mail_index_record *rec)
+{
+	enum mail_flags flags, pvt_flags_mask;
+	uint64_t modseq;
+	int ret;
+
+	switch (arg->type) {
+	case SEARCH_UIDSET:
+	case SEARCH_INTHREAD:
+		return seq_range_exists(&arg->value.seqset, rec->uid) ? 1 : 0;
+	case SEARCH_FLAGS:
+		/* recent flag shouldn't be set, but indexes from v1.0.x
+		   may contain it. */
+		flags = rec->flags & ENUM_NEGATE(MAIL_RECENT);
+		if ((arg->value.flags & MAIL_RECENT) != 0 &&
+		    mailbox_recent_flags_have_uid(ctx->box, rec->uid))
+			flags |= MAIL_RECENT;
+		if (ctx->box->view_pvt == NULL) {
+			/* no private view (set by view syncing) ->
+			   no private flags */
+		} else {
+			pvt_flags_mask = mailbox_get_private_flags_mask(ctx->box);
+			flags &= ENUM_NEGATE(pvt_flags_mask);
+			if (index_search_get_pvt(ctx, rec->uid)) {
+				rec = mail_index_lookup(ctx->mail_ctx.transaction->view_pvt,
+							ctx->pvt_seq);
+				flags |= rec->flags & pvt_flags_mask;
+			}
+		}
+		return (flags & arg->value.flags) == arg->value.flags ? 1 : 0;
+	case SEARCH_KEYWORDS:
+		T_BEGIN {
+			ret = search_arg_match_keywords(ctx, arg);
+		} T_END;
+		return ret;
+	case SEARCH_MODSEQ: {
+		if (arg->value.flags != 0) {
+			modseq = mail_index_modseq_lookup_flags(ctx->view,
+					arg->value.flags, ctx->mail_ctx.seq);
+		} else if (arg->initialized.keywords != NULL) {
+			modseq = mail_index_modseq_lookup_keywords(ctx->view,
+					arg->initialized.keywords, ctx->mail_ctx.seq);
+		} else {
+			modseq = mail_index_modseq_lookup(ctx->view,
+						ctx->mail_ctx.seq);
+		}
+		return modseq >= arg->value.modseq->modseq ? 1 : 0;
+	}
+	default:
+		return -1;
+	}
+}
+
+static void search_index_arg(struct mail_search_arg *arg,
+			     struct index_search_context *ctx)
+{
+	const struct mail_index_record *rec;
+
+	rec = mail_index_lookup(ctx->view, ctx->mail_ctx.seq);
+	switch (search_arg_match_index(ctx, arg, rec)) {
+	case -1:
+		/* unknown */
+		break;
+	case 0:
+		ARG_SET_RESULT(arg, 0);
+		break;
+	default:
+		ARG_SET_RESULT(arg, 1);
+		break;
+	}
+}
+
+/* Returns >0 = matched, 0 = not matched, -1 = unknown */
+static int search_arg_match_mailbox(struct index_search_context *ctx,
+				    struct mail_search_arg *arg)
+{
+	struct mailbox *box = ctx->cur_mail->box;
+	const char *str;
+
+	switch (arg->type) {
+	case SEARCH_MAILBOX:
+		/* first try to match the mailbox name itself. this is
+		   important when using "mailbox virtual/foo" parameter foin
+		   doveadm's search query, otherwise we can never fetch
+		   anything with doveadm from virtual mailboxes because the
+		   mailbox parameter is compared to the mail's backend
+		   mailbox. */
+		if (strcmp(box->vname, arg->value.str) == 0)
+			return 1;
+		if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME,
+				     &str) < 0) {
+			search_cur_mail_failed(ctx);
+			return -1;
+		}
+
+		if (strcasecmp(str, "INBOX") == 0)
+			return strcasecmp(arg->value.str, "INBOX") == 0 ? 1 : 0;
+		return strcmp(str, arg->value.str) == 0 ? 1 : 0;
+	case SEARCH_MAILBOX_GLOB:
+		if (imap_match(arg->initialized.mailbox_glob, box->vname) == IMAP_MATCH_YES)
+			return 1;
+		if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME,
+				     &str) < 0) {
+			search_cur_mail_failed(ctx);
+			return -1;
+		}
+		return imap_match(arg->initialized.mailbox_glob, str) == IMAP_MATCH_YES ? 1 : 0;
+	default:
+		return -1;
+	}
+}
+
+static void search_mailbox_arg(struct mail_search_arg *arg,
+			       struct index_search_context *ctx)
+{
+	switch (search_arg_match_mailbox(ctx, arg)) {
+	case -1:
+		/* unknown */
+		break;
+	case 0:
+		ARG_SET_RESULT(arg, 0);
+		break;
+	default:
+		ARG_SET_RESULT(arg, 1);
+		break;
+	}
+}
+
+/* Returns >0 = matched, 0 = not matched, -1 = unknown */
+static int search_arg_match_cached(struct index_search_context *ctx,
+				   struct mail_search_arg *arg)
+{
+	const char *str;
+	struct tm *tm;
+	uoff_t virtual_size;
+	time_t date;
+	int tz_offset;
+	bool have_tz_offset;
+	int ret;
+
+	switch (arg->type) {
+	/* internal dates */
+	case SEARCH_BEFORE:
+	case SEARCH_ON:
+	case SEARCH_SINCE:
+		have_tz_offset = FALSE; tz_offset = 0; date = (time_t)-1;
+		switch (arg->value.date_type) {
+		case MAIL_SEARCH_DATE_TYPE_SENT:
+			if (mail_get_date(ctx->cur_mail, &date, &tz_offset) < 0) {
+				search_cur_mail_failed(ctx);
+				return -1;
+			}
+			have_tz_offset = TRUE;
+			break;
+		case MAIL_SEARCH_DATE_TYPE_RECEIVED:
+			if (mail_get_received_date(ctx->cur_mail, &date) < 0) {
+				search_cur_mail_failed(ctx);
+				return -1;
+			}
+			break;
+		case MAIL_SEARCH_DATE_TYPE_SAVED:
+			if (mail_get_save_date(ctx->cur_mail, &date) < 0) {
+				search_cur_mail_failed(ctx);
+				return -1;
+			}
+			break;
+		}
+
+		if ((arg->value.search_flags &
+		     MAIL_SEARCH_ARG_FLAG_UTC_TIMES) == 0) {
+			if (!have_tz_offset) {
+				tm = localtime(&date);
+				tz_offset = utc_offset(tm, date);
+			}
+			date += tz_offset * 60;
+		}
+
+		switch (arg->type) {
+		case SEARCH_BEFORE:
+			return date < arg->value.time ? 1 : 0;
+		case SEARCH_ON:
+			return (date >= arg->value.time &&
+				date < arg->value.time + 3600*24) ? 1 : 0;
+		case SEARCH_SINCE:
+			return date >= arg->value.time ? 1 : 0;
+		default:
+			i_unreached();
+		}
+
+	/* save date attribute */
+	case SEARCH_SAVEDATESUPPORTED:
+		ret = mail_get_save_date(ctx->cur_mail, &date);
+		if (ret < 0) {
+			search_cur_mail_failed(ctx);
+			return -1;
+		}
+		return ret;
+
+	/* sizes */
+	case SEARCH_SMALLER:
+	case SEARCH_LARGER:
+		if (mail_get_virtual_size(ctx->cur_mail, &virtual_size) < 0) {
+			search_cur_mail_failed(ctx);
+			return -1;
+		}
+
+		if (arg->type == SEARCH_SMALLER)
+			return virtual_size < arg->value.size ? 1 : 0;
+		else
+			return virtual_size > arg->value.size ? 1 : 0;
+
+	case SEARCH_GUID:
+		if (mail_get_special(ctx->cur_mail, MAIL_FETCH_GUID, &str) < 0) {
+			search_cur_mail_failed(ctx);
+			return -1;
+		}
+		return strcmp(str, arg->value.str) == 0 ? 1 : 0;
+	case SEARCH_REAL_UID: {
+		struct mail *real_mail;
+
+		if (mail_get_backend_mail(ctx->cur_mail, &real_mail) < 0) {
+			search_cur_mail_failed(ctx);
+			return -1;
+		}
+		return seq_range_exists(&arg->value.seqset, real_mail->uid) ? 1 : 0;
+	}
+	default:
+		return -1;
+	}
+}
+
+static void search_cached_arg(struct mail_search_arg *arg,
+			      struct index_search_context *ctx)
+{
+	switch (search_arg_match_cached(ctx, arg)) {
+	case -1:
+		/* unknown */
+		break;
+	case 0:
+		ARG_SET_RESULT(arg, 0);
+		break;
+	default:
+		ARG_SET_RESULT(arg, 1);
+		break;
+	}
+}
+
+static int search_sent(enum mail_search_arg_type type, time_t search_time,
+		       const unsigned char *sent_value, size_t sent_value_len)
+{
+	time_t sent_time;
+	int timezone_offset;
+
+	if (sent_value == NULL)
+		return 0;
+
+	/* NOTE: RFC-3501 specifies that timezone is ignored
+	   in searches. sent_time is returned as UTC, so change it. */
+	if (!message_date_parse(sent_value, sent_value_len,
+				&sent_time, &timezone_offset))
+		return 0;
+	sent_time += timezone_offset * 60;
+
+	switch (type) {
+	case SEARCH_BEFORE:
+		return sent_time < search_time ? 1 : 0;
+	case SEARCH_ON:
+		return (sent_time >= search_time &&
+			sent_time < search_time + 3600*24) ? 1 : 0;
+	case SEARCH_SINCE:
+		return sent_time >= search_time ? 1 : 0;
+	default:
+                i_unreached();
+	}
+}
+
+static struct message_search_context *
+msg_search_arg_context(struct index_search_context *ctx,
+		       struct mail_search_arg *arg)
+{
+	enum message_search_flags flags = 0;
+
+	if (arg->context == NULL) T_BEGIN {
+		string_t *dtc = t_str_new(128);
+
+		if (ctx->mail_ctx.normalizer(arg->value.str,
+					     strlen(arg->value.str), dtc) < 0)
+			i_panic("search key not utf8: %s", arg->value.str);
+
+		if (arg->type == SEARCH_BODY)
+			flags |= MESSAGE_SEARCH_FLAG_SKIP_HEADERS;
+		/* we don't get here if arg is "", but dtc can be "" if it
+		   only contains characters that we need to ignore. handle
+		   those searches by returning them as non-matched. */
+		if (str_len(dtc) > 0) {
+			arg->context =
+				message_search_init(str_c(dtc),
+						    ctx->mail_ctx.normalizer,
+						    flags);
+		}
+	} T_END;
+	return arg->context;
+}
+
+static void compress_lwsp(string_t *dest, const unsigned char *src,
+			  size_t src_len)
+{
+	size_t i;
+	bool prev_lwsp = TRUE;
+
+	for (i = 0; i < src_len; i++) {
+		if (IS_LWSP(src[i])) {
+			if (!prev_lwsp) {
+				prev_lwsp = TRUE;
+				str_append_c(dest, ' ');
+			}
+		} else {
+			prev_lwsp = FALSE;
+			str_append_c(dest, src[i]);
+		}
+	}
+}
+
+static void search_header_arg(struct mail_search_arg *arg,
+			      struct search_header_context *ctx)
+{
+        struct message_search_context *msg_search_ctx;
+	struct message_block block;
+	struct message_header_line hdr;
+	int ret;
+
+	/* first check that the field name matches to argument. */
+	switch (arg->type) {
+	case SEARCH_BEFORE:
+	case SEARCH_ON:
+	case SEARCH_SINCE:
+		if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_SENT)
+			return;
+
+		/* date is handled differently than others */
+		if (strcasecmp(ctx->hdr->name, "Date") == 0) {
+			if (ctx->hdr->continues) {
+				ctx->hdr->use_full_value = TRUE;
+				return;
+			}
+			ret = search_sent(arg->type, arg->value.time,
+					  ctx->hdr->full_value,
+					  ctx->hdr->full_value_len);
+			ARG_SET_RESULT(arg, ret);
+		}
+		return;
+
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+		ctx->custom_header = TRUE;
+
+		if (strcasecmp(ctx->hdr->name, arg->hdr_field_name) != 0)
+			return;
+		break;
+	default:
+		return;
+	}
+
+	if (arg->value.str[0] == '\0') {
+		/* we're just testing existence of the field. always matches. */
+		ARG_SET_RESULT(arg, 1);
+		return;
+	}
+
+	if (ctx->hdr->continues) {
+		ctx->hdr->use_full_value = TRUE;
+		return;
+	}
+
+	i_zero(&block);
+
+	/* We're searching only for values, so drop header name and middle
+	   parts. We use header searching so that MIME words will be decoded. */
+	hdr = *ctx->hdr;
+	hdr.name = ""; hdr.name_len = 0;
+	hdr.middle_len = 0;
+	block.hdr = &hdr;
+
+	msg_search_ctx = msg_search_arg_context(ctx->index_ctx, arg);
+	if (msg_search_ctx == NULL)
+		return;
+
+	if (!ctx->decoded_block_set) { T_BEGIN {
+		struct message_address *addr;
+		string_t *str;
+
+		switch (arg->type) {
+		case SEARCH_HEADER:
+			/* simple match */
+			break;
+		case SEARCH_HEADER_ADDRESS:
+			/* we have to match against normalized address */
+			addr = message_address_parse(pool_datastack_create(),
+						     ctx->hdr->full_value,
+						     ctx->hdr->full_value_len,
+						     UINT_MAX,
+						     MESSAGE_ADDRESS_PARSE_FLAG_FILL_MISSING);
+			str = t_str_new(ctx->hdr->value_len);
+			message_address_write(str, addr);
+			hdr.value = hdr.full_value = str_data(str);
+			hdr.value_len = hdr.full_value_len = str_len(str);
+			break;
+		case SEARCH_HEADER_COMPRESS_LWSP:
+			/* convert LWSP to single spaces */
+			str = t_str_new(hdr.full_value_len);
+			compress_lwsp(str, hdr.full_value, hdr.full_value_len);
+			hdr.value = hdr.full_value = str_data(str);
+			hdr.value_len = hdr.full_value_len = str_len(str);
+			break;
+		default:
+			i_unreached();
+		}
+		ret = message_search_more_get_decoded(msg_search_ctx, &block,
+						      &ctx->decoded_block) ? 1 : 0;
+		ctx->decoded_block_set = TRUE;
+	} T_END; } else {
+		/* this block was already decoded and saved by an earlier
+		   search arg. use the already-decoded block to avoid
+		   duplicating work. */
+		ret = message_search_more_decoded(msg_search_ctx,
+						  &ctx->decoded_block) ? 1 : 0;
+	}
+
+	/* there may be multiple headers. don't mark this failed yet. */
+	if (ret > 0)
+		ARG_SET_RESULT(arg, 1);
+}
+
+static void search_header_unmatch(struct mail_search_arg *arg,
+				  struct search_header_context *ctx ATTR_UNUSED)
+{
+	switch (arg->type) {
+	case SEARCH_BEFORE:
+	case SEARCH_ON:
+	case SEARCH_SINCE:
+		if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_SENT)
+			break;
+
+		if (arg->match_not) {
+			/* date header not found, so we match only for
+			   NOT searches */
+			ARG_SET_RESULT(arg, 0);
+		}
+		break;
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+		ARG_SET_RESULT(arg, 0);
+		break;
+	default:
+		break;
+	}
+}
+
+static void search_header(struct message_header_line *hdr,
+			  struct search_header_context *ctx)
+{
+	if (ctx->parse_headers)
+		index_mail_parse_header(NULL, hdr, ctx->imail);
+
+	if (hdr == NULL) {
+		/* end of headers, mark all unknown SEARCH_HEADERs unmatched */
+		(void)mail_search_args_foreach(ctx->args, search_header_unmatch,
+					       ctx);
+		return;
+	}
+
+	if (hdr->eoh)
+		return;
+
+	if (ctx->custom_header || strcasecmp(hdr->name, "Date") == 0) {
+		ctx->hdr = hdr;
+
+		ctx->decoded_block_set = FALSE;
+		ctx->custom_header = FALSE;
+		(void)mail_search_args_foreach(ctx->args, search_header_arg, ctx);
+	}
+}
+
+static void search_body(struct mail_search_arg *arg,
+			struct search_body_context *ctx)
+{
+	struct message_search_context *msg_search_ctx;
+	const char *error;
+	int ret;
+
+	switch (arg->type) {
+	case SEARCH_BODY:
+	case SEARCH_TEXT:
+		break;
+	default:
+		return;
+	}
+
+	msg_search_ctx = msg_search_arg_context(ctx->index_ctx, arg);
+	if (msg_search_ctx == NULL) {
+		ARG_SET_RESULT(arg, 0);
+		return;
+	}
+
+	i_stream_seek(ctx->input, 0);
+	ret = message_search_msg(msg_search_ctx, ctx->input, ctx->part, &error);
+	if (ret < 0 && ctx->input->stream_errno == 0) {
+		/* try again without cached parts */
+		index_mail_set_message_parts_corrupted(ctx->index_ctx->cur_mail, error);
+
+		i_stream_seek(ctx->input, 0);
+		ret = message_search_msg(msg_search_ctx, ctx->input, NULL, &error);
+		i_assert(ret >= 0 || ctx->input->stream_errno != 0);
+	}
+	if (ctx->input->stream_errno != 0) {
+		mailbox_set_critical(ctx->index_ctx->box,
+			"read(%s) failed: %s", i_stream_get_name(ctx->input),
+			i_stream_get_error(ctx->input));
+	}
+
+	ARG_SET_RESULT(arg, ret);
+}
+
+static int search_arg_match_text(struct mail_search_arg *args,
+				 struct index_search_context *ctx)
+{
+	const enum message_header_parser_flags hdr_parser_flags =
+		MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE;
+	struct index_mail *imail = INDEX_MAIL(ctx->cur_mail);
+	struct mail *real_mail;
+	struct istream *input = NULL;
+	struct mailbox_header_lookup_ctx *headers_ctx;
+	struct search_header_context hdr_ctx;
+	struct search_body_context body_ctx;
+	const char *const *headers;
+	bool have_headers, have_body, failed = FALSE;
+	int ret;
+
+	/* first check what we need to use */
+	headers = mail_search_args_analyze(args, &have_headers, &have_body);
+	if (!have_headers && !have_body)
+		return -1;
+
+	i_zero(&hdr_ctx);
+	hdr_ctx.index_ctx = ctx;
+	/* hdr_ctx.imail is different from imail for mails in
+	   virtual mailboxes */
+	if (mail_get_backend_mail(ctx->cur_mail, &real_mail) < 0) {
+		search_cur_mail_failed(ctx);
+		return -1;
+	}
+	hdr_ctx.imail = INDEX_MAIL(real_mail);
+	hdr_ctx.custom_header = TRUE;
+	hdr_ctx.args = args;
+
+	headers_ctx = headers == NULL ? NULL :
+		mailbox_header_lookup_init(ctx->box, headers);
+	if (headers != NULL &&
+	    (!have_body ||
+	     ctx->cur_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)) {
+		/* try to look up the specified headers from cache */
+		i_assert(*headers != NULL);
+
+		if (mail_get_header_stream(ctx->cur_mail, headers_ctx,
+					   &input) < 0) {
+			search_cur_mail_failed(ctx);
+			failed = TRUE;
+		} else {
+			message_parse_header(input, NULL, hdr_parser_flags,
+					     search_header, &hdr_ctx);
+		}
+		input = NULL;
+	} else if (have_headers) {
+		/* we need to read the entire header */
+		ret = have_body ?
+			mail_get_stream_because(ctx->cur_mail, NULL, NULL, "search", &input) :
+			mail_get_hdr_stream_because(ctx->cur_mail, NULL, "search", &input);
+		if (ret < 0) {
+			search_cur_mail_failed(ctx);
+			failed = TRUE;
+		} else {
+			/* FIXME: The header parsing here is an optimization to
+			   avoid parsing the header twice: First when checking
+			   whether the search matches, and secondly when
+			   generating wanted fields. However, if we already
+			   know that we want to generate a BODYSTRUCTURE reply,
+			   index_mail_parse_header() must have a non-NULL part
+			   parameter. That's not easily possible at this point
+			   without larger code changes, so for now we'll just
+			   disable this optimization for that case. */
+			hdr_ctx.parse_headers =
+				!hdr_ctx.imail->data.save_bodystructure_header &&
+				index_mail_want_parse_headers(hdr_ctx.imail);
+			if (hdr_ctx.parse_headers) {
+				index_mail_parse_header_init(hdr_ctx.imail,
+							     headers_ctx);
+			}
+			message_parse_header(input, NULL, hdr_parser_flags,
+					     search_header, &hdr_ctx);
+			if (input->stream_errno != 0) {
+				mailbox_set_critical(ctx->box,
+					"read(%s) failed: %s",
+					i_stream_get_name(input),
+					i_stream_get_error(input));
+				failed = TRUE;
+				search_set_failed(ctx);
+			}
+		}
+	}
+	mailbox_header_lookup_unref(&headers_ctx);
+
+	if (failed) {
+		/* opening mail failed. maybe because of lookup_abort.
+		   update access_parts for prefetching */
+		if (have_body)
+			imail->data.access_part |= READ_HDR | READ_BODY;
+		else 
+			imail->data.access_part |= READ_HDR;
+		return -1;
+	}
+
+	if (have_headers) {
+		/* see if the header search succeeded in finishing the search */
+		ret = mail_search_args_foreach(args, search_none, NULL);
+		if (ret >= 0 || !have_body)
+			return ret;
+	}
+
+	i_assert(have_body);
+
+	if (ctx->cur_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) {
+		imail->data.access_part |= READ_HDR | READ_BODY;
+		return -1;
+	}
+
+	if (input == NULL) {
+		/* we didn't search headers. */
+		struct message_size hdr_size;
+
+		if (mail_get_stream_because(ctx->cur_mail, &hdr_size, NULL, "search", &input) < 0) {
+			search_cur_mail_failed(ctx);
+			return -1;
+		}
+		i_stream_seek(input, hdr_size.physical_size);
+	}
+
+	i_zero(&body_ctx);
+	body_ctx.index_ctx = ctx;
+	body_ctx.input = input;
+	/* Get parts if they already exist in cache. If they don't,
+	   message-search will parse the mail automatically. */
+	ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+	(void)mail_get_parts(ctx->cur_mail, &body_ctx.part);
+	ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+
+	return mail_search_args_foreach(args, search_body, &body_ctx);
+}
+
+static bool
+search_msgset_fix_limits(unsigned int messages_count,
+			 ARRAY_TYPE(seq_range) *seqset, bool match_not)
+{
+	struct seq_range *range;
+	unsigned int count;
+
+	i_assert(messages_count > 0);
+
+	range = array_get_modifiable(seqset, &count);
+	if (count > 0) {
+		i_assert(range[0].seq1 != 0);
+		if (range[count-1].seq2 == (uint32_t)-1) {
+			/* "*" used, make sure the last message is in the range
+			   (e.g. with count+1:* we still want to include it) */
+			seq_range_array_add(seqset, messages_count);
+		}
+		/* remove all nonexistent messages */
+		seq_range_array_remove_range(seqset, messages_count + 1,
+					     (uint32_t)-1);
+	}
+	if (!match_not)
+		return array_count(seqset) > 0;
+	else {
+		/* if all messages are in the range, it can't match */
+		range = array_get_modifiable(seqset, &count);
+		return count == 0 || range[0].seq1 != 1 ||
+			range[count-1].seq2 != messages_count;
+	}
+}
+
+static void
+search_msgset_fix(unsigned int messages_count,
+		  ARRAY_TYPE(seq_range) *seqset,
+		  uint32_t *seq1_r, uint32_t *seq2_r, bool match_not)
+{
+	const struct seq_range *range;
+	unsigned int count;
+	uint32_t min_seq, max_seq;
+
+	if (!search_msgset_fix_limits(messages_count, seqset, match_not)) {
+		*seq1_r = (uint32_t)-1;
+		*seq2_r = 0;
+		return;
+	}
+
+	range = array_get(seqset, &count);
+	if (!match_not) {
+		min_seq = range[0].seq1;
+		max_seq = range[count-1].seq2;
+	} else if (count == 0) {
+		/* matches all messages */
+		min_seq = 1;
+		max_seq = messages_count;
+	} else {
+		min_seq = range[0].seq1 > 1 ? 1 : range[0].seq2 + 1;
+		max_seq = range[count-1].seq2 < messages_count ?
+			messages_count : range[count-1].seq1 - 1;
+		if (min_seq > max_seq) {
+			*seq1_r = (uint32_t)-1;
+			*seq2_r = 0;
+			return;
+		}
+	}
+
+	if (*seq1_r < min_seq || *seq1_r == 0)
+		*seq1_r = min_seq;
+	if (*seq2_r > max_seq)
+		*seq2_r = max_seq;
+}
+
+static void search_or_parse_msgset_args(unsigned int messages_count,
+					struct mail_search_arg *args,
+					uint32_t *seq1_r, uint32_t *seq2_r)
+{
+	uint32_t seq1, seq2, min_seq1 = 0, max_seq2 = 0;
+
+	for (; args != NULL; args = args->next) {
+		seq1 = 1; seq2 = messages_count;
+
+		switch (args->type) {
+		case SEARCH_SUB:
+			i_assert(!args->match_not);
+			search_parse_msgset_args(messages_count,
+						 args->value.subargs,
+						 &seq1, &seq2);
+			break;
+		case SEARCH_OR:
+			i_assert(!args->match_not);
+			search_or_parse_msgset_args(messages_count,
+						    args->value.subargs,
+						    &seq1, &seq2);
+			break;
+		case SEARCH_SEQSET:
+			search_msgset_fix(messages_count, &args->value.seqset,
+					  &seq1, &seq2, args->match_not);
+			break;
+		default:
+			break;
+		}
+
+		if (min_seq1 == 0) {
+			min_seq1 = seq1;
+			max_seq2 = seq2;
+		} else {
+			if (seq1 < min_seq1)
+				min_seq1 = seq1;
+			if (seq2 > max_seq2)
+				max_seq2 = seq2;
+		}
+	}
+	i_assert(min_seq1 != 0);
+
+	if (min_seq1 > *seq1_r)
+		*seq1_r = min_seq1;
+	if (max_seq2 < *seq2_r)
+		*seq2_r = max_seq2;
+}
+
+static void search_parse_msgset_args(unsigned int messages_count,
+				     struct mail_search_arg *args,
+				     uint32_t *seq1_r, uint32_t *seq2_r)
+{
+	for (; args != NULL; args = args->next) {
+		switch (args->type) {
+		case SEARCH_SUB:
+			i_assert(!args->match_not);
+			search_parse_msgset_args(messages_count,
+						 args->value.subargs,
+						 seq1_r, seq2_r);
+			break;
+		case SEARCH_OR:
+			/* go through our children and use the widest seqset
+			   range */
+			i_assert(!args->match_not);
+			search_or_parse_msgset_args(messages_count,
+						    args->value.subargs,
+						    seq1_r, seq2_r);
+			break;
+		case SEARCH_SEQSET:
+			search_msgset_fix(messages_count, &args->value.seqset,
+					  seq1_r, seq2_r, args->match_not);
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+static void search_limit_lowwater(struct index_search_context *ctx,
+				  uint32_t uid_lowwater, uint32_t *first_seq)
+{
+	uint32_t seq1, seq2;
+
+	if (uid_lowwater == 0)
+		return;
+
+	(void)mail_index_lookup_seq_range(ctx->view, uid_lowwater, (uint32_t)-1,
+					  &seq1, &seq2);
+	if (*first_seq < seq1)
+		*first_seq = seq1;
+}
+
+static bool search_limit_by_hdr(struct index_search_context *ctx,
+				struct mail_search_arg *args,
+				uint32_t *seq1, uint32_t *seq2)
+{
+	const struct mail_index_header *hdr;
+	enum mail_flags pvt_flags_mask;
+	uint64_t highest_modseq;
+
+	hdr = mail_index_get_header(ctx->view);
+	/* we can't trust that private view's header is fully up to date,
+	   so do this optimization only for non-private flags */
+	pvt_flags_mask = ctx->box->view_pvt == NULL ? 0 :
+		mailbox_get_private_flags_mask(ctx->box);
+
+	for (; args != NULL; args = args->next) {
+		switch (args->type) {
+		case SEARCH_ALL:
+			if (args->match_not) {
+				/* NOT ALL - pointless noop query */
+				return FALSE;
+			}
+			continue;
+		case SEARCH_MODSEQ:
+			/* MODSEQ higher than current HIGHESTMODSEQ? */
+			highest_modseq = mail_index_modseq_get_highest(ctx->view);
+			if (args->value.modseq->modseq > highest_modseq)
+				return FALSE;
+			continue;
+		default:
+			continue;
+		case SEARCH_FLAGS:
+			break;
+		}
+		if ((args->value.flags & MAIL_SEEN) != 0 &&
+		    (pvt_flags_mask & MAIL_SEEN) == 0) {
+			/* SEEN with 0 seen? */
+			if (!args->match_not && hdr->seen_messages_count == 0)
+				return FALSE;
+
+			if (hdr->seen_messages_count == hdr->messages_count) {
+				/* UNSEEN with all seen? */
+				if (args->match_not)
+					return FALSE;
+			} else if (args->match_not) {
+				/* UNSEEN with lowwater limiting */
+				search_limit_lowwater(ctx,
+                                	hdr->first_unseen_uid_lowwater, seq1);
+			}
+		}
+		if ((args->value.flags & MAIL_DELETED) != 0 &&
+		    (pvt_flags_mask & MAIL_DELETED) == 0) {
+			/* DELETED with 0 deleted? */
+			if (!args->match_not &&
+			    hdr->deleted_messages_count == 0)
+				return FALSE;
+
+			if (hdr->deleted_messages_count == hdr->messages_count) {
+				/* UNDELETED with all deleted? */
+				if (args->match_not)
+					return FALSE;
+			} else if (!args->match_not) {
+				/* DELETED with lowwater limiting */
+				search_limit_lowwater(ctx,
+                                	hdr->first_deleted_uid_lowwater, seq1);
+			}
+		}
+	}
+
+	return *seq1 <= *seq2;
+}
+
+static void search_get_seqset(struct index_search_context *ctx,
+			      unsigned int messages_count,
+			      struct mail_search_arg *args)
+{
+	if (messages_count == 0) {
+		/* no messages, don't check sequence ranges. although we could
+		   give error message then for FETCH, we shouldn't do it for
+		   UID FETCH. */
+		ctx->seq1 = 1;
+		ctx->seq2 = 0;
+		return;
+	}
+
+	ctx->seq1 = 1;
+	ctx->seq2 = messages_count;
+
+	search_parse_msgset_args(messages_count, args, &ctx->seq1, &ctx->seq2);
+	if (ctx->seq1 == 0) {
+		ctx->seq1 = 1;
+		ctx->seq2 = messages_count;
+	}
+	if (ctx->seq1 > ctx->seq2) {
+		/* no matches */
+		return;
+	}
+
+	/* See if this search query can never match based on data in index's
+	   header. We'll scan only the root level args, which is usually
+	   enough. */
+	if (!search_limit_by_hdr(ctx, args, &ctx->seq1, &ctx->seq2)) {
+		/* no matches */
+		ctx->seq1 = 1;
+		ctx->seq2 = 0;
+	}
+}
+
+static int search_build_subthread(struct mail_thread_iterate_context *iter,
+				  ARRAY_TYPE(seq_range) *uids)
+{
+	struct mail_thread_iterate_context *child_iter;
+	const struct mail_thread_child_node *node;
+	int ret = 0;
+
+	while ((node = mail_thread_iterate_next(iter, &child_iter)) != NULL) {
+		if (child_iter != NULL) {
+			if (search_build_subthread(child_iter, uids) < 0)
+				ret = -1;
+		}
+		seq_range_array_add(uids, node->uid);
+	}
+	if (mail_thread_iterate_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+static int search_build_inthread_result(struct index_search_context *ctx,
+					struct mail_search_arg *arg)
+{
+	struct mail_thread_iterate_context *iter, *child_iter;
+	const struct mail_thread_child_node *node;
+	const ARRAY_TYPE(seq_range) *search_uids;
+	ARRAY_TYPE(seq_range) thread_uids;
+	int ret = 0;
+
+	/* mail_search_args_init() must have been called by now */
+	i_assert(arg->initialized.search_args != NULL);
+
+	p_array_init(&arg->value.seqset, ctx->mail_ctx.args->pool, 64);
+	if (mailbox_search_result_build(ctx->mail_ctx.transaction,
+					arg->initialized.search_args,
+					MAILBOX_SEARCH_RESULT_FLAG_UPDATE |
+					MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC,
+					&arg->value.search_result) < 0)
+		return -1;
+	if (ctx->thread_ctx == NULL) {
+		/* failed earlier */
+		return -1;
+	}
+
+	search_uids = mailbox_search_result_get(arg->value.search_result);
+	if (array_count(search_uids) == 0) {
+		/* search found nothing - no threads can match */
+		return 0;
+	}
+
+	t_array_init(&thread_uids, 128);
+	iter = mail_thread_iterate_init(ctx->thread_ctx,
+					arg->value.thread_type, FALSE);
+	while ((node = mail_thread_iterate_next(iter, &child_iter)) != NULL) {
+		seq_range_array_add(&thread_uids, node->uid);
+		if (child_iter != NULL) {
+			if (search_build_subthread(child_iter,
+						   &thread_uids) < 0)
+				ret = -1;
+		}
+		if (seq_range_array_have_common(&thread_uids, search_uids)) {
+			/* yes, we want this thread */
+			seq_range_array_merge(&arg->value.seqset, &thread_uids);
+		}
+		array_clear(&thread_uids);
+	}
+	if (mail_thread_iterate_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+static int search_build_inthreads(struct index_search_context *ctx,
+				  struct mail_search_arg *arg)
+{
+	int ret = 0;
+
+	for (; arg != NULL; arg = arg->next) {
+		switch (arg->type) {
+		case SEARCH_OR:
+		case SEARCH_SUB:
+			if (search_build_inthreads(ctx, arg->value.subargs) < 0)
+				ret = -1;
+			break;
+		case SEARCH_INTHREAD:
+			if (search_build_inthread_result(ctx, arg) < 0)
+				ret = -1;
+			break;
+		default:
+			break;
+		}
+	}
+	return ret;
+}
+
+static void
+wanted_sort_fields_get(struct mailbox *box,
+		       const enum mail_sort_type *sort_program,
+		       struct mailbox_header_lookup_ctx *wanted_headers,
+		       enum mail_fetch_field *wanted_fields_r,
+		       struct mailbox_header_lookup_ctx **headers_ctx_r)
+{
+	ARRAY_TYPE(const_string) headers;
+	const char *header;
+	unsigned int i;
+
+	*wanted_fields_r = 0;
+	*headers_ctx_r = NULL;
+
+	t_array_init(&headers, 8);
+	for (i = 0; sort_program[i] != MAIL_SORT_END; i++) {
+		header = NULL;
+
+		switch (sort_program[i] & MAIL_SORT_MASK) {
+		case MAIL_SORT_ARRIVAL:
+			*wanted_fields_r |= MAIL_FETCH_RECEIVED_DATE;
+			break;
+		case MAIL_SORT_CC:
+			header = "Cc";
+			break;
+		case MAIL_SORT_DATE:
+			*wanted_fields_r |= MAIL_FETCH_DATE;
+			break;
+		case MAIL_SORT_FROM:
+			header = "From";
+			break;
+		case MAIL_SORT_SIZE:
+			*wanted_fields_r |= MAIL_FETCH_VIRTUAL_SIZE;
+			break;
+		case MAIL_SORT_SUBJECT:
+			header = "Subject";
+			break;
+		case MAIL_SORT_TO:
+			header = "To";
+			break;
+		}
+		if (header != NULL)
+			array_push_back(&headers, &header);
+	}
+
+	if (wanted_headers != NULL) {
+		for (i = 0; wanted_headers->name[i] != NULL; i++)
+			array_push_back(&headers, &wanted_headers->name[i]);
+	}
+
+	if (array_count(&headers) > 0) {
+		array_append_zero(&headers);
+		*headers_ctx_r = mailbox_header_lookup_init(box,
+							array_front(&headers));
+	}
+}
+
+struct mail_search_context *
+index_storage_search_init(struct mailbox_transaction_context *t,
+			  struct mail_search_args *args,
+			  const enum mail_sort_type *sort_program,
+			  enum mail_fetch_field wanted_fields,
+			  struct mailbox_header_lookup_ctx *wanted_headers)
+{
+	struct index_search_context *ctx;
+	struct mailbox_status status;
+
+	ctx = i_new(struct index_search_context, 1);
+	ctx->mail_ctx.transaction = t;
+	ctx->mail_ctx.normalizer = t->box->storage->user->default_normalizer;
+	ctx->box = t->box;
+	ctx->view = t->view;
+	ctx->mail_ctx.args = args;
+	ctx->mail_ctx.sort_program = index_sort_program_init(t, sort_program);
+
+	ctx->mail_ctx.max_mails = t->box->storage->set->mail_prefetch_count + 1;
+	if (ctx->mail_ctx.max_mails == 0)
+		ctx->mail_ctx.max_mails = UINT_MAX;
+	ctx->next_time_check_cost = SEARCH_INITIAL_MAX_COST;
+	i_gettimeofday(&ctx->last_nonblock_timeval);
+
+	mailbox_get_open_status(t->box, STATUS_MESSAGES, &status);
+	ctx->mail_ctx.progress_max = status.messages;
+
+	i_array_init(&ctx->mail_ctx.results, 5);
+	array_create(&ctx->mail_ctx.module_contexts, default_pool,
+		     sizeof(void *), 5);
+	i_array_init(&ctx->mail_ctx.mails, ctx->mail_ctx.max_mails);
+
+	mail_search_args_reset(ctx->mail_ctx.args->args, TRUE);
+	if (args->have_inthreads) {
+		if (mail_thread_init(t->box, NULL, &ctx->thread_ctx) < 0)
+			search_set_failed(ctx);
+		if (search_build_inthreads(ctx, args->args) < 0)
+			search_set_failed(ctx);
+	}
+
+	if (sort_program != NULL) {
+		wanted_sort_fields_get(ctx->box, sort_program, wanted_headers,
+				       &ctx->mail_ctx.wanted_fields,
+				       &ctx->mail_ctx.wanted_headers);
+	} else if (wanted_headers != NULL) {
+		ctx->mail_ctx.wanted_headers = wanted_headers;
+		mailbox_header_lookup_ref(wanted_headers);
+	}
+	ctx->mail_ctx.wanted_fields |= wanted_fields;
+
+	search_get_seqset(ctx, status.messages, args->args);
+	(void)mail_search_args_foreach(args->args, search_init_arg, ctx);
+
+	/* Need to reset results for match_always cases */
+	mail_search_args_reset(ctx->mail_ctx.args->args, FALSE);
+	return &ctx->mail_ctx;
+}
+
+static void ATTR_NULL(2)
+search_arg_deinit(struct mail_search_arg *arg,
+		  struct index_search_context *ctx)
+{
+	switch (arg->type) {
+	case SEARCH_MIMEPART:
+		index_search_mime_arg_deinit(arg, ctx);
+		break;
+	default:
+		if (arg->context != NULL) {
+			struct message_search_context *search_ctx = arg->context;
+			message_search_deinit(&search_ctx);
+			arg->context = NULL;
+		}
+	}
+}
+
+int index_storage_search_deinit(struct mail_search_context *_ctx)
+{
+        struct index_search_context *ctx = (struct index_search_context *)_ctx;
+	struct mail *mail;
+	int ret;
+
+	ret = ctx->failed ? -1 : 0;
+
+	mail_search_args_reset(ctx->mail_ctx.args->args, FALSE);
+	(void)mail_search_args_foreach(ctx->mail_ctx.args->args,
+				       search_arg_deinit, ctx);
+
+	mailbox_header_lookup_unref(&ctx->mail_ctx.wanted_headers);
+	if (ctx->mail_ctx.sort_program != NULL) {
+		if (index_sort_program_deinit(&ctx->mail_ctx.sort_program) < 0)
+			ret = -1;
+	}
+	if (ctx->thread_ctx != NULL)
+		mail_thread_deinit(&ctx->thread_ctx);
+	array_free(&ctx->mail_ctx.results);
+	array_free(&ctx->mail_ctx.module_contexts);
+
+	array_foreach_elem(&ctx->mail_ctx.mails, mail) {
+		struct index_mail *imail = INDEX_MAIL(mail);
+
+		imail->mail.search_mail = FALSE;
+		mail_free(&mail);
+	}
+
+	if (ctx->failed)
+		mail_storage_last_error_pop(ctx->box->storage);
+	array_free(&ctx->mail_ctx.mails);
+	i_free(ctx);
+	return ret;
+}
+
+static unsigned long long
+search_get_cost(struct mailbox_transaction_context *trans)
+{
+	return trans->stats.open_lookup_count * SEARCH_COST_DENTRY +
+		trans->stats.stat_lookup_count * SEARCH_COST_DENTRY +
+		trans->stats.fstat_lookup_count * SEARCH_COST_ATTR +
+		trans->stats.cache_hit_count * SEARCH_COST_CACHE +
+		trans->stats.files_read_count * SEARCH_COST_FILES_READ +
+		(trans->stats.files_read_bytes/1024) * SEARCH_COST_KBYTE;
+}
+
+static int search_match_once(struct index_search_context *ctx)
+{
+	int ret;
+
+	ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
+				       search_cached_arg, ctx);
+	if (ret < 0)
+		ret = search_arg_match_text(ctx->mail_ctx.args->args, ctx);
+	if (ret < 0)
+		ret = index_search_mime_arg_match(ctx->mail_ctx.args->args, ctx);
+	return ret;
+}
+
+static bool search_arg_is_static(struct mail_search_arg *arg)
+{
+	struct mail_search_arg *subarg;
+
+	switch (arg->type) {
+	case SEARCH_OR:
+	case SEARCH_SUB:
+		/* they're static only if all subargs are static */
+		subarg = arg->value.subargs;
+		for (; subarg != NULL; subarg = subarg->next) {
+			if (!search_arg_is_static(subarg))
+				return FALSE;
+		}
+		return TRUE;
+	case SEARCH_SEQSET:
+		/* changes between syncs, but we can't really handle this
+		   currently. seqsets should be converted to uidsets first. */
+	case SEARCH_FLAGS:
+	case SEARCH_KEYWORDS:
+	case SEARCH_MODSEQ:
+	case SEARCH_INTHREAD:
+		break;
+	case SEARCH_ALL:
+	case SEARCH_UIDSET:
+	case SEARCH_BEFORE:
+	case SEARCH_ON:
+	case SEARCH_SINCE:
+	case SEARCH_SMALLER:
+	case SEARCH_LARGER:
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+	case SEARCH_BODY:
+	case SEARCH_TEXT:
+	case SEARCH_SAVEDATESUPPORTED:
+	case SEARCH_GUID:
+	case SEARCH_MAILBOX:
+	case SEARCH_MAILBOX_GUID:
+	case SEARCH_MAILBOX_GLOB:
+	case SEARCH_REAL_UID:
+	case SEARCH_MIMEPART:
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static void search_set_static_matches(struct mail_search_arg *arg)
+{
+	for (; arg != NULL; arg = arg->next) {
+		if (search_arg_is_static(arg))
+			arg->result = 1;
+	}
+}
+
+static bool search_has_static_nonmatches(struct mail_search_arg *arg)
+{
+	for (; arg != NULL; arg = arg->next) {
+		if (arg->result == 0 && search_arg_is_static(arg))
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static void search_match_finish(struct index_search_context *ctx, int match)
+{
+	if (match == 0 &&
+	    search_has_static_nonmatches(ctx->mail_ctx.args->args)) {
+		/* if there are saved search results remember
+		   that this message never matches */
+		mailbox_search_results_never(&ctx->mail_ctx,
+					     ctx->cur_mail->uid);
+	}
+}
+
+static int search_match_next(struct index_search_context *ctx)
+{
+	static enum mail_lookup_abort cache_lookups[] = {
+		MAIL_LOOKUP_ABORT_NOT_IN_CACHE,
+		MAIL_LOOKUP_ABORT_READ_MAIL,
+		MAIL_LOOKUP_ABORT_NEVER
+	};
+	unsigned int i, n = N_ELEMENTS(cache_lookups);
+	int ret = -1;
+
+	if (ctx->have_mailbox_args) {
+		/* check that the mailbox name matches.
+		   this makes sense only with virtual mailboxes. */
+		ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
+					       search_mailbox_arg, ctx);
+	}
+
+	/* avoid doing extra work for as long as possible */
+	if (ctx->mail_ctx.max_mails > 1) {
+		/* we're doing prefetching. if we have to read the mail,
+		   do a prefetch first and the final search later */
+		n--;
+	}
+
+	i_assert(ctx->cur_mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER);
+	for (i = 0; i < n && ret < 0; i++) {
+		ctx->cur_mail->lookup_abort = cache_lookups[i];
+		T_BEGIN {
+			ret = search_match_once(ctx);
+		} T_END;
+	}
+	ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+	search_match_finish(ctx, ret);
+	return ret;
+}
+
+static void index_storage_search_notify(struct mailbox *box,
+					struct index_search_context *ctx)
+{
+	float percentage;
+	unsigned int msecs, secs;
+
+	if (ctx->last_notify.tv_sec == 0) {
+		/* set the search time in here, in case a plugin
+		   already spent some time indexing the mailbox */
+		ctx->search_start_time = ioloop_timeval;
+	} else if (box->storage->callbacks.notify_ok != NULL &&
+		   !ctx->mail_ctx.progress_hidden) {
+		percentage = ctx->mail_ctx.progress_cur * 100.0 /
+			ctx->mail_ctx.progress_max;
+		msecs = timeval_diff_msecs(&ioloop_timeval,
+					   &ctx->search_start_time);
+		secs = (msecs / (percentage / 100.0) - msecs) / 1000;
+
+		T_BEGIN {
+			const char *text;
+
+			text = t_strdup_printf("Searched %d%% of the mailbox, "
+					       "ETA %d:%02d", (int)percentage,
+					       secs/60, secs%60);
+			box->storage->callbacks.
+				notify_ok(box, text,
+					  box->storage->callback_context);
+		} T_END;
+	}
+	ctx->last_notify = ioloop_timeval;
+}
+
+static bool search_would_block(struct index_search_context *ctx)
+{
+	struct timeval now;
+	unsigned long long guess_cost;
+	long long usecs;
+	bool ret;
+
+	if (ctx->cost < ctx->next_time_check_cost)
+		return FALSE;
+
+	i_gettimeofday(&now);
+
+	usecs = timeval_diff_usecs(&now, &ctx->last_nonblock_timeval);
+	if (usecs < 0) {
+		/* clock moved backwards. */
+		ctx->last_nonblock_timeval = now;
+		ctx->next_time_check_cost = SEARCH_INITIAL_MAX_COST;
+		return TRUE;
+	} else if (usecs < SEARCH_MIN_NONBLOCK_USECS) {
+		/* not finished yet. estimate the next time lookup */
+		ret = FALSE;
+	} else {
+		/* done, or close enough anyway */
+		ctx->last_nonblock_timeval = now;
+		ret = TRUE;
+	}
+	guess_cost = ctx->cost *
+		(SEARCH_MAX_NONBLOCK_USECS / (double)usecs);
+	if (usecs < SEARCH_RECALC_MIN_USECS) {
+		/* the estimate may not be very good since we spent
+		   so little time doing this search. don't allow huge changes
+		   to the guess, but allow anyway large enough so that we can
+		   move to right direction. */
+		if (guess_cost > ctx->next_time_check_cost*3)
+			guess_cost = ctx->next_time_check_cost*3;
+		else if (guess_cost < ctx->next_time_check_cost/3)
+			guess_cost = ctx->next_time_check_cost/3;
+	}
+	if (ret)
+		ctx->cost = 0;
+	ctx->next_time_check_cost = guess_cost;
+	return ret;
+}
+
+int index_storage_search_next_match_mail(struct mail_search_context *_ctx,
+					 struct mail *mail)
+{
+	struct index_search_context *ctx =
+		container_of(_ctx, struct index_search_context, mail_ctx);
+	struct index_mail *imail = INDEX_MAIL(mail);
+	int match;
+
+	ctx->cur_mail = mail;
+	/* mail's access_type is SEARCH only while using it to process
+	   the search query. afterwards the mail can still be accessed
+	   for fetching. */
+	ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_SEARCH;
+	T_BEGIN {
+		match = search_match_next(ctx);
+	} T_END;
+	ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_DEFAULT;
+	ctx->cur_mail = NULL;
+
+	i_assert(imail->data.search_results == NULL);
+	if (match < 0) {
+		/* result isn't known yet, do a prefetch and
+		   finish later */
+		imail->data.search_results =
+			buffer_create_dynamic(imail->mail.data_pool, 64);
+		mail_search_args_result_serialize(_ctx->args,
+						  imail->data.search_results);
+	}
+
+	mail_search_args_reset(_ctx->args->args, FALSE);
+
+	if (match != 0) {
+		/* either matched or result is still unknown.
+		   anyway we're far enough now that we probably want
+		   to update the access_parts. the only problem here is
+		   if searching would want fewer access_parts than the
+		   fetching part, but that's probably not a big problem
+		   usually. */
+		index_mail_update_access_parts_pre(mail);
+		return 1;
+	}
+
+	/* non-match */
+	if (_ctx->args->stop_on_nonmatch)
+		return -1;
+	return 0;
+}
+
+static int search_more_with_mail(struct index_search_context *ctx,
+				 struct mail *mail)
+{
+	struct mail_search_context *_ctx = &ctx->mail_ctx;
+	struct mailbox *box = _ctx->transaction->box;
+	unsigned long long cost1, cost2;
+	int ret;
+
+	if (search_would_block(ctx)) {
+		/* this lookup is useful when a large number of
+		   messages match */
+		return 0;
+	}
+
+	if (ioloop_time - ctx->last_notify.tv_sec >=
+	    SEARCH_NOTIFY_INTERVAL_SECS)
+		index_storage_search_notify(box, ctx);
+
+	mail_search_args_reset(_ctx->args->args, FALSE);
+
+	cost1 = search_get_cost(mail->transaction);
+	ret = -1;
+	while (box->v.search_next_update_seq(_ctx)) {
+		mail_set_seq(mail, _ctx->seq);
+
+		ret = box->v.search_next_match_mail(_ctx, mail);
+		if (ret != 0)
+			break;
+
+		cost2 = search_get_cost(mail->transaction);
+		ctx->cost += cost2 - cost1;
+		cost1 = cost2;
+
+		if (search_would_block(ctx))
+			break;
+		ret = -1;
+	}
+	cost2 = search_get_cost(mail->transaction);
+	ctx->cost += cost2 - cost1;
+	return ret;
+}
+
+struct mail *index_search_get_mail(struct index_search_context *ctx)
+{
+	struct index_mail *imail;
+	struct mail *const *mails, *mail;
+	unsigned int count;
+
+	if (ctx->mail_ctx.unused_mail_idx == ctx->mail_ctx.max_mails)
+		return NULL;
+
+	mails = array_get(&ctx->mail_ctx.mails, &count);
+	if (ctx->mail_ctx.unused_mail_idx < count)
+		return mails[ctx->mail_ctx.unused_mail_idx];
+
+	mail = mail_alloc(ctx->mail_ctx.transaction,
+			  ctx->mail_ctx.wanted_fields,
+			  ctx->mail_ctx.wanted_headers);
+	imail = INDEX_MAIL(mail);
+	imail->mail.search_mail = TRUE;
+	ctx->mail_ctx.transaction->stats_track = TRUE;
+
+	array_push_back(&ctx->mail_ctx.mails, &mail);
+	return mail;
+}
+
+static int search_more_with_prefetching(struct index_search_context *ctx,
+					struct mail **mail_r)
+{
+	struct mail *mail, *const *mails;
+	unsigned int count;
+	int ret = 0;
+
+	while ((mail = index_search_get_mail(ctx)) != NULL) {
+		T_BEGIN {
+			ret = search_more_with_mail(ctx, mail);
+		} T_END;
+		if (ret <= 0)
+			break;
+
+		if (ctx->mail_ctx.sort_program != NULL) {
+			/* don't prefetch when using a sort program,
+			   since the mails' access order will change */
+			i_assert(ctx->mail_ctx.unused_mail_idx == 0);
+			*mail_r = mail;
+			return 1;
+		}
+		if (mail_prefetch(mail) && ctx->mail_ctx.unused_mail_idx == 0) {
+			/* no prefetching done, return it immediately */
+			*mail_r = mail;
+			return 1;
+		}
+		ctx->mail_ctx.unused_mail_idx++;
+	}
+
+	if (mail != NULL) {
+		if (ret == 0) {
+			/* wait */
+			return 0;
+		}
+		i_assert(ret < 0);
+		if (ctx->mail_ctx.unused_mail_idx == 0) {
+			/* finished */
+			return -1;
+		}
+	} else {
+		/* prefetch buffer is full. */
+	}
+
+	/* return the next message */
+	i_assert(ctx->mail_ctx.unused_mail_idx > 0);
+
+	mails = array_get(&ctx->mail_ctx.mails, &count);
+	*mail_r = mails[0];
+	if (--ctx->mail_ctx.unused_mail_idx > 0) {
+		array_pop_front(&ctx->mail_ctx.mails);
+		array_push_back(&ctx->mail_ctx.mails, mail_r);
+	}
+	index_mail_update_access_parts_post(*mail_r);
+	return 1;
+}
+
+static bool search_finish_prefetch(struct index_search_context *ctx,
+				   struct index_mail *imail)
+{
+	int ret;
+
+	i_assert(imail->mail.mail.lookup_abort == MAIL_LOOKUP_ABORT_NEVER);
+
+	ctx->cur_mail = &imail->mail.mail;
+	ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_SEARCH;
+	mail_search_args_result_deserialize(ctx->mail_ctx.args,
+					    imail->data.search_results->data,
+					    imail->data.search_results->used);
+	T_BEGIN {
+		ret = search_match_once(ctx);
+		search_match_finish(ctx, ret);
+	} T_END;
+	ctx->cur_mail->access_type = MAIL_ACCESS_TYPE_DEFAULT;
+	ctx->cur_mail = NULL;
+	return ret > 0;
+}
+
+static int search_more(struct index_search_context *ctx,
+		       struct mail **mail_r)
+{
+	struct index_mail *imail;
+	int ret;
+
+	while ((ret = search_more_with_prefetching(ctx, mail_r)) > 0) {
+		imail = INDEX_MAIL(*mail_r);
+		if (imail->data.search_results == NULL)
+			break;
+
+		/* prefetch running - searching wasn't finished yet */
+		if (search_finish_prefetch(ctx, imail))
+			break;
+		/* search finished as non-match */
+		if (ctx->mail_ctx.args->stop_on_nonmatch) {
+			ret = -1;
+			break;
+		}
+	}
+	return ret;
+}
+
+bool index_storage_search_next_nonblock(struct mail_search_context *_ctx,
+					struct mail **mail_r, bool *tryagain_r)
+{
+        struct index_search_context *ctx = (struct index_search_context *)_ctx;
+	struct mail *mail, *const *mailp;
+	uint32_t seq;
+	int ret;
+
+	*tryagain_r = FALSE;
+
+	if (_ctx->sort_program == NULL) {
+		ret = search_more(ctx, &mail);
+		if (ret == 0) {
+			*tryagain_r = TRUE;
+			return FALSE;
+		}
+		if (ret < 0)
+			return FALSE;
+		*mail_r = mail;
+		return TRUE;
+	}
+
+	if (!ctx->sorted) {
+		while ((ret = search_more(ctx, &mail)) > 0)
+			index_sort_list_add(_ctx->sort_program, mail);
+
+		if (ret == 0) {
+			*tryagain_r = TRUE;
+			return FALSE;
+		}
+		/* finished searching the messages. now sort them and start
+		   returning the messages. */
+		ctx->sorted = TRUE;
+		index_sort_list_finish(_ctx->sort_program);
+	}
+
+	/* everything searched at this point already. just returning
+	   matches from sort list. FIXME: we could do prefetching here also. */
+	if (!index_sort_list_next(_ctx->sort_program, &seq))
+		return FALSE;
+
+	mailp = array_front(&ctx->mail_ctx.mails);
+	mail_set_seq(*mailp, seq);
+	index_mail_update_access_parts_pre(*mailp);
+	index_mail_update_access_parts_post(*mailp);
+	*mail_r = *mailp;
+	return TRUE;
+}
+
+bool index_storage_search_next_update_seq(struct mail_search_context *_ctx)
+{
+        struct index_search_context *ctx = (struct index_search_context *)_ctx;
+	uint32_t uid;
+	int ret;
+
+	if (_ctx->seq == 0) {
+		/* first time */
+		_ctx->seq = ctx->seq1;
+	} else {
+		_ctx->seq++;
+	}
+
+	if (!ctx->have_seqsets && !ctx->have_index_args &&
+	    !ctx->have_nonmatch_always && _ctx->update_result == NULL) {
+		_ctx->progress_cur = _ctx->seq;
+		return _ctx->seq <= ctx->seq2;
+	}
+
+	ret = 0;
+	while (_ctx->seq <= ctx->seq2) {
+		/* check if the sequence matches */
+		ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
+					       search_seqset_arg, ctx);
+		if (ret != 0 && ctx->have_index_args) {
+			/* check if flags/keywords match before anything else
+			   is done. mail_set_seq() can be a bit slow. */
+			ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
+						       search_index_arg, ctx);
+		}
+		if (ret != 0 && _ctx->update_result != NULL) {
+			/* see if this message never matches */
+			mail_index_lookup_uid(ctx->view, _ctx->seq, &uid);
+			if (seq_range_exists(&_ctx->update_result->never_uids,
+					     uid))
+				ret = 0;
+		}
+		if (ret != 0)
+			break;
+
+		/* doesn't, try next one */
+		_ctx->seq++;
+		mail_search_args_reset(ctx->mail_ctx.args->args, FALSE);
+	}
+
+	if (ret != 0 && _ctx->update_result != NULL) {
+		mail_index_lookup_uid(ctx->view, _ctx->seq, &uid);
+		if (seq_range_exists(&_ctx->update_result->uids, uid)) {
+			/* we already know that the static data
+			   matches. mark it as such. */
+			search_set_static_matches(_ctx->args->args);
+		}
+	}
+	ctx->mail_ctx.progress_cur = _ctx->seq;
+	return ret != 0;
+}
diff --git a/src/lib-storage/index/index-sort-private.h b/src/lib-storage/index/index-sort-private.h
new file mode 100644
index 0000000..3dd0772
--- /dev/null
+++ b/src/lib-storage/index/index-sort-private.h
@@ -0,0 +1,35 @@
+#ifndef INDEX_SORT_PRIVATE_H
+#define INDEX_SORT_PRIVATE_H
+
+#include "index-sort.h"
+
+struct mail_search_sort_program {
+	struct mailbox_transaction_context *t;
+	enum mail_sort_type sort_program[MAX_SORT_PROGRAM_SIZE];
+	struct mail *temp_mail;
+	unsigned int slow_mails_left;
+
+	void (*sort_list_add)(struct mail_search_sort_program *program,
+			      struct mail *mail);
+	void (*sort_list_finish)(struct mail_search_sort_program *program);
+	void *context;
+
+	ARRAY_TYPE(uint32_t) seqs;
+	unsigned int iter_idx;
+
+	bool failed;
+};
+
+/* Returns 1 on success, 0 if mail is already expunged, -1 on other errors. */
+int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq,
+			  enum mail_sort_type sort_type, string_t *dest);
+int index_sort_node_cmp_type(struct mail_search_sort_program *program,
+			     const enum mail_sort_type *sort_program,
+			     uint32_t seq1, uint32_t seq2);
+
+void index_sort_list_init_string(struct mail_search_sort_program *program);
+void index_sort_list_add_string(struct mail_search_sort_program *program,
+				struct mail *mail);
+void index_sort_list_finish_string(struct mail_search_sort_program *program);
+
+#endif
diff --git a/src/lib-storage/index/index-sort-string.c b/src/lib-storage/index/index-sort-string.c
new file mode 100644
index 0000000..c518e78
--- /dev/null
+++ b/src/lib-storage/index/index-sort-string.c
@@ -0,0 +1,944 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+/* The idea is that we use 32bit integers for string sort IDs which specify
+   the sort order for primary sort condition. The whole 32bit integer space is
+   used and whenever adding a string, the available space is halved and the new
+   ID is added in the middle. For example if we add one mail the first time, it
+   gets ID 2^31. If we then add two mails which are sorted before the first
+   one, they get IDs 2^31/3 and 2^31/3*2. Once we run out of the available
+   space between IDs, more space is made by renumbering some IDs.
+*/
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "index-storage.h"
+#include "index-sort-private.h"
+
+
+struct mail_sort_node {
+	uint32_t seq:29;
+	bool wanted:1;
+	bool no_update:1;
+	bool sort_id_changed:1;
+	uint32_t sort_id;
+};
+ARRAY_DEFINE_TYPE(mail_sort_node, struct mail_sort_node);
+
+struct sort_string_context {
+	struct mail_search_sort_program *program;
+	const char *primary_sort_name;
+
+	ARRAY_TYPE(mail_sort_node) zero_nodes, nonzero_nodes, sorted_nodes;
+	const char **sort_strings;
+	pool_t sort_string_pool;
+	unsigned int first_missing_sort_id_idx;
+
+	uint32_t ext_id, last_seq, highest_reset_id, prev_seq;
+	uint32_t lowest_nonexpunged_zero;
+
+	bool regetting:1;
+	bool have_all_wanted:1;
+	bool no_writing:1;
+	bool reverse:1;
+	bool seqs_nonsorted:1;
+	bool broken:1;
+	bool failed:1;
+};
+
+static struct sort_string_context *static_zero_cmp_context;
+
+static void index_sort_list_reset_broken(struct sort_string_context *ctx,
+					 const char *reason);
+static void index_sort_node_add(struct sort_string_context *ctx,
+				struct mail_sort_node *node);
+
+void index_sort_list_init_string(struct mail_search_sort_program *program)
+{
+	struct sort_string_context *ctx;
+	const char *name;
+
+	switch (program->sort_program[0] & MAIL_SORT_MASK) {
+	case MAIL_SORT_CC:
+		name = "sort-c";
+		break;
+	case MAIL_SORT_FROM:
+		name = "sort-f";
+		break;
+	case MAIL_SORT_SUBJECT:
+		name = "sort-s";
+		break;
+	case MAIL_SORT_TO:
+		name = "sort-t";
+		break;
+	case MAIL_SORT_DISPLAYFROM:
+		name = "sort-df";
+		break;
+	case MAIL_SORT_DISPLAYTO:
+		name = "sort-dt";
+		break;
+	default:
+		i_unreached();
+	}
+
+	program->context = ctx = i_new(struct sort_string_context, 1);
+	ctx->reverse = (program->sort_program[0] & MAIL_SORT_FLAG_REVERSE) != 0;
+	ctx->program = program;
+	ctx->primary_sort_name = name;
+	ctx->ext_id = mail_index_ext_register(program->t->box->index, name, 0,
+					      sizeof(uint32_t),
+					      sizeof(uint32_t));
+	i_array_init(&ctx->zero_nodes, 128);
+	i_array_init(&ctx->nonzero_nodes, 128);
+}
+
+static int sort_node_seq_cmp(const struct mail_sort_node *n1,
+			     const struct mail_sort_node *n2)
+{
+	if (n1->seq < n2->seq)
+		return -1;
+	if (n1->seq > n2->seq)
+		return 1;
+	return 0;
+}
+
+static void index_sort_generate_seqs(struct sort_string_context *ctx)
+{
+	struct mail_sort_node *nodes, *nodes2;
+	unsigned int i, j, count, count2;
+	uint32_t seq;
+
+	nodes = array_get_modifiable(&ctx->nonzero_nodes, &count);
+	nodes2 = array_get_modifiable(&ctx->zero_nodes, &count2);
+
+	if (!array_is_created(&ctx->program->seqs))
+		i_array_init(&ctx->program->seqs, count + count2);
+	else
+		array_clear(&ctx->program->seqs);
+
+	for (i = j = 0;;) {
+		if (i < count && j < count2) {
+			if (nodes[i].seq < nodes2[j].seq)
+				seq = nodes[i++].seq;
+			else
+				seq = nodes2[j++].seq;
+		} else if (i < count) {
+			seq = nodes[i++].seq;
+		} else if (j < count2) {
+			seq = nodes2[j++].seq;
+		} else {
+			break;
+		}
+		array_push_back(&ctx->program->seqs, &seq);
+	}
+}
+
+static void index_sort_reget_sort_ids(struct sort_string_context *ctx)
+{
+	struct mail_sort_node node;
+	const uint32_t *seqs;
+	unsigned int i, count;
+
+	i_assert(!ctx->regetting);
+	ctx->regetting = TRUE;
+
+	index_sort_generate_seqs(ctx);
+	array_clear(&ctx->zero_nodes);
+	array_clear(&ctx->nonzero_nodes);
+
+	i_zero(&node);
+	node.wanted = TRUE;
+	seqs = array_get(&ctx->program->seqs, &count);
+	for (i = 0; i < count; i++) {
+		node.seq = seqs[i];
+		index_sort_node_add(ctx, &node);
+	}
+	ctx->regetting = FALSE;
+}
+
+static void index_sort_node_add(struct sort_string_context *ctx,
+				struct mail_sort_node *node)
+{
+	struct mail_index_map *map;
+	const void *data;
+	uint32_t reset_id;
+	bool expunged;
+
+	mail_index_lookup_ext_full(ctx->program->t->view, node->seq,
+				   ctx->ext_id, &map, &data, &expunged);
+	if (expunged) {
+		/* we don't want to update expunged messages' sort IDs */
+		node->no_update = TRUE;
+		/* we can't trust expunged messages' sort IDs. they might be
+		   valid, but it's also possible that sort IDs were updated
+		   and the expunged messages' sort IDs became invalid. we could
+		   use sort ID if we could know the extension's reset_id at the
+		   time of the expunge so we could compare it to
+		   highest_reset_id, but this isn't currently possible. */
+		node->sort_id = 0;
+	} else {
+		node->sort_id = ctx->broken || data == NULL ? 0 :
+			*(const uint32_t *)data;
+		if (node->sort_id == 0) {
+			if (ctx->lowest_nonexpunged_zero > node->seq ||
+			    ctx->lowest_nonexpunged_zero == 0)
+				ctx->lowest_nonexpunged_zero = node->seq;
+		} else if (ctx->lowest_nonexpunged_zero != 0 &&
+			   ctx->lowest_nonexpunged_zero <= node->seq) {
+			uint32_t nonzero_uid, zero_uid;
+
+			mail_index_lookup_uid(ctx->program->t->view,
+					      node->seq, &nonzero_uid);
+			mail_index_lookup_uid(ctx->program->t->view,
+				ctx->lowest_nonexpunged_zero, &zero_uid);
+			index_sort_list_reset_broken(ctx, t_strdup_printf(
+				"sort_id=0 found in the middle "
+				"(uid=%u has sort_id, uid=%u doesn't)",
+				nonzero_uid, zero_uid));
+			ctx->broken = TRUE;
+			node->sort_id = 0;
+		}
+	}
+
+	if (node->sort_id != 0) {
+		/* if reset ID increases, lookup all existing messages' sort
+		   IDs again. if it decreases, ignore the sort ID. */
+		if (!mail_index_ext_get_reset_id(ctx->program->t->view, map,
+						 ctx->ext_id, &reset_id))
+			reset_id = 0;
+		if (reset_id != ctx->highest_reset_id) {
+			if (reset_id < ctx->highest_reset_id) {
+				i_assert(expunged);
+				node->sort_id = 0;
+			} else if (ctx->have_all_wanted) {
+				/* a bit late to start changing the reset_id.
+				   the node lists aren't ordered by sequence
+				   anymore. */
+				node->sort_id = 0;
+				ctx->no_writing = TRUE;
+			} else {
+				ctx->highest_reset_id = reset_id;
+				index_sort_reget_sort_ids(ctx);
+			}
+		}
+	}
+
+	if (node->sort_id == 0)
+		array_push_back(&ctx->zero_nodes, node);
+	else
+		array_push_back(&ctx->nonzero_nodes, node);
+	if (ctx->last_seq < node->seq)
+		ctx->last_seq = node->seq;
+}
+
+void index_sort_list_add_string(struct mail_search_sort_program *program,
+				struct mail *mail)
+{
+	struct sort_string_context *ctx = program->context;
+	struct mail_sort_node node;
+
+	i_zero(&node);
+	node.seq = mail->seq;
+	node.wanted = TRUE;
+
+	if (mail->seq < ctx->prev_seq)
+		ctx->seqs_nonsorted = TRUE;
+	ctx->prev_seq = mail->seq;
+
+	index_sort_node_add(ctx, &node);
+}
+
+static int sort_node_zero_string_cmp(const struct mail_sort_node *n1,
+				     const struct mail_sort_node *n2)
+{
+	struct sort_string_context *ctx = static_zero_cmp_context;
+	int ret;
+
+	ret = strcmp(ctx->sort_strings[n1->seq], ctx->sort_strings[n2->seq]);
+	if (ret != 0)
+		return !ctx->reverse ? ret : -ret;
+
+	return index_sort_node_cmp_type(ctx->program,
+					ctx->program->sort_program + 1,
+					n1->seq, n2->seq);
+}
+
+static void index_sort_zeroes(struct sort_string_context *ctx)
+{
+	enum mail_sort_type sort_type = ctx->program->sort_program[0];
+	string_t *str;
+	pool_t pool;
+	struct mail_sort_node *nodes;
+	unsigned int i, count;
+
+	/* first get all the messages' sort strings. although this takes more
+	   memory, it makes error handling easier and probably also helps
+	   CPU caching. */
+	ctx->sort_strings = i_new(const char *, ctx->last_seq + 1);
+	ctx->sort_string_pool = pool =
+		pool_alloconly_create("sort strings", 1024*64);
+	str = str_new(default_pool, 512);
+	nodes = array_get_modifiable(&ctx->zero_nodes, &count);
+	for (i = 0; i < count; i++) {
+		i_assert(nodes[i].seq <= ctx->last_seq);
+
+		T_BEGIN {
+			if (index_sort_header_get(ctx->program, nodes[i].seq,
+						  sort_type, str) < 0) {
+				nodes[i].no_update = TRUE;
+				ctx->failed = TRUE;
+			}
+			ctx->sort_strings[nodes[i].seq] =
+				str_len(str) == 0 ? "" :
+				p_strdup(pool, str_c(str));
+		} T_END;
+	}
+	str_free(&str);
+
+	/* we have all strings, sort nodes based on them */
+	static_zero_cmp_context = ctx;
+	array_sort(&ctx->zero_nodes, sort_node_zero_string_cmp);
+}
+
+static bool
+index_sort_get_expunged_string(struct sort_string_context *ctx, uint32_t idx,
+			       string_t *str, const char **result_r)
+{
+	enum mail_sort_type sort_type = ctx->program->sort_program[0];
+	const struct mail_sort_node *nodes;
+	const char *result = NULL;
+	unsigned int i, count;
+	uint32_t sort_id;
+
+	/* Look forwards and backwards to see if there are
+	   identical sort_ids. If we do find them, try to get
+	   their sort string and use it to update the rest. */
+	nodes = array_get(&ctx->nonzero_nodes, &count);
+	sort_id = nodes[idx].sort_id;
+	/* If previous sort ID is identical and its sort string is set, we can
+	   trust it. If it's expunged, we already verified that there are no
+	   non-expunged messages. */
+	if (idx > 0 && nodes[idx-1].sort_id == sort_id &&
+	    ctx->sort_strings[nodes[idx].seq] != NULL) {
+		*result_r = ctx->sort_strings[nodes[idx].seq];
+		return TRUE;
+	}
+
+	/* Go forwards as long as there are identical sort IDs. If we find one
+	   that's not expunged, update string table for all messages with
+	   identical sort IDs. */
+	for (i = idx + 1; i < count; i++) {
+		if (nodes[i].sort_id != sort_id)
+			break;
+
+		if (ctx->sort_strings[nodes[i].seq] != NULL) {
+			/* usually we fill all identical sort_ids and this
+			   shouldn't happen, but we can get here if we skipped
+			   over messages when binary searching */
+			result = ctx->sort_strings[nodes[i].seq];
+			break;
+		}
+		if (index_sort_header_get(ctx->program, nodes[i].seq,
+					  sort_type, str) > 0) {
+			result = str_len(str) == 0 ? "" :
+				p_strdup(ctx->sort_string_pool, str_c(str));
+			break;
+		}
+	}
+	if (result == NULL) {
+		/* unknown */
+		return FALSE;
+	}
+
+	/* fill all identical sort_ids with the same value */
+	for (i = idx; i > 0 && nodes[i-1].sort_id == sort_id; i--) ;
+	for (i = idx; i < count && nodes[i].sort_id == sort_id; i++)
+		ctx->sort_strings[nodes[i].seq] = result;
+	*result_r = result;
+	return TRUE;
+}
+
+static bool
+index_sort_get_string(struct sort_string_context *ctx,
+		      uint32_t idx, struct mail_sort_node *node,
+		      const char **str_r)
+{
+	uint32_t seq = node->seq;
+	int ret = 1;
+
+	if (node->no_update) {
+		/* we've already determined that we can't do this lookup */
+		*str_r = ctx->sort_strings[seq];
+		return FALSE;
+	}
+
+	if (ctx->sort_strings[seq] == NULL) T_BEGIN {
+		string_t *str;
+		const char *result;
+
+		str = t_str_new(256);
+		ret = index_sort_header_get(ctx->program, seq,
+					    ctx->program->sort_program[0], str);
+		if (ret < 0)
+			ctx->failed = TRUE;
+		else if (ret == 0) {
+			if (!index_sort_get_expunged_string(ctx, idx, str, &result))
+				ctx->sort_strings[seq] = "";
+			else {
+				/* found the expunged string - return success */
+				ctx->sort_strings[seq] = result;
+				ret = 1;
+			}
+		} else {
+			ctx->sort_strings[seq] = str_len(str) == 0 ? "" :
+				p_strdup(ctx->sort_string_pool, str_c(str));
+		}
+	} T_END;
+
+	if (ret <= 0)
+		node->no_update = TRUE;
+	*str_r = ctx->sort_strings[seq];
+	return ret > 0;
+}
+
+static void
+index_sort_bsearch(struct sort_string_context *ctx, const char *key,
+		   unsigned int start_idx, unsigned int *idx_r,
+		   const char **prev_str_r)
+{
+	struct mail_sort_node *nodes;
+	const char *str, *str2;
+	unsigned int idx, left_idx, right_idx, prev;
+	int ret;
+
+	nodes = array_get_modifiable(&ctx->nonzero_nodes, &right_idx);
+	i_assert(right_idx < INT_MAX);
+	idx = left_idx = start_idx;
+	while (left_idx < right_idx) {
+		idx = (left_idx + right_idx) / 2;
+		if (index_sort_get_string(ctx, idx, &nodes[idx], &str))
+			ret = strcmp(key, str);
+		else {
+			/* put expunged (and otherwise failed) messages first */
+			ret = 1;
+			for (prev = idx; prev > 0; ) {
+				prev--;
+				if (index_sort_get_string(ctx, prev,
+							  &nodes[prev],
+							  &str2)) {
+					ret = strcmp(key, str2);
+					if (ret <= 0) {
+						idx = prev;
+						str = str2;
+					}
+					break;
+				}
+			}
+		}
+		if (ret > 0)
+			left_idx = idx+1;
+		else if (ret < 0)
+			right_idx = idx;
+		else {
+			*idx_r = idx + 1;
+			*prev_str_r = str;
+			return;
+		}
+	}
+
+	if (left_idx > idx)
+		idx++;
+
+	*idx_r = idx;
+	if (idx > start_idx) {
+		bool success;
+
+		prev = idx;
+		do {
+			prev--;
+			success = index_sort_get_string(ctx, prev,
+							&nodes[prev], &str2);
+		} while (!success && prev > 0 &&
+			 nodes[prev-1].sort_id == nodes[prev].sort_id);
+		*prev_str_r = str2;
+	}
+}
+
+static void index_sort_merge(struct sort_string_context *ctx)
+{
+	struct mail_sort_node *znodes, *nznodes;
+	const char *zstr, *nzstr, *prev_str;
+	unsigned int zpos, nzpos, nz_next_pos, zcount, nzcount;
+	int ret;
+
+	/* both zero_nodes and nonzero_nodes are sorted. we'll now just have
+	   to merge them together. use sorted_nodes as the result array. */
+	i_array_init(&ctx->sorted_nodes, array_count(&ctx->nonzero_nodes) +
+		     array_count(&ctx->zero_nodes));
+
+	znodes = array_get_modifiable(&ctx->zero_nodes, &zcount);
+	nznodes = array_get_modifiable(&ctx->nonzero_nodes, &nzcount);
+
+	prev_str = NULL;
+	for (zpos = nzpos = 0; zpos < zcount && nzpos < nzcount; ) {
+		zstr = ctx->sort_strings[znodes[zpos].seq];
+		if (index_sort_get_string(ctx, nzpos, &nznodes[nzpos], &nzstr))
+			ret = strcmp(zstr, nzstr);
+		else if (prev_str != NULL && strcmp(zstr, prev_str) == 0) {
+			/* identical to previous message, must keep them
+			   together */
+			ret = -1;
+		} else {
+			/* we can't be yet sure about the order, but future
+			   nznodes may reveal that the znode must be added
+			   later. if future nznodes don't reveal that, we have
+			   no idea about these nodes' order. so just always
+			   put the expunged message first. */
+			ret = 1;
+		}
+
+		if (ret == 0) {
+			ret = index_sort_node_cmp_type(ctx->program,
+					ctx->program->sort_program + 1,
+					znodes[zpos].seq, nznodes[nzpos].seq);
+		}
+		if (ret <= 0) {
+			array_push_back(&ctx->sorted_nodes, &znodes[zpos]);
+			prev_str = zstr;
+			zpos++;
+		} else {
+			array_push_back(&ctx->sorted_nodes, &nznodes[nzpos]);
+			prev_str = nzstr;
+			nzpos++;
+
+			/* avoid looking up all existing messages' strings by
+			   binary searching the next zero-node position. don't
+			   bother if it looks like more work than linear
+			   scanning. */
+			if (zcount - zpos < (nzcount - nzpos)/2) {
+				index_sort_bsearch(ctx, zstr, nzpos,
+						   &nz_next_pos, &prev_str);
+				array_append(&ctx->sorted_nodes,
+					     &nznodes[nzpos],
+					     nz_next_pos - nzpos);
+				nzpos = nz_next_pos;
+			}
+		}
+	}
+	/* only one of zero_nodes and nonzero_nodes can be non-empty now */
+	for (; zpos < zcount; zpos++)
+		array_push_back(&ctx->sorted_nodes, &znodes[zpos]);
+	for (; nzpos < nzcount; nzpos++)
+		array_push_back(&ctx->sorted_nodes, &nznodes[nzpos]);
+
+	/* future index_sort_get_string() calls use ctx->nonzero_nodes, but we
+	   use only ctx->sorted_nodes. make them identical. */
+	array_free(&ctx->nonzero_nodes);
+	ctx->nonzero_nodes = ctx->sorted_nodes;
+}
+
+static int
+index_sort_add_ids_range(struct sort_string_context *ctx,
+			 unsigned int left_idx, unsigned int right_idx,
+			 const char **reason_r)
+{
+
+	struct mail_sort_node *nodes;
+	unsigned int i, count, rightmost_idx, skip;
+	const char *left_str = NULL, *right_str = NULL, *str = NULL;
+	uint32_t left_sort_id, right_sort_id, diff, left_str_idx = 0;
+	bool no_left_str = FALSE, no_right_str = FALSE;
+	int ret;
+
+	nodes = array_get_modifiable(&ctx->sorted_nodes, &count);
+	rightmost_idx = count - 1;
+
+	/* get the sort IDs from left and right */
+	left_sort_id = nodes[left_idx].sort_id;
+	right_sort_id = nodes[right_idx].sort_id;
+	/* check if all of them should have the same sort IDs. we don't want
+	   to hit the renumbering code in that situation. */
+	if (left_sort_id == right_sort_id && left_sort_id != 0) {
+		/* they should all have the same sort ID */
+		for (i = left_idx + 1; i < right_idx; i++) {
+			nodes[i].sort_id = left_sort_id;
+			nodes[i].sort_id_changed = TRUE;
+		}
+		return 0;
+	}
+
+	if (left_sort_id == 0) {
+		i_assert(left_idx == 0);
+		left_sort_id = 1;
+	}
+	if (right_sort_id == 0) {
+		i_assert(right_idx == rightmost_idx);
+		right_sort_id = (uint32_t)-1;
+	}
+	i_assert(left_sort_id <= right_sort_id);
+
+	diff = right_sort_id - left_sort_id;
+	while (diff / (right_idx-left_idx + 2) == 0) {
+		/* we most likely don't have enough space. we have to
+		   renumber some of the existing sort IDs. do this by
+		   widening the area we're giving sort IDs. */
+		while (left_idx > 0) {
+			if (nodes[--left_idx].sort_id != left_sort_id) {
+				left_sort_id = nodes[left_idx].sort_id;
+				if (left_sort_id == 0) {
+					i_assert(left_idx == 0);
+					left_sort_id = 1;
+				}
+				break;
+			}
+		}
+
+		while (right_idx < rightmost_idx) {
+			right_idx++;
+			if (nodes[right_idx].sort_id > right_sort_id)
+				break;
+		}
+		right_sort_id = nodes[right_idx].sort_id;
+		if (right_sort_id == 0) {
+			i_assert(right_idx == rightmost_idx);
+			right_sort_id = (uint32_t)-1;
+		}
+		i_assert(left_sort_id <= right_sort_id);
+
+		if (diff == right_sort_id - left_sort_id) {
+			/* we did nothing, but there's still not enough space.
+			   have to renumber the leftmost/rightmost node(s) */
+			i_assert(left_idx == 0 && right_idx == rightmost_idx);
+			if (left_sort_id > 1) {
+				left_sort_id = 1;
+				no_left_str = TRUE;
+			} else {
+				i_assert(right_sort_id != (uint32_t)-1);
+				right_sort_id = (uint32_t)-1;
+				no_right_str = TRUE;
+			}
+		}
+		diff = right_sort_id - left_sort_id;
+	}
+
+	if (nodes[left_idx].sort_id != 0 && !no_left_str) {
+		if (!index_sort_get_string(ctx, left_idx,
+					   &nodes[left_idx], &left_str)) {
+			/* not equivalent with any message */
+			left_str = NULL;
+		} else {
+			left_str_idx = left_idx;
+		}
+		left_idx++;
+	}
+	if (nodes[right_idx].sort_id != 0 && !no_right_str) {
+		if (!index_sort_get_string(ctx, right_idx,
+					   &nodes[right_idx], &right_str)) {
+			/* not equivalent with any message */
+			right_str = NULL;
+		}
+		right_idx--;
+	}
+	i_assert(left_idx <= right_idx);
+
+	/* give (new) sort IDs to all nodes in left_idx..right_idx range.
+	   divide the available space so that each message gets an equal sized
+	   share. some messages' sort strings may be equivalent, so give them
+	   the same sort IDs. */
+	for (i = left_idx; i <= right_idx; i++) {
+		if (!index_sort_get_string(ctx, i, &nodes[i], &str)) {
+			/* it doesn't really matter what we give to this
+			   message, since it's only temporary and we don't
+			   know its correct position anyway. so let's assume
+			   it's equivalent to previous message. */
+			nodes[i].sort_id = left_sort_id;
+			continue;
+		}
+
+		ret = left_str == NULL ? 1 : strcmp(str, left_str);
+		if (ret <= 0) {
+			if (ret < 0) {
+				/* broken sort_ids */
+				uint32_t str_uid, left_str_uid;
+
+				mail_index_lookup_uid(ctx->program->t->view,
+						      nodes[i].seq, &str_uid);
+				mail_index_lookup_uid(ctx->program->t->view,
+					nodes[left_str_idx].seq, &left_str_uid);
+				*reason_r = t_strdup_printf(
+					"(idx=%u, seq=%u, uid=%u) '%s' < left string (idx=%u, seq=%u, uid=%u) '%s'",
+					i, nodes[i].seq, str_uid, str,
+					left_str_idx, nodes[left_str_idx].seq, left_str_uid, left_str);
+				return -1;
+			}
+			nodes[i].sort_id = left_sort_id;
+		} else if (right_str != NULL && strcmp(str, right_str) == 0) {
+			/* the rest of the sort IDs should be the same */
+			nodes[i].sort_id = right_sort_id;
+			left_sort_id = right_sort_id;
+		} else {
+			/* divide the available space equally. leave the same
+			   sized space also between the first and the last
+			   messages */
+			skip = (right_sort_id - left_sort_id) /
+				(right_idx - i + 2);
+			if (skip == 0) {
+				/* broken sort IDs (we previously assigned
+				   left_sort_id=right_sort_id) */
+				uint32_t uid;
+				mail_index_lookup_uid(ctx->program->t->view,
+						      nodes[i].seq, &uid);
+				*reason_r = t_strdup_printf(
+					"no sort_id space for uid=%u", uid);
+				return -1;
+			}
+			left_sort_id += skip;
+			i_assert(left_sort_id < right_sort_id);
+
+			nodes[i].sort_id = left_sort_id;
+			left_str = str;
+			left_str_idx = i;
+		}
+		nodes[i].sort_id_changed = TRUE;
+	}
+	i_assert(str != NULL);
+
+	if (right_str == NULL || strcmp(str, right_str) < 0 ||
+	    (strcmp(str, right_str) == 0 &&
+	     nodes[i-1].sort_id == right_sort_id))
+		return 0;
+
+	*reason_r = t_strdup_printf("Invalid sort_id order ('%s' > '%s')",
+				    str, right_str);
+	return -1;
+}
+
+static int
+index_sort_add_sort_ids(struct sort_string_context *ctx, const char **reason_r)
+{
+	const struct mail_sort_node *nodes;
+	unsigned int i, left_idx, right_idx, count;
+
+	nodes = array_get(&ctx->sorted_nodes, &count);
+	for (i = 0; i < count; i++) {
+		if (nodes[i].sort_id != 0)
+			continue;
+
+		/* get the range for all sort_id=0 nodes. include the nodes
+		   left and right of the range as well */
+		for (right_idx = i + 1; right_idx < count; right_idx++) {
+			if (nodes[right_idx].sort_id != 0)
+				break;
+		}
+		if (right_idx == count)
+			right_idx--;
+		left_idx = i == 0 ? 0 : i - 1;
+		if (index_sort_add_ids_range(ctx, left_idx, right_idx, reason_r) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static void index_sort_write_changed_sort_ids(struct sort_string_context *ctx)
+{
+	struct mail_index_transaction *itrans = ctx->program->t->itrans;
+	uint32_t ext_id = ctx->ext_id;
+	const struct mail_sort_node *nodes;
+	unsigned int i, count;
+	uint32_t lowest_failed_seq;
+
+	if (ctx->no_writing) {
+		/* our reset_id is already stale - don't even bother
+		   trying to write */
+		return;
+	}
+
+	mail_index_ext_reset_inc(itrans, ext_id,
+				 ctx->highest_reset_id, FALSE);
+
+	/* We require that there aren't sort_id=0 gaps in the middle of the
+	   mails. At this point they could exist though, because some of the
+	   mail lookups may have failed. Failures due to expunges don't matter,
+	   because on the next lookup those mails will be lost anyway.
+	   Otherwise, make sure we don't write those gaps out
+
+	   First find the lowest non-expunged mail that has no_update set. */
+	nodes = array_get_modifiable(&ctx->sorted_nodes, &count);
+	lowest_failed_seq = (uint32_t)-1;
+	for (i = 0; i < count; i++) {
+		uint32_t seq = nodes[i].seq;
+
+		if (nodes[i].no_update && lowest_failed_seq > seq &&
+		    !mail_index_is_expunged(ctx->program->t->view, seq))
+			lowest_failed_seq = seq;
+	}
+
+	/* add the missing sort IDs to index, but only for those sequences
+	   that are below lowest_failed_seq */
+	nodes = array_get_modifiable(&ctx->sorted_nodes, &count);
+	for (i = 0; i < count; i++) {
+		i_assert(nodes[i].sort_id != 0);
+		if (!nodes[i].sort_id_changed || nodes[i].no_update ||
+		    nodes[i].seq >= lowest_failed_seq)
+			continue;
+
+		mail_index_update_ext(itrans, nodes[i].seq, ext_id,
+				      &nodes[i].sort_id, NULL);
+	}
+}
+
+static int sort_node_cmp(const struct mail_sort_node *n1,
+			 const struct mail_sort_node *n2)
+{
+	struct sort_string_context *ctx = static_zero_cmp_context;
+
+	if (n1->sort_id < n2->sort_id)
+		return !ctx->reverse ? -1 : 1;
+	if (n1->sort_id > n2->sort_id)
+		return !ctx->reverse ? 1 : -1;
+
+	return index_sort_node_cmp_type(ctx->program,
+					ctx->program->sort_program + 1,
+					n1->seq, n2->seq);
+}
+
+static void index_sort_add_missing(struct sort_string_context *ctx)
+{
+	struct mail_sort_node node;
+	const uint32_t *seqs;
+	unsigned int i, count;
+	uint32_t seq, next_seq;
+
+	ctx->have_all_wanted = TRUE;
+
+	seqs = array_get(&ctx->program->seqs, &count);
+	for (i = 0, next_seq = 1; i < count; i++) {
+		if (seqs[i] == next_seq)
+			next_seq++;
+		else {
+			i_assert(next_seq < seqs[i]);
+			for (seq = next_seq; seq < seqs[i]; seq++) {
+				i_zero(&node);
+				node.seq = seq;
+				index_sort_node_add(ctx, &node);
+			}
+			next_seq = seqs[i] + 1;
+		}
+	}
+
+	if (ctx->lowest_nonexpunged_zero == 0) {
+		/* we're handling only expunged zeros. if it causes us to
+		   renumber some existing sort IDs, don't save them. */
+		ctx->no_writing = TRUE;
+	}
+}
+
+static void index_sort_list_reset_broken(struct sort_string_context *ctx,
+					 const char *reason)
+{
+	struct mailbox *box = ctx->program->t->box;
+	struct mail_sort_node *node;
+
+	mailbox_set_critical(box, "Broken %s indexes, resetting: %s",
+			     ctx->primary_sort_name, reason);
+
+	array_clear(&ctx->zero_nodes);
+	array_append_array(&ctx->zero_nodes,
+			   &ctx->nonzero_nodes);
+	array_clear(&ctx->nonzero_nodes);
+
+	array_foreach_modifiable(&ctx->zero_nodes, node)
+		node->sort_id = 0;
+}
+
+void index_sort_list_finish_string(struct mail_search_sort_program *program)
+{
+	struct sort_string_context *ctx = program->context;
+	const struct mail_sort_node *nodes;
+	unsigned int i, count;
+	const char *reason;
+	uint32_t seq;
+
+	static_zero_cmp_context = ctx;
+	if (array_count(&ctx->zero_nodes) == 0) {
+		/* fast path: we have all sort IDs */
+		array_sort(&ctx->nonzero_nodes, sort_node_cmp);
+
+		nodes = array_get(&ctx->nonzero_nodes, &count);
+		if (!array_is_created(&program->seqs))
+			i_array_init(&program->seqs, count);
+		else
+			array_clear(&program->seqs);
+
+		for (i = 0; i < count; i++) {
+			seq = nodes[i].seq;
+			array_push_back(&program->seqs, &seq);
+		}
+		array_free(&ctx->nonzero_nodes);
+	} else {
+		if (ctx->seqs_nonsorted) {
+			/* the nodes need to be sorted by sequence initially */
+			array_sort(&ctx->zero_nodes, sort_node_seq_cmp);
+			array_sort(&ctx->nonzero_nodes, sort_node_seq_cmp);
+		}
+
+		/* we have to add some sort IDs. we'll do this for all
+		   messages, so first remember what messages we wanted
+		   to know about. */
+		index_sort_generate_seqs(ctx);
+		/* add messages not in seqs list */
+		index_sort_add_missing(ctx);
+		/* sort all messages with sort IDs */
+		array_sort(&ctx->nonzero_nodes, sort_node_cmp);
+		for (;;) {
+			/* sort all messages without sort IDs */
+			index_sort_zeroes(ctx);
+
+			if (ctx->reverse) {
+				/* sort lists are descending currently, but
+				   merging and sort ID assigning works only
+				   with ascending lists. reverse the lists
+				   temporarily. we can't do this while earlier
+				   because secondary sort conditions must not
+				   be reversed in results (but while assigning
+				   sort IDs it doesn't matter). */
+				array_reverse(&ctx->nonzero_nodes);
+				array_reverse(&ctx->zero_nodes);
+			}
+
+			/* merge zero and non-zero arrays into sorted_nodes */
+			index_sort_merge(ctx);
+			/* give sort IDs to messages missing them */
+			if (index_sort_add_sort_ids(ctx, &reason) == 0)
+				break;
+
+			/* broken, try again with sort IDs reset */
+			index_sort_list_reset_broken(ctx, reason);
+		}
+		index_sort_write_changed_sort_ids(ctx);
+
+		if (ctx->reverse) {
+			/* restore the correct sort order */
+			array_reverse(&ctx->sorted_nodes);
+		}
+
+		nodes = array_get(&ctx->sorted_nodes, &count);
+		array_clear(&program->seqs);
+		for (i = 0; i < count; i++) {
+			if (nodes[i].wanted) {
+				seq = nodes[i].seq;
+				array_push_back(&program->seqs, &seq);
+			}
+		}
+		pool_unref(&ctx->sort_string_pool);
+		i_free(ctx->sort_strings);
+		array_free(&ctx->sorted_nodes);
+		/* NOTE: we already freed nonzero_nodes and made it point to
+		   sorted_nodes. */
+	}
+	if (ctx->failed)
+		program->failed = TRUE;
+
+	array_free(&ctx->zero_nodes);
+	i_free(ctx);
+	program->context = NULL;
+}
diff --git a/src/lib-storage/index/index-sort.c b/src/lib-storage/index/index-sort.c
new file mode 100644
index 0000000..924a8d1
--- /dev/null
+++ b/src/lib-storage/index/index-sort.c
@@ -0,0 +1,738 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "unichar.h"
+#include "message-address.h"
+#include "message-header-decode.h"
+#include "imap-base-subject.h"
+#include "index-storage.h"
+#include "index-sort-private.h"
+
+
+struct mail_sort_node_date {
+	uint32_t seq;
+	time_t date;
+};
+ARRAY_DEFINE_TYPE(mail_sort_node_date, struct mail_sort_node_date);
+
+struct mail_sort_node_size {
+	uint32_t seq;
+	uoff_t size;
+};
+ARRAY_DEFINE_TYPE(mail_sort_node_size, struct mail_sort_node_size);
+
+struct mail_sort_node_float {
+	uint32_t seq;
+	float num;
+};
+ARRAY_DEFINE_TYPE(mail_sort_node_float, struct mail_sort_node_float);
+
+struct sort_cmp_context {
+	struct mail_search_sort_program *program;
+	bool reverse;
+};
+
+static struct sort_cmp_context static_node_cmp_context;
+
+static void
+index_sort_program_set_mail_failed(struct mail_search_sort_program *program,
+				   struct mail *mail)
+{
+	switch (mailbox_get_last_mail_error(mail->box)) {
+	case MAIL_ERROR_EXPUNGED:
+		break;
+	case MAIL_ERROR_LOOKUP_ABORTED:
+		/* just change the error message */
+		i_assert(program->slow_mails_left == 0);
+		mail_storage_set_error(program->t->box->storage, MAIL_ERROR_LIMIT,
+			"Requested sort would have taken too long.");
+		/* fall through */
+	default:
+		program->failed = TRUE;
+		break;
+	}
+}
+
+static time_t
+index_sort_program_set_date_failed(struct mail_search_sort_program *program,
+				   struct mail *mail)
+{
+	index_sort_program_set_mail_failed(program, mail);
+
+	if (mailbox_get_last_mail_error(mail->box) == MAIL_ERROR_LIMIT) {
+		/* limit reached - sort the rest of the mails at the end of
+		   the list by their UIDs */
+		return LONG_MAX;
+	} else {
+		/* expunged / some other error - sort in the beginning */
+		return 0;
+	}
+}
+
+static void
+index_sort_list_add_arrival(struct mail_search_sort_program *program,
+			    struct mail *mail)
+{
+	ARRAY_TYPE(mail_sort_node_date) *nodes = program->context;
+	struct mail_sort_node_date *node;
+
+	node = array_append_space(nodes);
+	node->seq = mail->seq;
+	if (mail_get_received_date(mail, &node->date) < 0)
+		node->date = index_sort_program_set_date_failed(program, mail);
+}
+
+static void
+index_sort_list_add_date(struct mail_search_sort_program *program,
+			 struct mail *mail)
+{
+	ARRAY_TYPE(mail_sort_node_date) *nodes = program->context;
+	struct mail_sort_node_date *node;
+	int tz;
+
+	node = array_append_space(nodes);
+	node->seq = mail->seq;
+	if (mail_get_date(mail, &node->date, &tz) < 0)
+		node->date = index_sort_program_set_date_failed(program, mail);
+	else if (node->date == 0) {
+		if (mail_get_received_date(mail, &node->date) < 0)
+			node->date = index_sort_program_set_date_failed(program, mail);
+	}
+}
+
+static void
+index_sort_list_add_size(struct mail_search_sort_program *program,
+			 struct mail *mail)
+{
+	ARRAY_TYPE(mail_sort_node_size) *nodes = program->context;
+	struct mail_sort_node_size *node;
+
+	node = array_append_space(nodes);
+	node->seq = mail->seq;
+	if (mail_get_virtual_size(mail, &node->size) < 0) {
+		index_sort_program_set_mail_failed(program, mail);
+		node->size = 0;
+	}
+}
+
+static int index_sort_get_pop3_order(struct mail *mail, uoff_t *size_r)
+{
+	const char *str;
+
+	if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) {
+		*size_r = (uint32_t)-1;
+		return -1;
+	}
+
+	if (str_to_uoff(str, size_r) < 0)
+		*size_r = (uint32_t)-1;
+	return 0;
+}
+
+static void
+index_sort_list_add_pop3_order(struct mail_search_sort_program *program,
+			       struct mail *mail)
+{
+	ARRAY_TYPE(mail_sort_node_size) *nodes = program->context;
+	struct mail_sort_node_size *node;
+
+	node = array_append_space(nodes);
+	node->seq = mail->seq;
+	(void)index_sort_get_pop3_order(mail, &node->size);
+}
+
+static int index_sort_get_relevancy(struct mail *mail, float *result_r)
+{
+	const char *str;
+
+	if (mail_get_special(mail, MAIL_FETCH_SEARCH_RELEVANCY, &str) < 0) {
+		*result_r = 0;
+		return -1;
+	}
+	*result_r = strtod(str, NULL);
+	return 0;
+}
+
+static void
+index_sort_list_add_relevancy(struct mail_search_sort_program *program,
+			      struct mail *mail)
+{
+	ARRAY_TYPE(mail_sort_node_float) *nodes = program->context;
+	struct mail_sort_node_float *node;
+
+	node = array_append_space(nodes);
+	node->seq = mail->seq;
+	(void)index_sort_get_relevancy(mail, &node->num);
+}
+
+void index_sort_list_add(struct mail_search_sort_program *program,
+			 struct mail *mail)
+{
+	enum mail_access_type orig_access_type = mail->access_type;
+	bool prev_slow = mail->mail_stream_accessed ||
+		mail->mail_metadata_accessed;
+
+	i_assert(mail->transaction == program->t);
+	/* if lookup_abort isn't NEVER, mail_sort_max_read_count handling
+	   doesn't work right. */
+	i_assert(mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER);
+
+	if (program->slow_mails_left == 0)
+		mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+
+	mail->access_type = MAIL_ACCESS_TYPE_SORT;
+	T_BEGIN {
+		program->sort_list_add(program, mail);
+	} T_END;
+	mail->access_type = orig_access_type;
+
+	if (!prev_slow && (mail->mail_stream_accessed ||
+			   mail->mail_metadata_accessed)) {
+		i_assert(program->slow_mails_left > 0);
+		program->slow_mails_left--;
+	}
+	mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+}
+
+static int sort_node_date_cmp(const struct mail_sort_node_date *n1,
+			      const struct mail_sort_node_date *n2)
+{
+	struct sort_cmp_context *ctx = &static_node_cmp_context;
+
+	if (n1->date < n2->date)
+		return !ctx->reverse ? -1 : 1;
+	if (n1->date > n2->date)
+		return !ctx->reverse ? 1 : -1;
+
+	return index_sort_node_cmp_type(ctx->program,
+					ctx->program->sort_program + 1,
+					n1->seq, n2->seq);
+}
+
+static void
+index_sort_list_finish_date(struct mail_search_sort_program *program)
+{
+	ARRAY_TYPE(mail_sort_node_date) *nodes = program->context;
+
+	array_sort(nodes, sort_node_date_cmp);
+	memcpy(&program->seqs, nodes, sizeof(program->seqs));
+	i_free(nodes);
+	program->context = NULL;
+}
+
+static int sort_node_size_cmp(const struct mail_sort_node_size *n1,
+			      const struct mail_sort_node_size *n2)
+{
+	struct sort_cmp_context *ctx = &static_node_cmp_context;
+
+	if (n1->size < n2->size)
+		return !ctx->reverse ? -1 : 1;
+	if (n1->size > n2->size)
+		return !ctx->reverse ? 1 : -1;
+
+	return index_sort_node_cmp_type(ctx->program,
+					ctx->program->sort_program + 1,
+					n1->seq, n2->seq);
+}
+
+static void
+index_sort_list_finish_size(struct mail_search_sort_program *program)
+{
+	ARRAY_TYPE(mail_sort_node_size) *nodes = program->context;
+
+	array_sort(nodes, sort_node_size_cmp);
+	memcpy(&program->seqs, nodes, sizeof(program->seqs));
+	i_free(nodes);
+	program->context = NULL;
+}
+
+static int sort_node_float_cmp(const struct mail_sort_node_float *n1,
+			       const struct mail_sort_node_float *n2)
+{
+	struct sort_cmp_context *ctx = &static_node_cmp_context;
+
+	if (n1->num < n2->num)
+		return !ctx->reverse ? -1 : 1;
+	if (n1->num > n2->num)
+		return !ctx->reverse ? 1 : -1;
+
+	return index_sort_node_cmp_type(ctx->program,
+					ctx->program->sort_program + 1,
+					n1->seq, n2->seq);
+}
+
+static void
+index_sort_list_finish_float(struct mail_search_sort_program *program)
+{
+	ARRAY_TYPE(mail_sort_node_float) *nodes = program->context;
+
+	/* NOTE: higher relevancy is returned first, unlike with all
+	   other number based sort keys, so temporarily reverse the search */
+	static_node_cmp_context.reverse = !static_node_cmp_context.reverse;
+	array_sort(nodes, sort_node_float_cmp);
+	static_node_cmp_context.reverse = !static_node_cmp_context.reverse;
+
+	memcpy(&program->seqs, nodes, sizeof(program->seqs));
+	i_free(nodes);
+	program->context = NULL;
+}
+
+void index_sort_list_finish(struct mail_search_sort_program *program)
+{
+	i_zero(&static_node_cmp_context);
+	static_node_cmp_context.program = program;
+	static_node_cmp_context.reverse =
+		(program->sort_program[0] & MAIL_SORT_FLAG_REVERSE) != 0;
+
+	struct event_reason *reason = event_reason_begin("mailbox:sort");
+	program->sort_list_finish(program);
+	event_reason_end(&reason);
+}
+
+bool index_sort_list_next(struct mail_search_sort_program *program,
+			  uint32_t *seq_r)
+{
+	const uint32_t *seqp;
+
+	if (program->iter_idx == array_count(&program->seqs))
+		return FALSE;
+
+	seqp = array_idx(&program->seqs, program->iter_idx++);
+	*seq_r = *seqp;
+	return TRUE;
+}
+
+static void
+get_wanted_fields(struct mailbox *box, const enum mail_sort_type *sort_program,
+		  enum mail_fetch_field *wanted_fields_r,
+		  struct mailbox_header_lookup_ctx **wanted_headers_r)
+{
+	enum mail_fetch_field fields = 0;
+	ARRAY_TYPE(const_string) headers;
+	const char *hdr_name;
+
+	t_array_init(&headers, 4);
+	for (unsigned int i = 0; sort_program[i] != MAIL_SORT_END; i++) {
+		enum mail_sort_type sort_type =
+			sort_program[i] & MAIL_SORT_MASK;
+		switch (sort_type) {
+		case MAIL_SORT_ARRIVAL:
+			fields |= MAIL_FETCH_RECEIVED_DATE;
+			break;
+		case MAIL_SORT_CC:
+			hdr_name = "Cc";
+			array_push_back(&headers, &hdr_name);
+			break;
+		case MAIL_SORT_DATE:
+			fields |= MAIL_FETCH_DATE;
+			break;
+		case MAIL_SORT_FROM:
+		case MAIL_SORT_DISPLAYFROM:
+			hdr_name = "From";
+			array_push_back(&headers, &hdr_name);
+			break;
+		case MAIL_SORT_SIZE:
+			fields |= MAIL_FETCH_VIRTUAL_SIZE;
+			break;
+		case MAIL_SORT_SUBJECT:
+			hdr_name = "Subject";
+			array_push_back(&headers, &hdr_name);
+			break;
+		case MAIL_SORT_TO:
+		case MAIL_SORT_DISPLAYTO:
+			hdr_name = "To";
+			array_push_back(&headers, &hdr_name);
+			break;
+		case MAIL_SORT_RELEVANCY:
+			fields |= MAIL_FETCH_SEARCH_RELEVANCY;
+			break;
+		case MAIL_SORT_POP3_ORDER:
+			fields |= MAIL_FETCH_POP3_ORDER;
+			break;
+
+		case MAIL_SORT_MASK:
+		case MAIL_SORT_FLAG_REVERSE:
+		case MAIL_SORT_END:
+			i_unreached();
+		}
+	}
+	*wanted_fields_r = fields;
+	if (array_count(&headers) == 0)
+		*wanted_headers_r = NULL;
+	else {
+		array_append_zero(&headers);
+		*wanted_headers_r =
+			mailbox_header_lookup_init(box, array_idx(&headers, 0));
+	}
+}
+
+struct mail_search_sort_program *
+index_sort_program_init(struct mailbox_transaction_context *t,
+			const enum mail_sort_type *sort_program)
+{
+	struct mail_search_sort_program *program;
+	enum mail_fetch_field wanted_fields;
+	struct mailbox_header_lookup_ctx *wanted_headers;
+	unsigned int i;
+
+	if (sort_program == NULL || sort_program[0] == MAIL_SORT_END)
+		return NULL;
+
+	get_wanted_fields(t->box, sort_program, &wanted_fields, &wanted_headers);
+
+	/* we support internal sorting by the primary condition */
+	program = i_new(struct mail_search_sort_program, 1);
+	program->t = t;
+	program->temp_mail = mail_alloc(t, wanted_fields, wanted_headers);
+	program->temp_mail->access_type = MAIL_ACCESS_TYPE_SORT;
+	if (wanted_headers != NULL)
+		mailbox_header_lookup_unref(&wanted_headers);
+
+	program->slow_mails_left =
+		program->t->box->storage->set->mail_sort_max_read_count;
+	if (program->slow_mails_left == 0)
+		program->slow_mails_left = UINT_MAX;
+
+	for (i = 0; i < MAX_SORT_PROGRAM_SIZE; i++) {
+		program->sort_program[i] = sort_program[i];
+		if (sort_program[i] == MAIL_SORT_END)
+			break;
+	}
+	if (i == MAX_SORT_PROGRAM_SIZE)
+		i_panic("index_sort_program_init(): Invalid sort program");
+
+	switch (program->sort_program[0] & MAIL_SORT_MASK) {
+	case MAIL_SORT_ARRIVAL:
+	case MAIL_SORT_DATE: {
+		ARRAY_TYPE(mail_sort_node_date) *nodes;
+
+		nodes = i_malloc(sizeof(*nodes));
+		i_array_init(nodes, 128);
+
+		if ((program->sort_program[0] &
+		     MAIL_SORT_MASK) == MAIL_SORT_ARRIVAL)
+			program->sort_list_add = index_sort_list_add_arrival;
+		else
+			program->sort_list_add = index_sort_list_add_date;
+		program->sort_list_finish = index_sort_list_finish_date;
+		program->context = nodes;
+		break;
+	}
+	case MAIL_SORT_SIZE: {
+		ARRAY_TYPE(mail_sort_node_size) *nodes;
+
+		nodes = i_malloc(sizeof(*nodes));
+		i_array_init(nodes, 128);
+		program->sort_list_add = index_sort_list_add_size;
+		program->sort_list_finish = index_sort_list_finish_size;
+		program->context = nodes;
+		break;
+	}
+	case MAIL_SORT_CC:
+	case MAIL_SORT_FROM:
+	case MAIL_SORT_SUBJECT:
+	case MAIL_SORT_TO:
+	case MAIL_SORT_DISPLAYFROM:
+	case MAIL_SORT_DISPLAYTO:
+		program->sort_list_add = index_sort_list_add_string;
+		program->sort_list_finish = index_sort_list_finish_string;
+		index_sort_list_init_string(program);
+		break;
+	case MAIL_SORT_RELEVANCY: {
+		ARRAY_TYPE(mail_sort_node_float) *nodes;
+
+		nodes = i_malloc(sizeof(*nodes));
+		i_array_init(nodes, 128);
+		program->sort_list_add = index_sort_list_add_relevancy;
+		program->sort_list_finish = index_sort_list_finish_float;
+		program->context = nodes;
+		break;
+	}
+	case MAIL_SORT_POP3_ORDER: {
+		ARRAY_TYPE(mail_sort_node_size) *nodes;
+
+		nodes = i_malloc(sizeof(*nodes));
+		i_array_init(nodes, 128);
+		program->sort_list_add = index_sort_list_add_pop3_order;
+		program->sort_list_finish = index_sort_list_finish_size;
+		program->context = nodes;
+		break;
+	}
+	default:
+		i_unreached();
+	}
+	return program;
+}
+
+int index_sort_program_deinit(struct mail_search_sort_program **_program)
+{
+	struct mail_search_sort_program *program = *_program;
+
+	*_program = NULL;
+
+	if (program->context != NULL)
+		index_sort_list_finish(program);
+	mail_free(&program->temp_mail);
+	array_free(&program->seqs);
+
+	int ret = program->failed ? -1 : 0;
+	i_free(program);
+	return ret;
+}
+
+static int
+get_first_addr(struct mail *mail, const char *header,
+	       struct message_address **addr_r)
+{
+	const char *str;
+	int ret;
+
+	if ((ret = mail_get_first_header(mail, header, &str)) <= 0) {
+		*addr_r = NULL;
+		return ret;
+	}
+
+	*addr_r = message_address_parse(pool_datastack_create(),
+					(const unsigned char *)str,
+					strlen(str), 1,
+					MESSAGE_ADDRESS_PARSE_FLAG_FILL_MISSING);
+	return 0;
+}
+
+static int
+get_first_mailbox(struct mail *mail, const char *header, const char **mailbox_r)
+{
+	struct message_address *addr;
+
+	if (get_first_addr(mail, header, &addr) < 0) {
+		*mailbox_r = "";
+		return -1;
+	}
+	*mailbox_r = addr != NULL && addr->mailbox != NULL ? addr->mailbox : "";
+	return 0;
+}
+
+static int
+get_display_name(struct mail *mail, const char *header, const char **name_r)
+{
+	struct message_address *addr;
+
+	*name_r = "";
+
+	if (get_first_addr(mail, header, &addr) < 0)
+		return -1;
+	if (addr == NULL)
+		return 0;
+
+	if (addr->name != NULL) {
+		string_t *str;
+		size_t len = strlen(addr->name);
+
+		str = t_str_new(len*2);
+		(void)message_header_decode_utf8(
+			(const unsigned char *)addr->name, len, str, NULL);
+		if (str_len(str) > 0) {
+			*name_r = str_c(str);
+			return 0;
+		}
+	}
+	if (addr->mailbox != NULL && addr->domain != NULL)
+		*name_r = t_strconcat(addr->mailbox, "@", addr->domain, NULL);
+	else if (addr->mailbox != NULL)
+		*name_r = addr->mailbox;
+	return 0;
+}
+
+static void
+index_sort_set_seq(struct mail_search_sort_program *program,
+		   struct mail *mail, uint32_t seq)
+{
+	if ((mail->mail_stream_accessed || mail->mail_metadata_accessed) &&
+	    program->slow_mails_left > 0)
+		program->slow_mails_left--;
+	mail_set_seq(mail, seq);
+	if (program->slow_mails_left == 0) {
+		/* too many slow lookups - just return the rest of the results
+		   in whatever order. */
+		mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+	}
+}
+
+int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq,
+			  enum mail_sort_type sort_type, string_t *dest)
+{
+	struct mail *mail = program->temp_mail;
+	const char *str;
+	int ret;
+	bool reply_or_fw;
+
+	index_sort_set_seq(program, mail, seq);
+	str_truncate(dest, 0);
+
+	switch (sort_type & MAIL_SORT_MASK) {
+	case MAIL_SORT_SUBJECT:
+		ret = mail_get_first_header(mail, "Subject", &str);
+		if (ret < 0)
+			break;
+		if (ret == 0) {
+			/* nonexistent header */
+			return 1;
+		}
+		str = imap_get_base_subject_cased(pool_datastack_create(),
+						  str, &reply_or_fw);
+		str_append(dest, str);
+		return 1;
+	case MAIL_SORT_CC:
+		ret = get_first_mailbox(mail, "Cc", &str);
+		break;
+	case MAIL_SORT_FROM:
+		ret = get_first_mailbox(mail, "From", &str);
+		break;
+	case MAIL_SORT_TO:
+		ret = get_first_mailbox(mail, "To", &str);
+		break;
+	case MAIL_SORT_DISPLAYFROM:
+		ret = get_display_name(mail, "From", &str);
+		break;
+	case MAIL_SORT_DISPLAYTO:
+		ret = get_display_name(mail, "To", &str);
+		break;
+	default:
+		i_unreached();
+	}
+	if (ret < 0) {
+		index_sort_program_set_mail_failed(program, mail);
+		if (!program->failed)
+			return 0;
+		return -1;
+	}
+
+	(void)uni_utf8_to_decomposed_titlecase(str, strlen(str), dest);
+	return 1;
+}
+
+int index_sort_node_cmp_type(struct mail_search_sort_program *program,
+			     const enum mail_sort_type *sort_program,
+			     uint32_t seq1, uint32_t seq2)
+{
+	struct mail *mail = program->temp_mail;
+	enum mail_sort_type sort_type;
+	time_t time1, time2;
+	uoff_t size1, size2;
+	float float1, float2;
+	int tz, ret = 0;
+
+	sort_type = *sort_program & MAIL_SORT_MASK;
+	switch (sort_type) {
+	case MAIL_SORT_CC:
+	case MAIL_SORT_FROM:
+	case MAIL_SORT_TO:
+	case MAIL_SORT_SUBJECT:
+	case MAIL_SORT_DISPLAYFROM:
+	case MAIL_SORT_DISPLAYTO:
+		T_BEGIN {
+			string_t *str1, *str2;
+
+			str1 = t_str_new(256);
+			str2 = t_str_new(256);
+			if (index_sort_header_get(program, seq1, sort_type, str1) < 0)
+				index_sort_program_set_mail_failed(program, mail);
+			if (index_sort_header_get(program, seq2, sort_type, str2) < 0)
+				index_sort_program_set_mail_failed(program, mail);
+
+			ret = strcmp(str_c(str1), str_c(str2));
+		} T_END;
+		break;
+	case MAIL_SORT_ARRIVAL:
+		index_sort_set_seq(program, mail, seq1);
+		if (mail_get_received_date(mail, &time1) < 0)
+			time1 = index_sort_program_set_date_failed(program, mail);
+
+		index_sort_set_seq(program, mail, seq2);
+		if (mail_get_received_date(mail, &time2) < 0)
+			time2 = index_sort_program_set_date_failed(program, mail);
+
+		ret = time1 < time2 ? -1 :
+			(time1 > time2 ? 1 : 0);
+		break;
+	case MAIL_SORT_DATE:
+		index_sort_set_seq(program, mail, seq1);
+		if (mail_get_date(mail, &time1, &tz) < 0)
+			time1 = index_sort_program_set_date_failed(program, mail);
+		else if (time1 == 0) {
+			if (mail_get_received_date(mail, &time1) < 0)
+				time1 = index_sort_program_set_date_failed(program, mail);
+		}
+
+		index_sort_set_seq(program, mail, seq2);
+		if (mail_get_date(mail, &time2, &tz) < 0)
+			time2 = index_sort_program_set_date_failed(program, mail);
+		else if (time2 == 0) {
+			if (mail_get_received_date(mail, &time2) < 0)
+				time2 = index_sort_program_set_date_failed(program, mail);
+		}
+
+		ret = time1 < time2 ? -1 :
+			(time1 > time2 ? 1 : 0);
+		break;
+	case MAIL_SORT_SIZE:
+		index_sort_set_seq(program, mail, seq1);
+		if (mail_get_virtual_size(mail, &size1) < 0) {
+			index_sort_program_set_mail_failed(program, mail);
+			size1 = 0;
+		}
+
+		index_sort_set_seq(program, mail, seq2);
+		if (mail_get_virtual_size(mail, &size2) < 0) {
+			index_sort_program_set_mail_failed(program, mail);
+			size2 = 0;
+		}
+
+		ret = size1 < size2 ? -1 :
+			(size1 > size2 ? 1 : 0);
+		break;
+	case MAIL_SORT_RELEVANCY:
+		index_sort_set_seq(program, mail, seq1);
+		if (index_sort_get_relevancy(mail, &float1) < 0)
+			index_sort_program_set_mail_failed(program, mail);
+		index_sort_set_seq(program, mail, seq2);
+		if (index_sort_get_relevancy(mail, &float2) < 0)
+			index_sort_program_set_mail_failed(program, mail);
+
+		/* NOTE: higher relevancy is returned first, unlike with all
+		   other number based sort keys */
+		ret = float1 < float2 ? 1 :
+			(float1 > float2 ? -1 : 0);
+		break;
+	case MAIL_SORT_POP3_ORDER:
+		/* 32bit numbers would be enough, but since there is already
+		   existing code for uoff_t in sizes, just use them. */
+		index_sort_set_seq(program, mail, seq1);
+		if (index_sort_get_pop3_order(mail, &size1) < 0)
+			index_sort_program_set_mail_failed(program, mail);
+		index_sort_set_seq(program, mail, seq2);
+		if (index_sort_get_pop3_order(mail, &size2) < 0)
+			index_sort_program_set_mail_failed(program, mail);
+
+		ret = size1 < size2 ? -1 :
+			(size1 > size2 ? 1 : 0);
+		break;
+	case MAIL_SORT_END:
+		return seq1 < seq2 ? -1 :
+			(seq1 > seq2 ? 1 : 0);
+	case MAIL_SORT_MASK:
+	case MAIL_SORT_FLAG_REVERSE:
+		i_unreached();
+	}
+
+	if (ret == 0) {
+		return index_sort_node_cmp_type(program, sort_program+1,
+						seq1, seq2);
+	}
+
+	if ((*sort_program & MAIL_SORT_FLAG_REVERSE) != 0)
+		ret = ret < 0 ? 1 : -1;
+	return ret;
+}
diff --git a/src/lib-storage/index/index-sort.h b/src/lib-storage/index/index-sort.h
new file mode 100644
index 0000000..b0e2770
--- /dev/null
+++ b/src/lib-storage/index/index-sort.h
@@ -0,0 +1,18 @@
+#ifndef INDEX_SORT_H
+#define INDEX_SORT_H
+
+struct mail_search_sort_program;
+
+struct mail_search_sort_program *
+index_sort_program_init(struct mailbox_transaction_context *t,
+			const enum mail_sort_type *sort_program);
+int index_sort_program_deinit(struct mail_search_sort_program **program);
+
+void index_sort_list_add(struct mail_search_sort_program *program,
+			 struct mail *mail);
+void index_sort_list_finish(struct mail_search_sort_program *program);
+
+bool index_sort_list_next(struct mail_search_sort_program *program,
+			  uint32_t *seq_r);
+
+#endif
diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c
new file mode 100644
index 0000000..9d751b9
--- /dev/null
+++ b/src/lib-storage/index/index-status.c
@@ -0,0 +1,344 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-cache.h"
+#include "mail-index-modseq.h"
+#include "mailbox-recent-flags.h"
+#include "index-storage.h"
+
+static void
+get_last_cached_seq(struct mailbox *box, uint32_t *last_cached_seq_r)
+{
+	const struct mail_index_header *hdr;
+	struct mail_cache_view *cache_view;
+	uint32_t seq;
+
+	*last_cached_seq_r = 0;
+	if (!mail_cache_exists(box->cache))
+		return;
+
+	cache_view = mail_cache_view_open(box->cache, box->view);
+	hdr = mail_index_get_header(box->view);
+	for (seq = hdr->messages_count; seq > 0; seq--) {
+		if (mail_cache_field_exists_any(cache_view, seq)) {
+			*last_cached_seq_r = seq;
+			break;
+		}
+	}
+	mail_cache_view_close(&cache_view);
+}
+
+int index_storage_get_status(struct mailbox *box,
+			     enum mailbox_status_items items,
+			     struct mailbox_status *status_r)
+{
+	if (items == 0) {
+		/* caller could have wanted only e.g. mailbox_status.have_*
+		   flags */
+		return 0;
+	}
+
+	if (!box->opened) {
+		if (mailbox_open(box) < 0)
+			return -1;
+		if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0)
+			return -1;
+	}
+	index_storage_get_open_status(box, items, status_r);
+	return 0;
+}
+
+static unsigned int index_storage_count_pvt_unseen(struct mailbox *box)
+{
+	const struct mail_index_record *pvt_rec;
+	uint32_t shared_seq, pvt_seq, shared_count, pvt_count;
+	uint32_t shared_uid;
+	unsigned int unseen_count = 0;
+
+	/* we can't trust private index to be up to date. we'll need to go
+	   through the shared index and for each existing mail lookup its
+	   private flags. if a mail doesn't exist in private index then its
+	   flags are 0. */
+	shared_count = mail_index_view_get_messages_count(box->view);
+	pvt_count = mail_index_view_get_messages_count(box->view_pvt);
+	shared_seq = pvt_seq = 1;
+	while (shared_seq <= shared_count && pvt_seq <= pvt_count) {
+		mail_index_lookup_uid(box->view, shared_seq, &shared_uid);
+		pvt_rec = mail_index_lookup(box->view_pvt, pvt_seq);
+
+		if (shared_uid == pvt_rec->uid) {
+			if ((pvt_rec->flags & MAIL_SEEN) == 0)
+				unseen_count++;
+			shared_seq++; pvt_seq++;
+		} else if (shared_uid < pvt_rec->uid) {
+			shared_seq++;
+		} else {
+			pvt_seq++;
+		}
+	}
+	unseen_count += (shared_count+1) - shared_seq;
+	return unseen_count;
+}
+
+static uint32_t index_storage_find_first_pvt_unseen_seq(struct mailbox *box)
+{
+	const struct mail_index_header *pvt_hdr;
+	const struct mail_index_record *pvt_rec;
+	uint32_t pvt_seq, pvt_count, shared_seq, seq2;
+
+	pvt_count = mail_index_view_get_messages_count(box->view_pvt);
+	mail_index_lookup_first(box->view_pvt, 0, MAIL_SEEN, &pvt_seq);
+	if (pvt_seq == 0)
+		pvt_seq = pvt_count+1;
+	for (; pvt_seq <= pvt_count; pvt_seq++) {
+		pvt_rec = mail_index_lookup(box->view_pvt, pvt_seq);
+		if ((pvt_rec->flags & MAIL_SEEN) == 0 &&
+		    mail_index_lookup_seq(box->view, pvt_rec->uid, &shared_seq))
+			return shared_seq;
+	}
+	/* if shared index has any messages that don't exist in private index,
+	   the first of them is the first unseen message */
+	pvt_hdr = mail_index_get_header(box->view_pvt);
+	if (mail_index_lookup_seq_range(box->view,
+					pvt_hdr->next_uid, (uint32_t)-1,
+					&shared_seq, &seq2))
+		return shared_seq;
+	return 0;
+}
+
+void index_storage_get_open_status(struct mailbox *box,
+				   enum mailbox_status_items items,
+				   struct mailbox_status *status_r)
+{
+	const struct mail_index_header *hdr;
+
+	/* we can get most of the status items without any trouble */
+	hdr = mail_index_get_header(box->view);
+	status_r->messages = hdr->messages_count;
+	if ((items & STATUS_RECENT) != 0) {
+		if ((box->flags & MAILBOX_FLAG_DROP_RECENT) != 0) {
+			/* recent flags are set and dropped by the previous
+			   sync while index was locked. if we updated the
+			   recent flags here we'd have a race condition. */
+			i_assert(box->synced);
+		} else {
+			/* make sure recent count is set, in case we haven't
+			   synced yet */
+			index_sync_update_recent_count(box);
+		}
+		status_r->recent = mailbox_recent_flags_count(box);
+		i_assert(status_r->recent <= status_r->messages);
+	}
+	if ((items & STATUS_UNSEEN) != 0) {
+		if (box->view_pvt == NULL ||
+		    (mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) {
+			status_r->unseen = hdr->messages_count -
+				hdr->seen_messages_count;
+		} else {
+			status_r->unseen = index_storage_count_pvt_unseen(box);
+		}
+	}
+	status_r->uidvalidity = hdr->uid_validity;
+	status_r->uidnext = hdr->next_uid;
+	status_r->first_recent_uid = hdr->first_recent_uid;
+	if ((items & STATUS_HIGHESTMODSEQ) != 0) {
+		status_r->nonpermanent_modseqs =
+			mail_index_is_in_memory(box->index);
+		status_r->no_modseq_tracking =
+			!mail_index_have_modseq_tracking(box->index);
+		status_r->highest_modseq =
+			mail_index_modseq_get_highest(box->view);
+		if (status_r->highest_modseq == 0) {
+			/* modseqs not enabled yet, but we can't return 0 */
+			status_r->highest_modseq = 1;
+		}
+	}
+	if ((items & STATUS_HIGHESTPVTMODSEQ) != 0 && box->view_pvt != NULL) {
+		status_r->highest_pvt_modseq =
+			mail_index_modseq_get_highest(box->view_pvt);
+		if (status_r->highest_pvt_modseq == 0) {
+			/* modseqs not enabled yet, but we can't return 0 */
+			status_r->highest_pvt_modseq = 1;
+		}
+	}
+
+	if ((items & STATUS_FIRST_UNSEEN_SEQ) != 0) {
+		if (box->view_pvt == NULL ||
+		    (mailbox_get_private_flags_mask(box) & MAIL_SEEN) == 0) {
+			mail_index_lookup_first(box->view, 0, MAIL_SEEN,
+						&status_r->first_unseen_seq);
+		} else {
+			status_r->first_unseen_seq =
+				index_storage_find_first_pvt_unseen_seq(box);
+		}
+	}
+	if ((items & STATUS_LAST_CACHED_SEQ) != 0)
+		get_last_cached_seq(box, &status_r->last_cached_seq);
+
+	if ((items & STATUS_KEYWORDS) != 0)
+		status_r->keywords = mail_index_get_keywords(box->index);
+	if ((items & STATUS_PERMANENT_FLAGS) != 0) {
+		if (!mailbox_is_readonly(box)) {
+			status_r->permanent_flags = MAIL_FLAGS_NONRECENT;
+			status_r->permanent_keywords = TRUE;
+			status_r->allow_new_keywords =
+				!box->disallow_new_keywords;
+		}
+		status_r->flags = MAIL_FLAGS_NONRECENT;
+	}
+}
+
+static void
+get_metadata_cache_fields(struct mailbox *box,
+			  struct mailbox_metadata *metadata_r)
+{
+	const struct mail_cache_field *fields;
+	enum mail_cache_decision_type dec;
+	ARRAY_TYPE(mailbox_cache_field) *cache_fields;
+	struct mailbox_cache_field *cf;
+	unsigned int i, count;
+
+	fields = mail_cache_register_get_list(box->cache,
+					      pool_datastack_create(), &count);
+
+	cache_fields = t_new(ARRAY_TYPE(mailbox_cache_field), 1);
+	t_array_init(cache_fields, count);
+	for (i = 0; i < count; i++) {
+		dec = fields[i].decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED);
+		if (dec != MAIL_CACHE_DECISION_NO) {
+			cf = array_append_space(cache_fields);
+			cf->name = fields[i].name;
+			cf->decision = fields[i].decision;
+			cf->last_used = fields[i].last_used;
+		}
+	}
+	metadata_r->cache_fields = cache_fields;
+}
+
+static void get_metadata_precache_fields(struct mailbox *box,
+					 struct mailbox_metadata *metadata_r)
+{
+	const struct mail_cache_field *fields;
+	unsigned int i, count;
+	enum mail_fetch_field cache = 0;
+
+	fields = mail_cache_register_get_list(box->cache,
+					      pool_datastack_create(), &count);
+	for (i = 0; i < count; i++) {
+		const char *name = fields[i].name;
+
+		if (str_begins(name, "hdr.") ||
+		    strcmp(name, "date.sent") == 0 ||
+		    strcmp(name, "imap.envelope") == 0)
+			cache |= MAIL_FETCH_STREAM_HEADER;
+		else if (strcmp(name, "mime.parts") == 0 ||
+			 strcmp(name, "binary.parts") == 0 ||
+			 strcmp(name, "imap.body") == 0 ||
+			 strcmp(name, "imap.bodystructure") == 0 ||
+			 strcmp(name, "body.snippet") == 0)
+			cache |= MAIL_FETCH_STREAM_BODY;
+		else if (strcmp(name, "date.received") == 0)
+			cache |= MAIL_FETCH_RECEIVED_DATE;
+		else if (strcmp(name, "date.save") == 0)
+			cache |= MAIL_FETCH_SAVE_DATE;
+		else if (strcmp(name, "size.virtual") == 0)
+			cache |= MAIL_FETCH_VIRTUAL_SIZE;
+		else if (strcmp(name, "size.physical") == 0)
+			cache |= MAIL_FETCH_PHYSICAL_SIZE;
+		else if (strcmp(name, "pop3.uidl") == 0)
+			cache |= MAIL_FETCH_UIDL_BACKEND;
+		else if (strcmp(name, "pop3.order") == 0)
+			cache |= MAIL_FETCH_POP3_ORDER;
+		else if (strcmp(name, "guid") == 0)
+			cache |= MAIL_FETCH_GUID;
+		else if (strcmp(name, "flags") == 0) {
+			/* just ignore for now at least.. */
+		} else
+			e_debug(box->event,
+				"Ignoring unknown cache field: %s", name);
+	}
+	metadata_r->precache_fields = cache;
+}
+
+static int
+index_mailbox_get_first_save_date(struct mailbox *box,
+				  struct mailbox_metadata *metadata_r)
+{
+	const struct mail_index_header *hdr;
+	struct mailbox_transaction_context *t;
+	struct mail *mail;
+	uint32_t seq;
+	int ret = -1;
+
+	hdr = mail_index_get_header(box->view);
+	if (hdr->messages_count == 0) {
+		metadata_r->first_save_date = (time_t)-1;
+		return 0;
+	}
+
+	t = mailbox_transaction_begin(box, 0, __func__);
+	mail = mail_alloc(t, 0, NULL);
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		mail_set_seq(mail, seq);
+		if (mail_get_save_date(mail, &metadata_r->first_save_date) >= 0) {
+			ret = 0;
+			break;
+		}
+		if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXPUNGED) {
+			/* failed */
+			break;
+		}
+	}
+	mail_free(&mail);
+	(void)mailbox_transaction_commit(&t);
+	if (seq > hdr->messages_count) {
+		/* all messages were expunged after all */
+		metadata_r->first_save_date = (time_t)-1;
+		return 0;
+	}
+	return ret;
+}
+
+int index_mailbox_get_metadata(struct mailbox *box,
+			       enum mailbox_metadata_items items,
+			       struct mailbox_metadata *metadata_r)
+{
+	/* handle items that don't require opening the mailbox */
+	if ((items & MAILBOX_METADATA_BACKEND_NAMESPACE) != 0) {
+		metadata_r->backend_ns_prefix = "";
+		metadata_r->backend_ns_type =
+			mailbox_list_get_namespace(box->list)->type;
+		items &= ENUM_NEGATE(MAILBOX_METADATA_BACKEND_NAMESPACE);
+	}
+	if (items == 0)
+		return 0;
+
+	/* handle items that require opening the mailbox */
+	if (!box->opened) {
+		if (mailbox_open(box) < 0)
+			return -1;
+	}
+	if (!box->synced && (items & MAILBOX_METADATA_SYNC_ITEMS) != 0) {
+		if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0)
+			return -1;
+	}
+
+	if ((items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0) {
+		if (index_mailbox_get_virtual_size(box, metadata_r) < 0)
+			return -1;
+	}
+	if ((items & MAILBOX_METADATA_PHYSICAL_SIZE) != 0) {
+		if (index_mailbox_get_physical_size(box, metadata_r) < 0)
+			return -1;
+	}
+	if ((items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0) {
+		if (index_mailbox_get_first_save_date(box, metadata_r) < 0)
+			return -1;
+	}
+	if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0)
+		get_metadata_cache_fields(box, metadata_r);
+	if ((items & MAILBOX_METADATA_PRECACHE_FIELDS) != 0)
+		get_metadata_precache_fields(box, metadata_r);
+	return 0;
+}
diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
new file mode 100644
index 0000000..e1998b0
--- /dev/null
+++ b/src/lib-storage/index/index-storage.c
@@ -0,0 +1,1291 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "ioloop.h"
+#include "str.h"
+#include "str-sanitize.h"
+#include "mkdir-parents.h"
+#include "dict.h"
+#include "fs-api.h"
+#include "message-header-parser.h"
+#include "mail-index-alloc-cache.h"
+#include "mail-index-private.h"
+#include "mail-index-modseq.h"
+#include "mailbox-log.h"
+#include "mailbox-list-private.h"
+#include "mail-search-build.h"
+#include "index-storage.h"
+#include "index-mail.h"
+#include "index-attachment.h"
+#include "index-thread-private.h"
+#include "index-mailbox-size.h"
+
+#include <time.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#define LOCK_NOTIFY_INTERVAL 30
+
+struct index_storage_module index_storage_module =
+	MODULE_CONTEXT_INIT(&mail_storage_module_register);
+
+static void set_cache_decisions(struct mail_cache *cache,
+				const char *set, const char *fields,
+				enum mail_cache_decision_type dec)
+{
+	struct mail_cache_field field;
+	const char *const *arr;
+	unsigned int idx;
+
+	if (fields == NULL || *fields == '\0')
+		return;
+
+	for (arr = t_strsplit_spaces(fields, " ,"); *arr != NULL; arr++) {
+		const char *name = *arr;
+
+		idx = mail_cache_register_lookup(cache, name);
+		if (idx != UINT_MAX) {
+			field = *mail_cache_register_get_field(cache, idx);
+		} else if (strncasecmp(name, "hdr.", 4) == 0) {
+			/* Do some sanity checking for the header name. Mainly
+			   to make sure there aren't UTF-8 characters that look
+			   like their ASCII equivalents or are completely
+			   invisible. */
+			if (message_header_name_is_valid(name+4)) {
+				i_zero(&field);
+				field.name = name;
+				field.type = MAIL_CACHE_FIELD_HEADER;
+			} else {
+				i_error("%s: Header name '%s' has invalid character, ignoring",
+					set, name);
+				continue;
+			}
+		} else {
+			i_error("%s: Unknown cache field name '%s', ignoring",
+				set, *arr);
+			continue;
+		}
+
+		field.decision = dec;
+		mail_cache_register_fields(cache, &field, 1);
+	}
+}
+
+static void index_cache_register_defaults(struct mailbox *box)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	const struct mail_storage_settings *set = box->storage->set;
+	struct mail_cache *cache = box->cache;
+
+	ibox->cache_fields = i_malloc(sizeof(global_cache_fields));
+	memcpy(ibox->cache_fields, global_cache_fields,
+	       sizeof(global_cache_fields));
+	mail_cache_register_fields(cache, ibox->cache_fields,
+				   MAIL_INDEX_CACHE_FIELD_COUNT);
+
+	if (strcmp(set->mail_never_cache_fields, "*") == 0) {
+		/* all caching disabled for now */
+		box->mail_cache_disabled = TRUE;
+		return;
+	}
+
+	set_cache_decisions(cache, "mail_cache_fields",
+			    set->mail_cache_fields,
+			    MAIL_CACHE_DECISION_TEMP);
+	set_cache_decisions(cache, "mail_always_cache_fields",
+			    set->mail_always_cache_fields,
+			    MAIL_CACHE_DECISION_YES |
+			    MAIL_CACHE_DECISION_FORCED);
+	set_cache_decisions(cache, "mail_never_cache_fields",
+			    set->mail_never_cache_fields,
+			    MAIL_CACHE_DECISION_NO |
+			    MAIL_CACHE_DECISION_FORCED);
+}
+
+void index_storage_lock_notify(struct mailbox *box,
+			       enum mailbox_lock_notify_type notify_type,
+			       unsigned int secs_left)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	struct mail_storage *storage = box->storage;
+	const char *str;
+	time_t now;
+
+	/* if notify type changes, print the message immediately */
+	now = time(NULL);
+	if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE ||
+	    ibox->last_notify_type == notify_type) {
+		if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE &&
+		    notify_type == MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE) {
+			/* first override notification, show it */
+		} else {
+			if (now < ibox->next_lock_notify || secs_left < 15)
+				return;
+		}
+	}
+
+	ibox->next_lock_notify = now + LOCK_NOTIFY_INTERVAL;
+        ibox->last_notify_type = notify_type;
+
+	switch (notify_type) {
+	case MAILBOX_LOCK_NOTIFY_NONE:
+		break;
+	case MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT:
+		if (storage->callbacks.notify_no == NULL)
+			break;
+
+		str = t_strdup_printf("Mailbox is locked, will abort in "
+				      "%u seconds", secs_left);
+		storage->callbacks.
+			notify_no(box, str, storage->callback_context);
+		break;
+	case MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE:
+		if (storage->callbacks.notify_ok == NULL)
+			break;
+
+		str = t_strdup_printf("Stale mailbox lock file detected, "
+				      "will override in %u seconds", secs_left);
+		storage->callbacks.
+			notify_ok(box, str, storage->callback_context);
+		break;
+	}
+}
+
+void index_storage_lock_notify_reset(struct mailbox *box)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+
+	ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL;
+	ibox->last_notify_type = MAILBOX_LOCK_NOTIFY_NONE;
+}
+
+static int
+index_mailbox_alloc_index(struct mailbox *box, struct mail_index **index_r)
+{
+	const char *index_dir, *mailbox_path;
+
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX,
+				&mailbox_path) < 0)
+		return -1;
+	if ((box->flags & MAILBOX_FLAG_NO_INDEX_FILES) != 0 ||
+	    mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX,
+				&index_dir) <= 0)
+		index_dir = NULL;
+	/* Note that this may cause box->event to live longer than box */
+	*index_r = mail_index_alloc_cache_get(box->event,
+					      mailbox_path, index_dir,
+					      box->index_prefix);
+	return 0;
+}
+
+int index_storage_mailbox_exists(struct mailbox *box, bool auto_boxes,
+				 enum mailbox_existence *existence_r)
+{
+	if (auto_boxes && mailbox_is_autocreated(box)) {
+		*existence_r = MAILBOX_EXISTENCE_SELECT;
+		return 0;
+	}
+
+	return index_storage_mailbox_exists_full(box, NULL, existence_r);
+}
+
+int index_storage_mailbox_exists_full(struct mailbox *box, const char *subdir,
+				      enum mailbox_existence *existence_r)
+{
+	struct stat st;
+	const char *path, *path2, *index_path;
+	int ret;
+
+	/* see if it's selectable */
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path);
+	if (ret < 0) {
+		if (mailbox_get_last_mail_error(box) != MAIL_ERROR_NOTFOUND)
+			return -1;
+		*existence_r = MAILBOX_EXISTENCE_NONE;
+		return 0;
+	}
+	if (ret == 0) {
+		/* no mailboxes in this storage? */
+		*existence_r = MAILBOX_EXISTENCE_NONE;
+		return 0;
+	}
+
+	ret = (subdir != NULL || !box->list->set.iter_from_index_dir) ? 0 :
+		mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &index_path);
+	if (ret > 0 && strcmp(path, index_path) != 0) {
+		/* index directory is different - prefer looking it up first
+		   since it might be on a faster storage. since the directory
+		   itself exists also for \NoSelect mailboxes, we'll need to
+		   check the dovecot.index.log existence. */
+		index_path = t_strconcat(index_path, "/", box->index_prefix,
+					 ".log", NULL);
+		if (stat(index_path, &st) == 0) {
+			*existence_r = MAILBOX_EXISTENCE_SELECT;
+			return 0;
+		}
+	}
+
+	if (subdir != NULL)
+		path = t_strconcat(path, "/", subdir, NULL);
+	if (stat(path, &st) == 0) {
+		*existence_r = MAILBOX_EXISTENCE_SELECT;
+		return 0;
+	}
+	if (!ENOTFOUND(errno) && errno != EACCES) {
+		mailbox_set_critical(box, "stat(%s) failed: %m", path);
+		return -1;
+	}
+
+	/* see if it's non-selectable */
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_DIR, &path2) <= 0 ||
+	    (strcmp(path, path2) != 0 && stat(path2, &st) == 0)) {
+		*existence_r = MAILBOX_EXISTENCE_NOSELECT;
+		return 0;
+	}
+	*existence_r = MAILBOX_EXISTENCE_NONE;
+	return 0;
+}
+
+int index_storage_mailbox_alloc_index(struct mailbox *box)
+{
+	const char *cache_dir;
+
+	if (box->index != NULL)
+		return 0;
+
+	if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX) < 0)
+		return -1;
+	if (index_mailbox_alloc_index(box, &box->index) < 0)
+		return -1;
+
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX_CACHE,
+				&cache_dir) > 0) {
+		if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX_CACHE) < 0)
+			return -1;
+		mail_index_set_cache_dir(box->index, cache_dir);
+	}
+	mail_index_set_fsync_mode(box->index,
+				  box->storage->set->parsed_fsync_mode, 0);
+	mail_index_set_lock_method(box->index,
+		box->storage->set->parsed_lock_method,
+		mail_storage_get_lock_timeout(box->storage, UINT_MAX));
+
+	const struct mail_storage_settings *set = box->storage->set;
+	struct mail_index_optimization_settings optimization_set = {
+		.index = {
+			.rewrite_min_log_bytes = set->mail_index_rewrite_min_log_bytes,
+			.rewrite_max_log_bytes = set->mail_index_rewrite_max_log_bytes,
+		},
+		.log = {
+			.min_size = set->mail_index_log_rotate_min_size,
+			.max_size = set->mail_index_log_rotate_max_size,
+			.min_age_secs = set->mail_index_log_rotate_min_age,
+			.log2_max_age_secs = set->mail_index_log2_max_age,
+		},
+		.cache = {
+			.unaccessed_field_drop_secs = set->mail_cache_unaccessed_field_drop,
+			.record_max_size = set->mail_cache_record_max_size,
+			.max_size = set->mail_cache_max_size,
+			.purge_min_size = set->mail_cache_purge_min_size,
+			.purge_delete_percentage = set->mail_cache_purge_delete_percentage,
+			.purge_continued_percentage = set->mail_cache_purge_continued_percentage,
+			.purge_header_continue_count = set->mail_cache_purge_header_continue_count,
+		},
+	};
+	mail_index_set_optimization_settings(box->index, &optimization_set);
+	return 0;
+}
+
+int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	enum mail_index_open_flags index_flags;
+	int ret;
+
+	i_assert(!box->opened);
+
+	index_flags = ibox->index_flags;
+	if (move_to_memory)
+		index_flags &= ENUM_NEGATE(MAIL_INDEX_OPEN_FLAG_CREATE);
+
+	if (index_storage_mailbox_alloc_index(box) < 0)
+		return -1;
+
+	/* make sure mail_index_set_permissions() has been called */
+	(void)mailbox_get_permissions(box);
+
+	ret = mail_index_open(box->index, index_flags);
+	if (ret <= 0 || move_to_memory) {
+		if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) {
+			i_assert(ret <= 0);
+			mailbox_set_index_error(box);
+			return -1;
+		}
+
+		if (mail_index_move_to_memory(box->index) < 0) {
+			/* try opening once more. it should be created
+			   directly into memory now. */
+			if (mail_index_open_or_create(box->index,
+						      index_flags) < 0)
+				i_panic("in-memory index creation failed");
+		}
+	}
+	if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) {
+		if (mail_index_is_in_memory(box->index)) {
+			mailbox_set_critical(box,
+				"Couldn't create index file");
+			mail_index_close(box->index);
+			return -1;
+		}
+	}
+
+	if ((box->flags & MAILBOX_FLAG_OPEN_DELETED) == 0) {
+		if (mail_index_is_deleted(box->index)) {
+			mailbox_set_deleted(box);
+			mail_index_close(box->index);
+			return -1;
+		}
+	}
+	if ((box->flags & MAILBOX_FLAG_FSCK) != 0) {
+		if (mail_index_fsck(box->index) < 0) {
+			mailbox_set_index_error(box);
+			return -1;
+		}
+	}
+
+	box->cache = mail_index_get_cache(box->index);
+	index_cache_register_defaults(box);
+	box->view = mail_index_view_open(box->index);
+	ibox->keyword_names = mail_index_get_keywords(box->index);
+	box->vsize_hdr_ext_id =
+		mail_index_ext_register(box->index, "hdr-vsize",
+					sizeof(struct mailbox_index_vsize), 0,
+					sizeof(uint64_t));
+	box->pop3_uidl_hdr_ext_id =
+		mail_index_ext_register(box->index, "hdr-pop3-uidl",
+					sizeof(struct mailbox_index_pop3_uidl), 0, 0);
+	box->box_name_hdr_ext_id =
+		mail_index_ext_register(box->index, "box-name", 0, 0, 0);
+
+	box->box_last_rename_stamp_ext_id =
+		mail_index_ext_register(box->index, "last-rename-stamp",
+					sizeof(uint32_t), 0, sizeof(uint32_t));
+	box->mail_vsize_ext_id = mail_index_ext_register(box->index, "vsize", 0,
+							 sizeof(uint32_t),
+							 sizeof(uint32_t));
+
+	box->opened = TRUE;
+
+	if ((box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0)
+		mail_index_modseq_enable(box->index);
+
+	index_thread_mailbox_opened(box);
+	hook_mailbox_opened(box);
+	return 0;
+}
+
+void index_storage_mailbox_alloc(struct mailbox *box, const char *vname,
+				 enum mailbox_flags flags,
+				 const char *index_prefix)
+{
+	static unsigned int mailbox_generation_sequence = 0;
+	struct index_mailbox_context *ibox;
+
+	i_assert(vname != NULL);
+
+	box->generation_sequence = ++mailbox_generation_sequence;
+	box->vname = p_strdup(box->pool, vname);
+	box->name = p_strdup(box->pool,
+			     mailbox_list_get_storage_name(box->list, vname));
+	box->flags = flags;
+	box->index_prefix = p_strdup(box->pool, index_prefix);
+	box->event = event_create(box->storage->event);
+	event_add_category(box->event, &event_category_mailbox);
+	event_add_str(box->event, "mailbox", box->vname);
+	event_set_append_log_prefix(box->event,
+		t_strdup_printf("Mailbox %s: ", str_sanitize(box->vname, 128)));
+
+	p_array_init(&box->search_results, box->pool, 16);
+	array_create(&box->module_contexts,
+		     box->pool, sizeof(void *), 5);
+
+	ibox = p_new(box->pool, struct index_mailbox_context, 1);
+	ibox->list_index_sync_ext_id = (uint32_t)-1;
+	ibox->index_flags = MAIL_INDEX_OPEN_FLAG_CREATE |
+		mail_storage_settings_to_index_flags(box->storage->set);
+	if ((box->flags & MAILBOX_FLAG_SAVEONLY) != 0)
+		ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_SAVEONLY;
+	if (event_want_debug(box->event))
+		ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_DEBUG;
+	ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL;
+	MODULE_CONTEXT_SET(box, index_storage_module, ibox);
+
+	box->inbox_user = strcmp(box->name, "INBOX") == 0 &&
+		(box->list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0;
+	box->inbox_any = strcmp(box->name, "INBOX") == 0 &&
+		(box->list->ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0;
+}
+
+int index_storage_mailbox_enable(struct mailbox *box,
+				 enum mailbox_feature feature)
+{
+	if ((feature & MAILBOX_FEATURE_CONDSTORE) != 0) {
+		box->enabled_features |= MAILBOX_FEATURE_CONDSTORE;
+		if (box->opened)
+			mail_index_modseq_enable(box->index);
+	}
+	return 0;
+}
+
+void index_storage_mailbox_close(struct mailbox *box)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+
+	mailbox_watch_remove_all(box);
+	i_stream_unref(&box->input);
+
+	if (box->view_pvt != NULL)
+		mail_index_view_close(&box->view_pvt);
+	if (box->index_pvt != NULL)
+		mail_index_close(box->index_pvt);
+	if (box->view != NULL) {
+		mail_index_view_close(&box->view);
+		mail_index_close(box->index);
+	}
+	box->cache = NULL;
+
+	ibox->keyword_names = NULL;
+	i_free_and_null(ibox->cache_fields);
+
+	ibox->sync_last_check = 0;
+}
+
+static void index_storage_mailbox_unref_indexes(struct mailbox *box)
+{
+	if (box->index_pvt != NULL)
+		mail_index_alloc_cache_unref(&box->index_pvt);
+	if (box->index != NULL)
+		mail_index_alloc_cache_unref(&box->index);
+}
+
+void index_storage_mailbox_free(struct mailbox *box)
+{
+	index_storage_mailbox_unref_indexes(box);
+	event_unref(&box->event);
+}
+
+static void
+index_storage_mailbox_update_cache(struct mailbox *box,
+				   const struct mailbox_update *update)
+{
+	const struct mailbox_cache_field *updates = update->cache_updates;
+	ARRAY(struct mail_cache_field) new_fields;
+	const struct mail_cache_field *old_fields;
+	struct mail_cache_field field;
+	unsigned int i, j, old_count;
+
+	old_fields = mail_cache_register_get_list(box->cache,
+						  pool_datastack_create(),
+						  &old_count);
+
+	/* There shouldn't be many fields, so don't worry about O(n^2). */
+	t_array_init(&new_fields, 32);
+	for (i = 0; updates[i].name != NULL; i++) {
+		/* see if it's an existing field */
+		for (j = 0; j < old_count; j++) {
+			if (strcmp(updates[i].name, old_fields[j].name) == 0)
+				break;
+		}
+		if (j != old_count) {
+			field = old_fields[j];
+		} else if (str_begins(updates[i].name, "hdr.")) {
+			/* new header */
+			i_zero(&field);
+			field.name = updates[i].name;
+			field.type = MAIL_CACHE_FIELD_HEADER;
+		} else {
+			/* new unknown field. we can't do anything about
+			   this since we don't know its type */
+			continue;
+		}
+		field.decision = updates[i].decision;
+		if (updates[i].last_used != (time_t)-1)
+			field.last_used = updates[i].last_used;
+		array_push_back(&new_fields, &field);
+	}
+	if (array_count(&new_fields) > 0) {
+		mail_cache_register_fields(box->cache,
+					   array_front_modifiable(&new_fields),
+					   array_count(&new_fields));
+	}
+}
+
+static int
+index_storage_mailbox_update_pvt(struct mailbox *box,
+				 const struct mailbox_update *update)
+{
+	struct mail_index_transaction *trans;
+	struct mail_index_view *view;
+	int ret;
+
+	if ((ret = mailbox_open_index_pvt(box)) <= 0)
+		return ret;
+
+	mail_index_refresh(box->index_pvt);
+	view = mail_index_view_open(box->index_pvt);
+	trans = mail_index_transaction_begin(view,
+					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+	if (update->min_highest_modseq != 0 &&
+	    mail_index_modseq_get_highest(view) < update->min_highest_pvt_modseq) {
+		mail_index_modseq_enable(box->index_pvt);
+		mail_index_update_highest_modseq(trans,
+						 update->min_highest_pvt_modseq);
+	}
+
+	if ((ret = mail_index_transaction_commit(&trans)) < 0)
+		mailbox_set_index_error(box);
+	mail_index_view_close(&view);
+	return ret;
+}
+
+int index_storage_mailbox_update_common(struct mailbox *box,
+					const struct mailbox_update *update)
+{
+	int ret = 0;
+
+	if (update->cache_updates != NULL)
+		index_storage_mailbox_update_cache(box, update);
+
+	if (update->min_highest_pvt_modseq != 0) {
+		if (index_storage_mailbox_update_pvt(box, update) < 0)
+			ret = -1;
+	}
+	return ret;
+}
+
+int index_storage_mailbox_update(struct mailbox *box,
+				 const struct mailbox_update *update)
+{
+	const struct mail_index_header *hdr;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	int ret;
+
+	if (mailbox_open(box) < 0)
+		return -1;
+
+	/* make sure we get the latest index info */
+	mail_index_refresh(box->index);
+	view = mail_index_view_open(box->index);
+	hdr = mail_index_get_header(view);
+
+	trans = mail_index_transaction_begin(view,
+					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+	if (update->uid_validity != 0 &&
+	    hdr->uid_validity != update->uid_validity) {
+		uint32_t uid_validity = update->uid_validity;
+
+		if (hdr->uid_validity != 0) {
+			/* UIDVALIDITY change requires index to be reset */
+			mail_index_reset(trans);
+		}
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+	if (update->min_next_uid != 0 &&
+	    hdr->next_uid < update->min_next_uid) {
+		uint32_t next_uid = update->min_next_uid;
+
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, next_uid),
+			&next_uid, sizeof(next_uid), FALSE);
+	}
+	if (update->min_first_recent_uid != 0 &&
+	    hdr->first_recent_uid < update->min_first_recent_uid) {
+		uint32_t first_recent_uid = update->min_first_recent_uid;
+
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, first_recent_uid),
+			&first_recent_uid, sizeof(first_recent_uid), FALSE);
+	}
+	if (update->min_highest_modseq != 0 &&
+	    mail_index_modseq_get_highest(view) < update->min_highest_modseq) {
+		mail_index_modseq_enable(box->index);
+		mail_index_update_highest_modseq(trans,
+						 update->min_highest_modseq);
+	}
+
+	if ((ret = mail_index_transaction_commit(&trans)) < 0)
+		mailbox_set_index_error(box);
+	mail_index_view_close(&view);
+	return ret < 0 ? -1 :
+		index_storage_mailbox_update_common(box, update);
+}
+
+int index_storage_mailbox_create(struct mailbox *box, bool directory)
+{
+	const char *path, *p;
+	enum mailbox_list_path_type type;
+	enum mailbox_existence existence;
+	bool create_parent_dir;
+	int ret;
+
+	if ((box->list->props & MAILBOX_LIST_PROP_NO_NOSELECT) != 0) {
+		/* Layout doesn't support creating \NoSelect mailboxes.
+		   Switch to creating a selectable mailbox. */
+		directory = FALSE;
+	}
+
+	type = directory ? MAILBOX_LIST_PATH_TYPE_DIR :
+		MAILBOX_LIST_PATH_TYPE_MAILBOX;
+	if ((ret = mailbox_get_path_to(box, type, &path)) < 0)
+		return -1;
+	if (ret == 0) {
+		/* layout=none */
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE,
+				       "Mailbox creation not supported");
+		return -1;
+	}
+	create_parent_dir = !directory &&
+		(box->list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) != 0;
+	if (create_parent_dir) {
+		/* we only need to make sure that the parent directory exists */
+		p = strrchr(path, '/');
+		if (p == NULL)
+			return 1;
+		path = t_strdup_until(path, p);
+	}
+
+	if ((ret = mailbox_mkdir(box, path, type)) < 0)
+		return -1;
+	if (box->list->set.iter_from_index_dir) {
+		/* need to also create the directory to index path or
+		   iteration won't find it. */
+		int ret2;
+
+		if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0)
+			i_unreached();
+		if ((ret2 = mailbox_mkdir(box, path, type)) < 0)
+			return -1;
+		if (ret == 0 && ret2 > 0) {
+			/* finish partial creation: existed in mail directory,
+			   but not in index directory. */
+			ret = 1;
+		}
+	}
+	mailbox_refresh_permissions(box);
+	if (ret == 0) {
+		/* directory already exists */
+		if (create_parent_dir)
+			return 1;
+		if (!directory && *box->list->set.mailbox_dir_name == '\0') {
+			/* For example: layout=fs, path=~/Maildir/foo
+			   might itself exist, but does it have the
+			   cur|new|tmp subdirs? */
+			if (mailbox_exists(box, FALSE, &existence) < 0)
+				return -1;
+			if (existence != MAILBOX_EXISTENCE_SELECT)
+				return 1;
+		} else if (!box->storage->rebuilding_list_index) {
+			/* ignore existing location if we are recovering list index */
+			mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS,
+					       "Mailbox already exists");
+			return -1;
+		}
+	}
+
+	if (directory) {
+		/* we only wanted to create the directory and it's done now */
+		return 0;
+	}
+	/* the caller should still create the mailbox */
+	return 1;
+}
+
+int index_storage_mailbox_delete_dir(struct mailbox *box, bool mailbox_deleted)
+{
+	guid_128_t dir_sha128;
+	enum mail_error error;
+
+	if (mailbox_list_delete_dir(box->list, box->name) == 0)
+		return 0;
+
+	mailbox_list_get_last_error(box->list, &error);
+	if (error != MAIL_ERROR_NOTFOUND || !mailbox_deleted) {
+		mail_storage_copy_list_error(box->storage, box->list);
+		return -1;
+	}
+	/* failed directory deletion, but mailbox deletion succeeded.
+	   this was probably maildir++, which internally deleted the
+	   directory as well. add changelog record about that too. */
+	mailbox_name_get_sha128(box->vname, dir_sha128);
+	mailbox_list_add_change(box->list, MAILBOX_LOG_RECORD_DELETE_DIR,
+				dir_sha128);
+	return 0;
+}
+
+static int
+mailbox_delete_all_attributes(struct mailbox_transaction_context *t,
+			      enum mail_attribute_type type)
+{
+	struct mailbox_attribute_iter *iter;
+	const char *key;
+	int ret = 0;
+	bool inbox = t->box->inbox_any;
+
+	iter = mailbox_attribute_iter_init(t->box, type, "");
+	while ((key = mailbox_attribute_iter_next(iter)) != NULL) {
+		if (inbox &&
+		    str_begins(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER))
+			continue;
+
+		if (mailbox_attribute_unset(t, type, key) < 0) {
+			if (mailbox_get_last_mail_error(t->box) != MAIL_ERROR_NOTPOSSIBLE) {
+				ret = -1;
+				break;
+			}
+		}
+	}
+	if (mailbox_attribute_iter_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+static int mailbox_expunge_all_data(struct mailbox *box)
+{
+	struct mail_search_context *ctx;
+        struct mailbox_transaction_context *t;
+	struct mail *mail;
+	struct mail_search_args *search_args;
+
+	(void)mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ);
+
+	t = mailbox_transaction_begin(box, 0, __func__);
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_all(search_args);
+	ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(ctx, &mail))
+		mail_expunge(mail);
+
+	if (mailbox_search_deinit(&ctx) < 0) {
+		mailbox_transaction_rollback(&t);
+		return -1;
+	}
+
+	if (mailbox_delete_all_attributes(t, MAIL_ATTRIBUTE_TYPE_PRIVATE) < 0 ||
+	    mailbox_delete_all_attributes(t, MAIL_ATTRIBUTE_TYPE_SHARED) < 0) {
+		mailbox_transaction_rollback(&t);
+		return -1;
+	}
+	if (mailbox_transaction_commit(&t) < 0)
+		return -1;
+	/* sync to actually perform the expunges */
+	return mailbox_sync(box, 0);
+}
+
+int index_storage_mailbox_delete_pre(struct mailbox *box)
+{
+	struct mailbox_status status;
+
+	if (!box->opened) {
+		/* \noselect mailbox, try deleting only the directory */
+		if (index_storage_mailbox_delete_dir(box, FALSE) == 0)
+			return 0;
+		if (mailbox_is_autocreated(box)) {
+			/* Return success when trying to delete autocreated
+			   mailbox. The client sees it as existing, so we
+			   shouldn't be returning an error. */
+			return 0;
+		}
+		return -1;
+	}
+
+	if ((box->list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) == 0) {
+		/* specifically support symlinked shared mailboxes. a deletion
+		   will simply remove the symlink, not actually expunge any
+		   mails */
+		if (mailbox_list_delete_symlink(box->list, box->name) == 0)
+			return 0;
+	}
+
+	/* we can't easily atomically delete all mails and the mailbox. so:
+	   1) expunge all mails
+	   2) mark the mailbox deleted (modifications after this will fail)
+	   3) check if a race condition between 1) and 2) added any mails:
+	     yes) abort and undelete mailbox
+	     no) finish deleting the mailbox
+	*/
+
+	if (!box->deleting_must_be_empty) {
+		if (mailbox_expunge_all_data(box) < 0)
+			return -1;
+	}
+	if (mailbox_mark_index_deleted(box, TRUE) < 0)
+		return -1;
+
+	if (!box->delete_skip_empty_check || box->deleting_must_be_empty) {
+		if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0)
+			return -1;
+		mailbox_get_open_status(box, STATUS_MESSAGES, &status);
+		if (status.messages == 0)
+			;
+		else if (box->deleting_must_be_empty) {
+			mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS,
+					       "Mailbox isn't empty");
+			return -1;
+		} else {
+			mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS,
+				"New mails were added to mailbox during deletion");
+			return -1;
+		}
+	}
+	return 1;
+}
+
+int index_storage_mailbox_delete_post(struct mailbox *box)
+{
+	struct mailbox_metadata metadata;
+	int ret_guid;
+
+	ret_guid = mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata);
+
+	/* Make sure the indexes are closed before trying to delete the
+	   directory that contains them. It can still fail with some NFS
+	   implementations if indexes are opened by another session, but
+	   that can't really be helped. */
+	mailbox_close(box);
+	index_storage_mailbox_unref_indexes(box);
+	mail_index_alloc_cache_destroy_unrefed();
+
+	if (box->list->v.delete_mailbox(box->list, box->name) < 0) {
+		mail_storage_copy_list_error(box->storage, box->list);
+		return -1;
+	} 
+
+	if (ret_guid == 0) {
+		mailbox_list_add_change(box->list,
+					MAILBOX_LOG_RECORD_DELETE_MAILBOX,
+					metadata.guid);
+	}
+	if (index_storage_mailbox_delete_dir(box, TRUE) < 0) {
+		if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXISTS)
+			return -1;
+		/* we deleted the mailbox, but couldn't delete the directory
+		   because it has children. that's not an error. */
+	}
+	return 0;
+}
+
+int index_storage_mailbox_delete(struct mailbox *box)
+{
+	int ret;
+
+	if ((ret = index_storage_mailbox_delete_pre(box)) <= 0)
+		return ret;
+	/* mails have been now successfully deleted. some mailbox formats may
+	   at this point do some other deletion that is required for it.
+	   the _post() deletion will close the index and delete the
+	   directory. */
+	return index_storage_mailbox_delete_post(box);
+}
+
+int index_storage_mailbox_rename(struct mailbox *src, struct mailbox *dest)
+{
+	guid_128_t guid;
+
+	if (src->list->v.rename_mailbox(src->list, src->name,
+					dest->list, dest->name) < 0) {
+		mail_storage_copy_list_error(src->storage, src->list);
+		return -1;
+	}
+
+	if (mailbox_open(dest) == 0) {
+		struct mail_index_transaction *t =
+			mail_index_transaction_begin(dest->view, 0);
+
+		uint32_t stamp = ioloop_time;
+
+		mail_index_update_header_ext(t, dest->box_last_rename_stamp_ext_id,
+					     0, &stamp, sizeof(stamp));
+
+		/* can't do much if this fails anyways */
+		(void)mail_index_transaction_commit(&t);
+	}
+
+	/* we'll track mailbox names, instead of GUIDs. We may be renaming a
+	   non-selectable mailbox (directory), which doesn't even have a GUID */
+	mailbox_name_get_sha128(dest->vname, guid);
+	mailbox_list_add_change(src->list, MAILBOX_LOG_RECORD_RENAME, guid);
+	return 0;
+}
+
+int index_mailbox_update_last_temp_file_scan(struct mailbox *box)
+{
+	uint32_t last_temp_file_scan = ioloop_time;
+	struct mail_index_transaction *trans =
+		mail_index_transaction_begin(box->view,
+			MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+	mail_index_update_header(trans,
+		offsetof(struct mail_index_header, last_temp_file_scan),
+		&last_temp_file_scan, sizeof(last_temp_file_scan), TRUE);
+	if (mail_index_transaction_commit(&trans) < 0) {
+		mailbox_set_index_error(box);
+		return -1;
+	}
+	return 0;
+}
+
+bool index_storage_is_readonly(struct mailbox *box)
+{
+	return (box->flags & MAILBOX_FLAG_READONLY) != 0;
+}
+
+bool index_storage_is_inconsistent(struct mailbox *box)
+{
+	return box->view != NULL &&
+		mail_index_view_is_inconsistent(box->view);
+}
+
+void index_save_context_free(struct mail_save_context *ctx)
+{
+	i_assert(ctx->dest_mail != NULL);
+
+	index_mail_save_finish(ctx);
+	if (ctx->data.keywords != NULL)
+		mailbox_keywords_unref(&ctx->data.keywords);
+	i_free_and_null(ctx->data.from_envelope);
+	i_free_and_null(ctx->data.guid);
+	i_free_and_null(ctx->data.pop3_uidl);
+	index_attachment_save_free(ctx);
+	i_zero(&ctx->data);
+
+	ctx->unfinished = FALSE;
+}
+
+static void
+mail_copy_cache_field(struct mail_save_context *ctx, struct mail *src_mail,
+		      uint32_t dest_seq, const char *name, buffer_t *buf)
+{
+	struct mailbox_transaction_context *dest_trans = ctx->transaction;
+	const struct mail_cache_field *dest_field;
+	unsigned int src_field_idx, dest_field_idx;
+	uint32_t t;
+	bool add = FALSE;
+
+	src_field_idx = mail_cache_register_lookup(src_mail->box->cache, name);
+	i_assert(src_field_idx != UINT_MAX);
+
+	dest_field_idx = mail_cache_register_lookup(dest_trans->box->cache, name);
+	if (dest_field_idx == UINT_MAX) {
+		/* unknown field */
+		return;
+	}
+	dest_field = mail_cache_register_get_field(dest_trans->box->cache,
+						   dest_field_idx);
+	if ((dest_field->decision &
+	     ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) == MAIL_CACHE_DECISION_NO) {
+		/* field not wanted in destination mailbox */
+		return;
+	}
+
+	buffer_set_used_size(buf, 0);
+	if (strcmp(name, "date.save") == 0) {
+		/* save date must update when mail is copied */
+		t = ioloop_time;
+		buffer_append(buf, &t, sizeof(t));
+		add = TRUE;
+	} else if (mail_cache_lookup_field(src_mail->transaction->cache_view, buf,
+					   src_mail->seq, src_field_idx) <= 0) {
+		/* error / not found */
+		buffer_set_used_size(buf, 0);
+	} else {
+		if (strcmp(name, "size.physical") == 0 ||
+		    strcmp(name, "size.virtual") == 0) {
+			/* FIXME: until mail_cache_lookup() can read unwritten
+			   cached data from buffer, we'll do this optimization
+			   to make quota plugin's work faster */
+			struct index_mail *imail =
+				INDEX_MAIL(ctx->dest_mail);
+			uoff_t size;
+
+			i_assert(buf->used == sizeof(size));
+			memcpy(&size, buf->data, sizeof(size));
+			if (strcmp(name, "size.physical") == 0)
+				imail->data.physical_size = size;
+			else
+				imail->data.virtual_size = size;
+		}
+		/* NOTE: we'll want to add also nonexistent headers, which
+		   will keep the buf empty */
+		add = TRUE;
+	}
+	if (add) {
+		mail_cache_add(dest_trans->cache_trans, dest_seq,
+			       dest_field_idx, buf->data, buf->used);
+	}
+}
+
+static void
+index_copy_vsize_extension(struct mail_save_context *ctx,
+			   struct mail *src_mail, uint32_t dest_seq)
+{
+	const uint32_t *vsizep;
+	bool expunged ATTR_UNUSED;
+
+	vsizep = index_mail_get_vsize_extension(src_mail);
+	if (vsizep == NULL || *vsizep == 0)
+		return;
+	uint32_t vsize = *vsizep;
+
+	if (vsize < (uint32_t)-1) {
+		/* copy the vsize record to the destination index */
+		mail_index_update_ext(ctx->transaction->itrans, dest_seq,
+				      ctx->transaction->box->mail_vsize_ext_id,
+				      &vsize, NULL);
+	}
+}
+
+void index_copy_cache_fields(struct mail_save_context *ctx,
+			     struct mail *src_mail, uint32_t dest_seq)
+{
+	T_BEGIN {
+		struct mailbox_metadata src_metadata, dest_metadata;
+		const struct mailbox_cache_field *field;
+		buffer_t *buf;
+
+		if (mailbox_get_metadata(src_mail->box,
+					 MAILBOX_METADATA_CACHE_FIELDS,
+					 &src_metadata) < 0)
+			i_unreached();
+		/* the only reason we're doing the destination lookup is to
+		   make sure that the cache file is opened and the cache
+		   decisions are up to date */
+		if (mailbox_get_metadata(ctx->transaction->box,
+					 MAILBOX_METADATA_CACHE_FIELDS,
+					 &dest_metadata) < 0)
+			i_unreached();
+
+		buf = t_buffer_create(1024);
+		array_foreach(src_metadata.cache_fields, field) {
+			mail_copy_cache_field(ctx, src_mail, dest_seq,
+					      field->name, buf);
+		}
+		index_copy_vsize_extension(ctx, src_mail, dest_seq);
+	} T_END;
+}
+
+int index_storage_set_subscribed(struct mailbox *box, bool set)
+{
+	struct mail_namespace *ns;
+	struct mailbox_list *list = box->list;
+	const char *subs_name;
+	guid_128_t guid;
+
+	if ((list->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) != 0)
+		subs_name = box->name;
+	else {
+		/* subscriptions=no namespace, find another one where we can
+		   add the subscription to */
+		ns = mail_namespace_find_subscribable(list->ns->user->namespaces,
+						      box->vname);
+		if (ns == NULL) {
+			mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE,
+				"This namespace has no subscriptions");
+			return -1;
+		}
+		/* use <orig ns prefix><orig storage name> as the
+		   subscription name */
+		subs_name = t_strconcat(list->ns->prefix, box->name, NULL);
+		/* drop the common prefix (typically there isn't one) */
+		i_assert(str_begins(subs_name, ns->prefix));
+		subs_name += strlen(ns->prefix);
+
+		list = ns->list;
+	}
+	if (mailbox_list_set_subscribed(list, subs_name, set) < 0) {
+		mail_storage_copy_list_error(box->storage, list);
+		return -1;
+	}
+
+	/* subscriptions are about names, not about mailboxes. it's possible
+	   to have a subscription to nonexistent mailbox. renames also don't
+	   change subscriptions. so instead of using actual GUIDs, we'll use
+	   hash of the name. */
+	mailbox_name_get_sha128(box->vname, guid);
+	mailbox_list_add_change(list, set ? MAILBOX_LOG_RECORD_SUBSCRIBE :
+				MAILBOX_LOG_RECORD_UNSUBSCRIBE, guid);
+	return 0;
+}
+
+void index_storage_destroy(struct mail_storage *storage)
+{
+	if (storage->_shared_attr_dict != NULL) {
+		dict_wait(storage->_shared_attr_dict);
+		dict_deinit(&storage->_shared_attr_dict);
+	}
+	fs_unref(&storage->mailboxes_fs);
+}
+
+static void index_storage_expunging_init(struct mailbox *box)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+
+	if (ibox->vsize_update != NULL)
+		return;
+
+	ibox->vsize_update = index_mailbox_vsize_update_init(box);
+	if (!index_mailbox_vsize_want_updates(ibox->vsize_update) ||
+	    !index_mailbox_vsize_update_wait_lock(ibox->vsize_update))
+		index_mailbox_vsize_update_deinit(&ibox->vsize_update);
+}
+
+void index_storage_expunging_deinit(struct mailbox *box)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+
+	if (ibox->vsize_update != NULL)
+		index_mailbox_vsize_update_deinit(&ibox->vsize_update);
+}
+
+static bool index_storage_expunging_want_updates(struct mailbox *box)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	bool ret;
+
+	i_assert(ibox->vsize_update == NULL);
+
+	ibox->vsize_update = index_mailbox_vsize_update_init(box);
+	ret = index_mailbox_vsize_want_updates(ibox->vsize_update);
+	index_mailbox_vsize_update_deinit(&ibox->vsize_update);
+	return ret;
+}
+
+int index_storage_expunged_sync_begin(struct mailbox *box,
+				      struct mail_index_sync_ctx **ctx_r,
+				      struct mail_index_view **view_r,
+				      struct mail_index_transaction **trans_r,
+				      enum mail_index_sync_flags flags)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	int ret;
+
+	/* try to avoid locking vsize updates by checking if we see any
+	   expunges */
+	if (mail_index_sync_have_any_expunges(box->index))
+		index_storage_expunging_init(box);
+
+	ret = mail_index_sync_begin(box->index, ctx_r, view_r,
+				    trans_r, flags);
+	if (ret <= 0) {
+		if (ret < 0)
+			mailbox_set_index_error(box);
+		index_storage_expunging_deinit(box);
+		return ret;
+	}
+	if (ibox->vsize_update == NULL &&
+	    mail_index_sync_has_expunges(*ctx_r) &&
+	    index_storage_expunging_want_updates(box)) {
+		/* race condition - need to abort the sync and retry with
+		   the vsize locked */
+		mail_index_sync_rollback(ctx_r);
+		index_storage_expunging_deinit(box);
+		return index_storage_expunged_sync_begin(box, ctx_r, view_r,
+							 trans_r, flags);
+	}
+	return 1;
+}
+
+int index_storage_save_continue(struct mail_save_context *ctx,
+				struct istream *input,
+				struct mail *cache_dest_mail)
+{
+	struct mail_storage *storage = ctx->transaction->box->storage;
+
+	do {
+		switch (o_stream_send_istream(ctx->data.output, input)) {
+		case OSTREAM_SEND_ISTREAM_RESULT_FINISHED:
+			break;
+		case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT:
+			break;
+		case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT:
+			i_unreached();
+		case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT:
+			/* handle below */
+			break;
+		case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT:
+			if (!mail_storage_set_error_from_errno(storage)) {
+				mail_set_critical(ctx->dest_mail,
+					"save: write(%s) failed: %s",
+					o_stream_get_name(ctx->data.output),
+					o_stream_get_error(ctx->data.output));
+			}
+			return -1;
+		}
+		if (cache_dest_mail != NULL)
+			index_mail_cache_parse_continue(cache_dest_mail);
+
+		/* both tee input readers may consume data from our primary
+		   input stream. we'll have to make sure we don't return with
+		   one of the streams still having data in them. */
+	} while (i_stream_read(input) > 0);
+
+	if (input->stream_errno != 0) {
+		mail_set_critical(ctx->dest_mail, "save: read(%s) failed: %s",
+			i_stream_get_name(input), i_stream_get_error(input));
+		return -1;
+	}
+	return 0;
+}
+
+void index_storage_save_abort_last(struct mail_save_context *ctx, uint32_t seq)
+{
+	struct index_mail *imail = INDEX_MAIL(ctx->dest_mail);
+
+	/* Close the mail before it's expunged. This allows it to be
+	   reset cleanly. */
+	imail->data.no_caching = TRUE;
+	imail->mail.v.close(&imail->mail.mail);
+
+	mail_index_expunge(ctx->transaction->itrans, seq);
+	/* currently we can't just drop pending cache updates for this one
+	   specific record, so we'll reset the whole cache transaction. */
+	mail_cache_transaction_reset(ctx->transaction->cache_trans);
+}
+
+int index_mailbox_fix_inconsistent_existence(struct mailbox *box,
+					     const char *path)
+{
+	const char *index_path;
+	struct stat st;
+
+	/* Could be a race condition or could be because ITERINDEX is used
+	   and the index directory exists, but the storage directory doesn't.
+	   Handle the existence inconsistency by creating this directory if
+	   the index directory exists (don't bother checking if ITERINDEX is
+	   set or not - it doesn't matter since either both dirs should exist
+	   or not). */
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX,
+				&index_path) < 0)
+		return -1;
+
+	if (strcmp(index_path, path) == 0) {
+		/* there's no separate index path - mailbox was just deleted */
+	} else if (stat(index_path, &st) == 0) {
+		/* inconsistency - create also the mail directory */
+		return mailbox_mkdir(box, path, MAILBOX_LIST_PATH_TYPE_MAILBOX);
+	} else if (errno == ENOENT) {
+		/* race condition - mailbox was just deleted */
+	} else {
+		mailbox_set_critical(box, "stat(%s) failed: %m", index_path);
+		return -1;
+	}
+	mailbox_set_deleted(box);
+	return -1;
+}
diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h
new file mode 100644
index 0000000..55cf459
--- /dev/null
+++ b/src/lib-storage/index/index-storage.h
@@ -0,0 +1,193 @@
+#ifndef INDEX_STORAGE_H
+#define INDEX_STORAGE_H
+
+#include "file-dotlock.h"
+#include "mail-storage-private.h"
+#include "mail-index-private.h"
+#include "mailbox-watch.h"
+
+#define MAILBOX_FULL_SYNC_INTERVAL 5
+
+enum mailbox_lock_notify_type {
+	MAILBOX_LOCK_NOTIFY_NONE,
+
+	/* Mailbox is locked, will abort in secs_left */
+	MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT,
+	/* Mailbox lock looks stale, will override in secs_left */
+	MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE
+};
+
+enum index_storage_list_change {
+	INDEX_STORAGE_LIST_CHANGE_ERROR = -1,
+	INDEX_STORAGE_LIST_CHANGE_NONE = 0,
+	INDEX_STORAGE_LIST_CHANGE_INMEMORY,
+	INDEX_STORAGE_LIST_CHANGE_NORECORD,
+	INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS,
+	INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED,
+	INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED
+};
+
+struct index_mailbox_context {
+	union mailbox_module_context module_ctx;
+	enum mail_index_open_flags index_flags;
+
+	time_t next_lock_notify; /* temporary */
+	enum mailbox_lock_notify_type last_notify_type;
+
+	const ARRAY_TYPE(keywords) *keyword_names;
+	struct mail_cache_field *cache_fields;
+
+	struct mailbox_vsize_update *vsize_update;
+
+	uint32_t recent_flags_prev_first_recent_uid;
+	uint32_t recent_flags_last_check_nextuid;
+
+	time_t sync_last_check;
+	uint32_t list_index_sync_ext_id;
+};
+
+#define INDEX_STORAGE_CONTEXT(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, index_storage_module)
+extern MODULE_CONTEXT_DEFINE(index_storage_module,
+			     &mail_storage_module_register);
+
+void index_storage_lock_notify(struct mailbox *box,
+			       enum mailbox_lock_notify_type notify_type,
+			       unsigned int secs_left);
+void index_storage_lock_notify_reset(struct mailbox *box);
+
+int index_storage_mailbox_alloc_index(struct mailbox *box);
+void index_storage_mailbox_alloc(struct mailbox *box, const char *vname,
+				 enum mailbox_flags flags,
+				 const char *index_prefix);
+int index_storage_mailbox_exists(struct mailbox *box, bool auto_boxes,
+				 enum mailbox_existence *existence_r);
+int index_storage_mailbox_exists_full(struct mailbox *box, const char *subdir,
+				      enum mailbox_existence *existence_r)
+	ATTR_NULL(2);
+int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory);
+int index_storage_mailbox_enable(struct mailbox *box,
+				 enum mailbox_feature feature);
+void index_storage_mailbox_close(struct mailbox *box);
+void index_storage_mailbox_free(struct mailbox *box);
+int index_storage_mailbox_update(struct mailbox *box,
+				 const struct mailbox_update *update);
+int index_storage_mailbox_update_common(struct mailbox *box,
+					const struct mailbox_update *update);
+int index_storage_mailbox_create(struct mailbox *box, bool directory);
+int index_storage_mailbox_delete_pre(struct mailbox *box);
+int index_storage_mailbox_delete_post(struct mailbox *box);
+int index_storage_mailbox_delete(struct mailbox *box);
+int index_storage_mailbox_delete_dir(struct mailbox *box, bool mailbox_deleted);
+int index_storage_mailbox_rename(struct mailbox *src, struct mailbox *dest);
+
+int index_mailbox_update_last_temp_file_scan(struct mailbox *box);
+int index_mailbox_fix_inconsistent_existence(struct mailbox *box,
+					     const char *path);
+
+bool index_storage_is_readonly(struct mailbox *box);
+bool index_storage_is_inconsistent(struct mailbox *box);
+
+enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box);
+bool index_mailbox_want_full_sync(struct mailbox *box,
+				  enum mailbox_sync_flags flags);
+struct mailbox_sync_context *
+index_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags,
+			bool failed);
+bool index_mailbox_sync_next(struct mailbox_sync_context *ctx,
+			     struct mailbox_sync_rec *sync_rec_r);
+int index_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
+			      struct mailbox_sync_status *status_r);
+
+int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags);
+enum mailbox_sync_type index_sync_type_convert(enum mail_index_sync_type type);
+void index_sync_update_recent_count(struct mailbox *box);
+int index_storage_get_status(struct mailbox *box,
+			     enum mailbox_status_items items,
+			     struct mailbox_status *status_r);
+void index_storage_get_open_status(struct mailbox *box,
+				   enum mailbox_status_items items,
+				   struct mailbox_status *status_r);
+int index_mailbox_get_metadata(struct mailbox *box,
+			       enum mailbox_metadata_items items,
+			       struct mailbox_metadata *metadata_r);
+int index_mailbox_get_virtual_size(struct mailbox *box,
+				   struct mailbox_metadata *metadata_r);
+int index_mailbox_get_physical_size(struct mailbox *box,
+				    struct mailbox_metadata *metadata_r);
+
+int index_storage_attribute_set(struct mailbox_transaction_context *t,
+				enum mail_attribute_type type_flags,
+				const char *key,
+				const struct mail_attribute_value *value);
+int index_storage_attribute_get(struct mailbox *box,
+				enum mail_attribute_type type_flags,
+				const char *key,
+				struct mail_attribute_value *value_r);
+struct mailbox_attribute_iter *
+index_storage_attribute_iter_init(struct mailbox *box,
+				  enum mail_attribute_type type_flags,
+				  const char *prefix);
+const char *
+index_storage_attribute_iter_next(struct mailbox_attribute_iter *iter);
+int index_storage_attribute_iter_deinit(struct mailbox_attribute_iter *iter);
+
+struct mail_search_context *
+index_storage_search_init(struct mailbox_transaction_context *t,
+			  struct mail_search_args *args,
+			  const enum mail_sort_type *sort_program,
+			  enum mail_fetch_field wanted_fields,
+			  struct mailbox_header_lookup_ctx *wanted_headers);
+int index_storage_search_deinit(struct mail_search_context *ctx);
+bool index_storage_search_next_nonblock(struct mail_search_context *ctx,
+					struct mail **mail_r, bool *tryagain_r);
+bool index_storage_search_next_update_seq(struct mail_search_context *ctx);
+int index_storage_search_next_match_mail(struct mail_search_context *ctx,
+					 struct mail *mail);
+
+struct mailbox_transaction_context *
+index_transaction_begin(struct mailbox *box,
+			enum mailbox_transaction_flags flags,
+			const char *reason);
+void index_transaction_init(struct mailbox_transaction_context *t,
+			    struct mailbox *box,
+			    enum mailbox_transaction_flags flags,
+			    const char *reason);
+void index_transaction_init_pvt(struct mailbox_transaction_context *t);
+int index_transaction_commit(struct mailbox_transaction_context *t,
+			     struct mail_transaction_commit_changes *changes_r);
+void index_transaction_rollback(struct mailbox_transaction_context *t);
+void index_save_context_free(struct mail_save_context *ctx);
+void index_copy_cache_fields(struct mail_save_context *ctx,
+			     struct mail *src_mail, uint32_t dest_seq);
+int index_storage_set_subscribed(struct mailbox *box, bool set);
+void index_storage_destroy(struct mail_storage *storage);
+
+bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1,
+			     const ARRAY_TYPE(keyword_indexes) *k2);
+
+int index_storage_list_index_has_changed(struct mailbox *box,
+					 struct mail_index_view *list_view,
+					 uint32_t seq, bool quick,
+					 const char **reason_r);
+enum index_storage_list_change
+index_storage_list_index_has_changed_full(struct mailbox *box,
+					  struct mail_index_view *list_view,
+					  uint32_t seq, const char **reason_r);
+void index_storage_list_index_update_sync(struct mailbox *box,
+					  struct mail_index_transaction *trans,
+					  uint32_t seq);
+
+int index_storage_expunged_sync_begin(struct mailbox *box,
+				      struct mail_index_sync_ctx **ctx_r,
+				      struct mail_index_view **view_r,
+				      struct mail_index_transaction **trans_r,
+				      enum mail_index_sync_flags flags);
+void index_storage_expunging_deinit(struct mailbox *box);
+
+int index_storage_save_continue(struct mail_save_context *ctx,
+				struct istream *input,
+				struct mail *cache_dest_mail);
+void index_storage_save_abort_last(struct mail_save_context *ctx, uint32_t seq);
+
+#endif
diff --git a/src/lib-storage/index/index-sync-changes.c b/src/lib-storage/index/index-sync-changes.c
new file mode 100644
index 0000000..6d8cd99
--- /dev/null
+++ b/src/lib-storage/index/index-sync-changes.c
@@ -0,0 +1,200 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "index-storage.h"
+#include "index-sync-changes.h"
+
+struct index_sync_changes_context {
+	struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *sync_trans;
+
+	ARRAY(struct mail_index_sync_rec) syncs;
+	struct mail_index_sync_rec sync_rec;
+	bool dirty_flag_updates;
+};
+
+struct index_sync_changes_context *
+index_sync_changes_init(struct mail_index_sync_ctx *index_sync_ctx,
+			struct mail_index_view *sync_view,
+			struct mail_index_transaction *sync_trans,
+			bool dirty_flag_updates)
+{
+	struct index_sync_changes_context *ctx;
+
+	ctx = i_new(struct index_sync_changes_context, 1);
+	ctx->index_sync_ctx = index_sync_ctx;
+	ctx->sync_view = sync_view;
+	ctx->sync_trans = sync_trans;
+	ctx->dirty_flag_updates = dirty_flag_updates;
+	i_array_init(&ctx->syncs, 16);
+	return ctx;
+}
+
+void index_sync_changes_deinit(struct index_sync_changes_context **_ctx)
+{
+	struct index_sync_changes_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+	array_free(&ctx->syncs);
+	i_free(ctx);
+}
+
+void index_sync_changes_reset(struct index_sync_changes_context *ctx)
+{
+	array_clear(&ctx->syncs);
+	i_zero(&ctx->sync_rec);
+}
+
+void index_sync_changes_delete_to(struct index_sync_changes_context *ctx,
+				  uint32_t last_uid)
+{
+	struct mail_index_sync_rec *syncs;
+	unsigned int src, dest, count;
+
+	syncs = array_get_modifiable(&ctx->syncs, &count);
+
+	for (src = dest = 0; src < count; src++) {
+		i_assert(last_uid >= syncs[src].uid1);
+		if (last_uid <= syncs[src].uid2) {
+			/* keep it */
+			if (src != dest)
+				syncs[dest] = syncs[src];
+			dest++;
+		}
+	}
+
+	array_delete(&ctx->syncs, dest, count - dest);
+}
+
+static bool
+index_sync_changes_have_expunges(struct index_sync_changes_context *ctx,
+				 unsigned int count,
+				 guid_128_t expunged_guid_128_r)
+{
+	const struct mail_index_sync_rec *syncs;
+	unsigned int i;
+
+	syncs = array_front(&ctx->syncs);
+	for (i = 0; i < count; i++) {
+		if (syncs[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
+			memcpy(expunged_guid_128_r, syncs[i].guid_128,
+			       GUID_128_SIZE);
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+void index_sync_changes_read(struct index_sync_changes_context *ctx,
+			     uint32_t uid, bool *sync_expunge_r,
+			     guid_128_t expunged_guid_128_r)
+{
+	struct mail_index_sync_rec *sync_rec = &ctx->sync_rec;
+	uint32_t seq1, seq2;
+	unsigned int orig_count;
+
+	*sync_expunge_r = FALSE;
+
+	index_sync_changes_delete_to(ctx, uid);
+	orig_count = array_count(&ctx->syncs);
+
+	while (uid >= sync_rec->uid1) {
+		if (uid <= sync_rec->uid2) {
+			array_push_back(&ctx->syncs, sync_rec);
+
+			if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
+				*sync_expunge_r = TRUE;
+				memcpy(expunged_guid_128_r, sync_rec->guid_128,
+				       GUID_128_SIZE);
+			}
+		}
+
+		if (!mail_index_sync_next(ctx->index_sync_ctx, sync_rec)) {
+			i_zero(sync_rec);
+			break;
+		}
+
+		switch (sync_rec->type) {
+		case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
+			break;
+		case MAIL_INDEX_SYNC_TYPE_FLAGS:
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
+			if (!ctx->dirty_flag_updates)
+				break;
+
+			/* mark the changes as dirty */
+			(void)mail_index_lookup_seq_range(ctx->sync_view,
+							  sync_rec->uid1,
+							  sync_rec->uid2,
+							  &seq1, &seq2);
+			i_zero(sync_rec);
+
+			if (seq1 == 0)
+				break;
+
+			mail_index_update_flags_range(ctx->sync_trans,
+				seq1, seq2, MODIFY_ADD,
+				(enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
+			break;
+		}
+	}
+
+	if (!*sync_expunge_r && orig_count > 0) {
+		*sync_expunge_r =
+			index_sync_changes_have_expunges(ctx, orig_count,
+							 expunged_guid_128_r);
+	}
+}
+
+bool index_sync_changes_have(struct index_sync_changes_context *ctx)
+{
+	return array_count(&ctx->syncs) > 0;
+}
+
+uint32_t
+index_sync_changes_get_next_uid(struct index_sync_changes_context *ctx)
+{
+	return ctx->sync_rec.uid1;
+}
+
+void index_sync_changes_apply(struct index_sync_changes_context *ctx,
+			      pool_t pool, uint8_t *flags,
+			      ARRAY_TYPE(keyword_indexes) *keywords,
+			      enum mail_index_sync_type *sync_type_r)
+{
+	const struct mail_index_sync_rec *syncs;
+	unsigned int i, count;
+	enum mail_index_sync_type sync_type = 0;
+
+	syncs = array_get(&ctx->syncs, &count);
+	for (i = 0; i < count; i++) {
+		switch (syncs[i].type) {
+		case MAIL_INDEX_SYNC_TYPE_FLAGS:
+			mail_index_sync_flags_apply(&syncs[i], flags);
+			sync_type |= MAIL_INDEX_SYNC_TYPE_FLAGS;
+			break;
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
+			if (!array_is_created(keywords)) {
+				/* no existing keywords */
+				if (syncs[i].type !=
+				    MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD)
+					break;
+
+				/* adding, create the array */
+				p_array_init(keywords, pool,
+					     I_MIN(10, count - i));
+			}
+			if (mail_index_sync_keywords_apply(&syncs[i], keywords))
+				sync_type |= syncs[i].type;
+			break;
+		default:
+			break;
+		}
+	}
+
+	*sync_type_r = sync_type;
+}
diff --git a/src/lib-storage/index/index-sync-changes.h b/src/lib-storage/index/index-sync-changes.h
new file mode 100644
index 0000000..e6f5d47
--- /dev/null
+++ b/src/lib-storage/index/index-sync-changes.h
@@ -0,0 +1,28 @@
+#ifndef INDEX_SYNC_CHANGES_H
+#define INDEX_SYNC_CHANGES_H
+
+struct index_sync_changes_context *
+index_sync_changes_init(struct mail_index_sync_ctx *index_sync_ctx,
+			struct mail_index_view *sync_view,
+			struct mail_index_transaction *sync_trans,
+			bool dirty_flag_updates);
+void index_sync_changes_deinit(struct index_sync_changes_context **_ctx);
+
+void index_sync_changes_reset(struct index_sync_changes_context *ctx);
+void index_sync_changes_delete_to(struct index_sync_changes_context *ctx,
+				  uint32_t last_uid);
+
+void index_sync_changes_read(struct index_sync_changes_context *ctx,
+			     uint32_t uid, bool *sync_expunge_r,
+			     guid_128_t expunged_guid_128);
+bool index_sync_changes_have(struct index_sync_changes_context *ctx);
+uint32_t
+index_sync_changes_get_next_uid(struct index_sync_changes_context *ctx);
+
+void index_sync_changes_apply(struct index_sync_changes_context *ctx,
+			      pool_t pool, uint8_t *flags,
+			      ARRAY_TYPE(keyword_indexes) *keywords,
+			      enum mail_index_sync_type *sync_type_r)
+	ATTR_NULL(2);
+
+#endif
diff --git a/src/lib-storage/index/index-sync-private.h b/src/lib-storage/index/index-sync-private.h
new file mode 100644
index 0000000..7d670c4
--- /dev/null
+++ b/src/lib-storage/index/index-sync-private.h
@@ -0,0 +1,38 @@
+#ifndef INDEX_SYNC_PRIVATE_H
+#define INDEX_SYNC_PRIVATE_H
+
+#include "index-storage.h"
+
+struct index_mailbox_sync_pvt_context;
+
+struct index_mailbox_sync_context {
+	struct mailbox_sync_context ctx;
+
+	struct mail_index_view_sync_ctx *sync_ctx;
+	uint32_t messages_count;
+
+	ARRAY_TYPE(seq_range) flag_updates;
+	ARRAY_TYPE(seq_range) hidden_updates;
+	ARRAY_TYPE(seq_range) all_flag_update_uids;
+	const ARRAY_TYPE(seq_range) *expunges;
+	unsigned int flag_update_idx, hidden_update_idx, expunge_pos;
+
+	bool failed;
+};
+
+void index_sync_search_results_uidify(struct index_mailbox_sync_context *ctx);
+void index_sync_search_results_update(struct index_mailbox_sync_context *ctx);
+void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx);
+
+/* Returns 1 = ok, 0 = no private indexes, -1 = error */
+int index_mailbox_sync_pvt_init(struct mailbox *box, bool lock,
+				enum mail_index_view_sync_flags flags,
+				struct index_mailbox_sync_pvt_context **ctx_r);
+int index_mailbox_sync_pvt_newmails(struct index_mailbox_sync_pvt_context *ctx,
+				    struct mailbox_transaction_context *trans);
+int index_mailbox_sync_pvt_view(struct index_mailbox_sync_pvt_context *ctx,
+				ARRAY_TYPE(seq_range) *flag_updates,
+				ARRAY_TYPE(seq_range) *hidden_updates);
+void index_mailbox_sync_pvt_deinit(struct index_mailbox_sync_pvt_context **ctx);
+
+#endif
diff --git a/src/lib-storage/index/index-sync-pvt.c b/src/lib-storage/index/index-sync-pvt.c
new file mode 100644
index 0000000..92eb4e8
--- /dev/null
+++ b/src/lib-storage/index/index-sync-pvt.c
@@ -0,0 +1,345 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mailbox-list-private.h"
+#include "index-sync-private.h"
+
+struct index_mailbox_sync_pvt_context {
+	struct mailbox *box;
+
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *view_pvt;
+	struct mail_index_transaction *trans_pvt;
+	struct mail_index_view *view_shared;
+
+	enum mail_index_view_sync_flags flags;
+};
+
+static int sync_pvt_expunges(struct index_mailbox_sync_pvt_context *ctx)
+{
+	uint32_t seq_shared, seq_pvt, count_shared, count_pvt;
+	uint32_t uid_shared, uid_pvt;
+
+	count_shared = mail_index_view_get_messages_count(ctx->view_shared);
+	count_pvt = mail_index_view_get_messages_count(ctx->view_pvt);
+	seq_shared = seq_pvt = 1;
+	while (seq_pvt <= count_pvt && seq_shared <= count_shared) {
+		mail_index_lookup_uid(ctx->view_pvt, seq_pvt, &uid_pvt);
+		mail_index_lookup_uid(ctx->view_shared, seq_shared, &uid_shared);
+		if (uid_pvt == uid_shared) {
+			seq_pvt++;
+			seq_shared++;
+		} else if (uid_pvt < uid_shared) {
+			/* message expunged */
+			mail_index_expunge(ctx->trans_pvt, seq_pvt);
+			seq_pvt++;
+		} else {
+			mailbox_set_critical(ctx->box,
+				"%s: Message UID=%u unexpectedly inserted to mailbox",
+				ctx->box->index_pvt->filepath, uid_shared);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static void
+sync_pvt_copy_self_flags(struct index_mailbox_sync_pvt_context *ctx,
+			 ARRAY_TYPE(keyword_indexes) *keywords,
+			 uint32_t seq_old, uint32_t seq_new)
+{
+	const struct mail_index_record *old_rec;
+
+	old_rec = mail_index_lookup(ctx->view_pvt, seq_old);
+	mail_index_lookup_keywords(ctx->view_pvt, seq_old, keywords);
+	if (old_rec->flags != 0) {
+		mail_index_update_flags(ctx->trans_pvt, seq_new,
+					MODIFY_ADD, old_rec->flags);
+	}
+	if (array_count(keywords) > 0) {
+		struct mail_keywords *kw;
+
+		kw = mail_index_keywords_create_from_indexes(ctx->box->index_pvt,
+							     keywords);
+		mail_index_update_keywords(ctx->trans_pvt, seq_new,
+					   MODIFY_ADD, kw);
+		mail_index_keywords_unref(&kw);
+	}
+}
+
+static void
+sync_pvt_copy_shared_flags(struct index_mailbox_sync_pvt_context *ctx,
+			   uint32_t seq_shared, uint32_t seq_pvt)
+{
+	const struct mail_index_record *rec;
+
+	rec = mail_index_lookup(ctx->view_shared, seq_shared);
+	mail_index_update_flags(ctx->trans_pvt, seq_pvt, MODIFY_ADD,
+		rec->flags & mailbox_get_private_flags_mask(ctx->box));
+}
+
+static int
+index_mailbox_sync_view_refresh(struct index_mailbox_sync_pvt_context *ctx)
+{
+	/* open a view for the latest version of the index */
+	if (mail_index_refresh(ctx->box->index_pvt) < 0 ||
+	    mail_index_refresh(ctx->box->index) < 0) {
+		mailbox_set_index_error(ctx->box);
+		return -1;
+	}
+	if (ctx->view_shared != NULL)
+		mail_index_view_close(&ctx->view_shared);
+	ctx->view_shared = mail_index_view_open(ctx->box->index);
+	return 0;
+}
+
+static int
+index_mailbox_sync_open(struct index_mailbox_sync_pvt_context *ctx, bool force)
+{
+	const struct mail_index_header *hdr_shared, *hdr_pvt;
+
+	if (index_mailbox_sync_view_refresh(ctx) < 0)
+		return -1;
+
+	hdr_shared = mail_index_get_header(ctx->view_shared);
+	if (hdr_shared->uid_validity == 0 && !force) {
+		/* the mailbox hasn't been fully created yet,
+		   no need for a private index yet */
+		return 0;
+	}
+	hdr_pvt = mail_index_get_header(ctx->box->view_pvt);
+	if (hdr_pvt->next_uid == hdr_shared->next_uid &&
+	    hdr_pvt->messages_count == hdr_shared->messages_count && !force) {
+		/* no new or expunged mails, don't bother syncing */
+		return 0;
+	}
+	if (mail_index_sync_begin(ctx->box->index_pvt, &ctx->sync_ctx,
+				  &ctx->view_pvt, &ctx->trans_pvt, 0) < 0) {
+		mailbox_set_index_error(ctx->box);
+		return -1;
+	}
+	/* refresh once more now that we're locked */
+	if (index_mailbox_sync_view_refresh(ctx) < 0)
+		return -1;
+	return 1;
+}
+
+int index_mailbox_sync_pvt_init(struct mailbox *box, bool lock,
+				enum mail_index_view_sync_flags flags,
+				struct index_mailbox_sync_pvt_context **ctx_r)
+{
+	struct index_mailbox_sync_pvt_context *ctx;
+	int ret;
+
+	*ctx_r = NULL;
+
+	if ((ret = mailbox_open_index_pvt(box)) <= 0)
+		return ret;
+
+	ctx = i_new(struct index_mailbox_sync_pvt_context, 1);
+	ctx->box = box;
+	ctx->flags = flags;
+	if (lock) {
+		if (index_mailbox_sync_open(ctx, TRUE) < 0) {
+			index_mailbox_sync_pvt_deinit(&ctx);
+			return -1;
+		}
+	}
+
+	*ctx_r = ctx;
+	return 1;
+}
+
+static int
+index_mailbox_sync_pvt_index(struct index_mailbox_sync_pvt_context *ctx,
+			     const struct mail_save_private_changes *pvt_changes,
+			     unsigned int pvt_changes_count)
+{
+	const struct mail_index_header *hdr_shared, *hdr_pvt;
+	ARRAY_TYPE(keyword_indexes) keywords;
+	uint32_t seq_shared, seq_pvt, seq_old_pvt, seq2, count_shared, uid;
+	unsigned int pc_idx = 0;
+	bool reset = FALSE, preserve_old_flags = FALSE, copy_shared_flags;
+	bool initial_index = FALSE;
+	int ret;
+
+	if (ctx->sync_ctx == NULL) {
+		if ((ret = index_mailbox_sync_open(ctx, FALSE)) <= 0)
+			return ret;
+	}
+	hdr_pvt = mail_index_get_header(ctx->view_pvt);
+	hdr_shared = mail_index_get_header(ctx->view_shared);
+
+	if (hdr_shared->uid_validity == hdr_pvt->uid_validity) {
+		/* same mailbox. expunge messages from private index that
+		   no longer exist. */
+		if (sync_pvt_expunges(ctx) < 0) {
+			reset = TRUE;
+			preserve_old_flags = TRUE;
+			t_array_init(&keywords, 32);
+		}
+	} else if (hdr_pvt->uid_validity == 0 && hdr_pvt->next_uid <= 1) {
+		/* creating the initial index - no logging */
+		reset = TRUE;
+		initial_index = TRUE;
+	} else {
+		/* mailbox created/recreated */
+		reset = TRUE;
+		i_info("Mailbox %s UIDVALIDITY changed (%u -> %u), reseting private index",
+		       ctx->box->vname, hdr_pvt->uid_validity,
+		       hdr_shared->uid_validity);
+	}
+	/* for public namespaces copy the initial private flags from the shared
+	   index. this allows Sieve scripts to set the initial flags. */
+	copy_shared_flags =
+		ctx->box->list->ns->type == MAIL_NAMESPACE_TYPE_PUBLIC;
+
+	count_shared = mail_index_view_get_messages_count(ctx->view_shared);
+	if (!reset) {
+		if (!mail_index_lookup_seq_range(ctx->view_shared,
+						 hdr_pvt->next_uid,
+						 hdr_shared->next_uid,
+						 &seq_shared, &seq2)) {
+			/* no new messages */
+			seq_shared = count_shared+1;
+		}
+	} else {
+		if (!initial_index)
+			mail_index_reset(ctx->trans_pvt);
+		mail_index_update_header(ctx->trans_pvt,
+			offsetof(struct mail_index_header, uid_validity),
+			&hdr_shared->uid_validity,
+			sizeof(hdr_shared->uid_validity), TRUE);
+		seq_shared = 1;
+	}
+
+	uid = 0;
+	for (; seq_shared <= count_shared; seq_shared++) {
+		mail_index_lookup_uid(ctx->view_shared, seq_shared, &uid);
+		mail_index_append(ctx->trans_pvt, uid, &seq_pvt);
+		if (preserve_old_flags &&
+		    mail_index_lookup_seq(ctx->view_pvt, uid, &seq_old_pvt)) {
+			/* copy flags from the original private index */
+			sync_pvt_copy_self_flags(ctx, &keywords,
+						 seq_old_pvt, seq_pvt);
+		} else if (copy_shared_flags) {
+			sync_pvt_copy_shared_flags(ctx, seq_shared, seq_pvt);
+		}
+
+		/* add private flags for the recently saved/copied messages */
+		while (pc_idx < pvt_changes_count &&
+		       pvt_changes[pc_idx].mailnum <= uid) {
+			if (pvt_changes[pc_idx].mailnum == uid) {
+				mail_index_update_flags(ctx->trans_pvt, seq_pvt,
+					MODIFY_ADD, pvt_changes[pc_idx].flags);
+			}
+			pc_idx++;
+		}
+	}
+
+	if (uid < hdr_shared->next_uid) {
+		mail_index_update_header(ctx->trans_pvt,
+			offsetof(struct mail_index_header, next_uid),
+			&hdr_shared->next_uid,
+			sizeof(hdr_shared->next_uid), FALSE);
+	}
+
+	if ((ret = mail_index_sync_commit(&ctx->sync_ctx)) < 0)
+		mailbox_set_index_error(ctx->box);
+	return ret;
+}
+
+static int
+mail_save_private_changes_mailnum_cmp(const struct mail_save_private_changes *c1,
+				      const struct mail_save_private_changes *c2)
+{
+	if (c1->mailnum < c2->mailnum)
+		return -1;
+	if (c1->mailnum > c2->mailnum)
+		return 1;
+	return 0;
+}
+
+int index_mailbox_sync_pvt_newmails(struct index_mailbox_sync_pvt_context *ctx,
+				    struct mailbox_transaction_context *trans)
+{
+	struct mail_save_private_changes *pvt_changes;
+	struct seq_range_iter iter;
+	unsigned int i, n, pvt_count;
+	uint32_t uid;
+
+	if (index_mailbox_sync_view_refresh(ctx) < 0)
+		return -1;
+
+	/* translate mail numbers to UIDs */
+	pvt_changes = array_get_modifiable(&trans->pvt_saves, &pvt_count);
+
+	n = i = 0;
+	seq_range_array_iter_init(&iter, &trans->changes->saved_uids);
+	while (seq_range_array_iter_nth(&iter, n, &uid)) {
+		if (pvt_changes[i].mailnum == n) {
+			pvt_changes[i].mailnum = uid;
+			i++;
+		}
+		n++;
+	}
+	/* sort the changes by UID */
+	array_sort(&trans->pvt_saves, mail_save_private_changes_mailnum_cmp);
+
+	/* add new mails to the private index with the private flags */
+	return index_mailbox_sync_pvt_index(ctx, pvt_changes, pvt_count);
+}
+
+int index_mailbox_sync_pvt_view(struct index_mailbox_sync_pvt_context *ctx,
+				ARRAY_TYPE(seq_range) *flag_updates,
+				ARRAY_TYPE(seq_range) *hidden_updates)
+{
+	struct mail_index_view_sync_ctx *view_sync_ctx;
+	struct mail_index_view_sync_rec sync_rec;
+	uint32_t seq1, seq2;
+	bool delayed_expunges;
+
+	/* sync private index against shared index by adding/removing mails */
+	if (index_mailbox_sync_pvt_index(ctx, NULL, 0) < 0)
+		return -1;
+
+	/* Indicate to view syncing that this is a secondary index view */
+	ctx->flags |= MAIL_INDEX_VIEW_SYNC_FLAG_2ND_INDEX;
+
+	/* sync the private view */
+	view_sync_ctx = mail_index_view_sync_begin(ctx->box->view_pvt, ctx->flags);
+	while (mail_index_view_sync_next(view_sync_ctx, &sync_rec)) {
+		if (sync_rec.type != MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS)
+			continue;
+
+		/* *_updates contains ctx->box->view sequences (not view_pvt
+		   sequences) */
+		if (mail_index_lookup_seq_range(ctx->box->view,
+						sync_rec.uid1, sync_rec.uid2,
+						&seq1, &seq2)) {
+			if (!sync_rec.hidden) {
+				seq_range_array_add_range(flag_updates,
+							  seq1, seq2);
+			} else {
+				seq_range_array_add_range(hidden_updates,
+							  seq1, seq2);
+			}
+		}
+	}
+	if (mail_index_view_sync_commit(&view_sync_ctx, &delayed_expunges) < 0)
+		return -1;
+	return 0;
+}
+
+void index_mailbox_sync_pvt_deinit(struct index_mailbox_sync_pvt_context **_ctx)
+{
+	struct index_mailbox_sync_pvt_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+
+	if (ctx->sync_ctx != NULL)
+		mail_index_sync_rollback(&ctx->sync_ctx);
+	if (ctx->view_shared != NULL)
+		mail_index_view_close(&ctx->view_shared);
+	i_free(ctx);
+}
diff --git a/src/lib-storage/index/index-sync-search.c b/src/lib-storage/index/index-sync-search.c
new file mode 100644
index 0000000..95cb89d
--- /dev/null
+++ b/src/lib-storage/index/index-sync-search.c
@@ -0,0 +1,97 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "seq-range-array.h"
+#include "mail-search.h"
+#include "mailbox-search-result-private.h"
+#include "index-search-result.h"
+#include "index-sync-private.h"
+
+static bool
+search_result_want_flag_updates(const struct mail_search_result *result)
+{
+	if (!result->args_have_flags && !result->args_have_keywords &&
+	    !result->args_have_modseq) {
+		/* search result doesn't care about flag changes */
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static void index_sync_uidify_array(struct index_mailbox_sync_context *ctx,
+				    const ARRAY_TYPE(seq_range) *changes)
+{
+	const struct seq_range *range;
+	uint32_t seq, uid;
+
+	array_foreach(changes, range) {
+		for (seq = range->seq1; seq <= range->seq2; seq++) {
+			mail_index_lookup_uid(ctx->ctx.box->view, seq, &uid);
+			seq_range_array_add(&ctx->all_flag_update_uids, uid);
+		}
+	}
+}
+
+static void index_sync_uidify(struct index_mailbox_sync_context *ctx)
+{
+	unsigned int count;
+
+	count = array_count(&ctx->flag_updates) +
+		array_count(&ctx->hidden_updates);
+	i_array_init(&ctx->all_flag_update_uids, count*2);
+
+	index_sync_uidify_array(ctx, &ctx->flag_updates);
+	index_sync_uidify_array(ctx, &ctx->hidden_updates);
+}
+
+void index_sync_search_results_uidify(struct index_mailbox_sync_context *ctx)
+{
+	struct mail_search_result *const *results;
+	unsigned int i, count;
+
+	i_assert(!array_is_created(&ctx->all_flag_update_uids));
+
+	results = array_get(&ctx->ctx.box->search_results, &count);
+	for (i = 0; i < count; i++) {
+		if ((results[i]->flags & MAILBOX_SEARCH_RESULT_FLAG_UPDATE) != 0 &&
+		    search_result_want_flag_updates(results[i])) {
+			index_sync_uidify(ctx);
+			break;
+		}
+	}
+}
+
+static void
+search_result_update(struct index_mailbox_sync_context *ctx,
+		     struct mail_search_result *result)
+{
+	if ((result->flags & MAILBOX_SEARCH_RESULT_FLAG_UPDATE) == 0) {
+		/* not an updateable search result */
+		return;
+	}
+
+	if (search_result_want_flag_updates(result)) {
+		(void)index_search_result_update_flags(result,
+						&ctx->all_flag_update_uids);
+	}
+	(void)index_search_result_update_appends(result, ctx->messages_count);
+}
+
+void index_sync_search_results_update(struct index_mailbox_sync_context *ctx)
+{
+	struct mail_search_result *const *results;
+	unsigned int i, count;
+
+	results = array_get(&ctx->ctx.box->search_results, &count);
+	for (i = 0; i < count; i++)
+		search_result_update(ctx, results[i]);
+}
+
+void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx)
+{
+	if (ctx->expunges != NULL) {
+		index_search_results_update_expunges(ctx->ctx.box,
+						     ctx->expunges);
+	}
+}
diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c
new file mode 100644
index 0000000..73ea99c
--- /dev/null
+++ b/src/lib-storage/index/index-sync.c
@@ -0,0 +1,560 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "seq-range-array.h"
+#include "ioloop.h"
+#include "array.h"
+#include "index-mailbox-size.h"
+#include "index-sync-private.h"
+#include "mailbox-recent-flags.h"
+
+struct index_storage_list_index_record {
+	uint32_t size;
+	uint32_t mtime;
+};
+
+enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box)
+{
+	enum mail_index_sync_flags sync_flags = 0;
+
+	if ((box->flags & MAILBOX_FLAG_DROP_RECENT) != 0)
+		sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
+	if (box->deleting) {
+		sync_flags |= box->delete_sync_check ?
+			MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX :
+			MAIL_INDEX_SYNC_FLAG_DELETING_INDEX;
+	}
+	return sync_flags;
+}
+
+bool index_mailbox_want_full_sync(struct mailbox *box,
+				  enum mailbox_sync_flags flags)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+
+	if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 &&
+	    ioloop_time < ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL)
+		return FALSE;
+
+	if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 &&
+	    (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) {
+		/* lib-lda is syncing the mailbox after saving a mail.
+		   it only wants to find the new mail for potentially copying
+		   to other mailboxes. that's mainly an optimization, and since
+		   the mail was most likely already added to index we don't
+		   need to do a full sync to find it. the main benefit here is
+		   to avoid a very costly sync with a large Maildir/new/ */
+		return FALSE;
+	}
+
+	if (box->to_notify != NULL)
+		timeout_reset(box->to_notify);
+	ibox->sync_last_check = ioloop_time;
+	return TRUE;
+}
+
+static void index_view_sync_recs_get(struct index_mailbox_sync_context *ctx)
+{
+	struct mail_index_view_sync_rec sync_rec;
+	uint32_t seq1, seq2;
+
+	i_array_init(&ctx->flag_updates, 128);
+	i_array_init(&ctx->hidden_updates, 32);
+	while (mail_index_view_sync_next(ctx->sync_ctx, &sync_rec)) {
+		switch (sync_rec.type) {
+		case MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ:
+		case MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS:
+			if (!mail_index_lookup_seq_range(ctx->ctx.box->view,
+							 sync_rec.uid1,
+							 sync_rec.uid2,
+							 &seq1, &seq2))
+				break;
+
+			if (!sync_rec.hidden &&
+			    sync_rec.type == MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS) {
+				seq_range_array_add_range(&ctx->flag_updates,
+							  seq1, seq2);
+			} else {
+				seq_range_array_add_range(&ctx->hidden_updates,
+							  seq1, seq2);
+			}
+			break;
+		}
+	}
+}
+
+static void
+index_view_sync_cleanup_updates(struct index_mailbox_sync_context *ctx)
+{
+	/* remove expunged messages from flag updates */
+	if (ctx->expunges != NULL) {
+		seq_range_array_remove_seq_range(&ctx->flag_updates,
+						 ctx->expunges);
+		seq_range_array_remove_seq_range(&ctx->hidden_updates,
+						 ctx->expunges);
+	}
+	/* remove flag updates from hidden updates */
+	seq_range_array_remove_seq_range(&ctx->hidden_updates,
+					 &ctx->flag_updates);
+}
+
+struct mailbox_sync_context *
+index_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags,
+			bool failed)
+{
+        struct index_mailbox_sync_context *ctx;
+	struct index_mailbox_sync_pvt_context *pvt_ctx;
+	enum mail_index_view_sync_flags sync_flags = 0;
+
+	ctx = i_new(struct index_mailbox_sync_context, 1);
+	ctx->ctx.box = box;
+	ctx->ctx.flags = flags;
+
+	if (failed) {
+		ctx->failed = TRUE;
+		return &ctx->ctx;
+	}
+
+	if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) != 0)
+		sync_flags |= MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES;
+
+	if ((flags & MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
+		sync_flags |= MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT;
+		ctx->messages_count = 0;
+	} else {
+		ctx->messages_count =
+			mail_index_view_get_messages_count(box->view);
+	}
+
+	if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0) {
+		/* we most likely did a fast sync. refresh the index anyway in
+		   case there were some new changes. */
+		(void)mail_index_refresh(box->index);
+	}
+	ctx->sync_ctx = mail_index_view_sync_begin(box->view, sync_flags);
+	if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) == 0) {
+		mail_index_view_sync_get_expunges(ctx->sync_ctx,
+						  &ctx->expunges);
+		ctx->expunge_pos = array_count(ctx->expunges);
+	}
+	index_view_sync_recs_get(ctx);
+	index_sync_search_results_expunge(ctx);
+
+	/* sync private index if needed. it doesn't use box->view, so it
+	   doesn't matter if it's called at _sync_init() or _sync_deinit().
+	   however we also need to know if any private flags have changed
+	   since last sync, so we need to call it before _sync_next() calls. */
+	if (index_mailbox_sync_pvt_init(box, FALSE, sync_flags, &pvt_ctx) > 0) {
+		(void)index_mailbox_sync_pvt_view(pvt_ctx, &ctx->flag_updates,
+						  &ctx->hidden_updates);
+		index_mailbox_sync_pvt_deinit(&pvt_ctx);
+
+	}
+	index_view_sync_cleanup_updates(ctx);
+	return &ctx->ctx;
+}
+
+static bool
+index_mailbox_sync_next_expunge(struct index_mailbox_sync_context *ctx,
+				struct mailbox_sync_rec *sync_rec_r)
+{
+	const struct seq_range *range;
+
+	if (ctx->expunge_pos == 0)
+		return FALSE;
+
+	/* expunges is a sorted array of sequences. it's easiest for
+	   us to print them from end to beginning. */
+	ctx->expunge_pos--;
+	range = array_idx(ctx->expunges, ctx->expunge_pos);
+	i_assert(range->seq2 <= ctx->messages_count);
+
+	mailbox_recent_flags_expunge_seqs(ctx->ctx.box, range->seq1, range->seq2);
+	ctx->messages_count -= range->seq2 - range->seq1 + 1;
+
+	sync_rec_r->seq1 = range->seq1;
+	sync_rec_r->seq2 = range->seq2;
+	sync_rec_r->type = MAILBOX_SYNC_TYPE_EXPUNGE;
+	return TRUE;
+}
+
+bool index_mailbox_sync_next(struct mailbox_sync_context *_ctx,
+			     struct mailbox_sync_rec *sync_rec_r)
+{
+	struct index_mailbox_sync_context *ctx =
+		(struct index_mailbox_sync_context *)_ctx;
+	const struct seq_range *range;
+	unsigned int count;
+
+	if (ctx->failed)
+		return FALSE;
+
+	range = array_get(&ctx->flag_updates, &count);
+	if (ctx->flag_update_idx < count) {
+		sync_rec_r->type = MAILBOX_SYNC_TYPE_FLAGS;
+		sync_rec_r->seq1 = range[ctx->flag_update_idx].seq1;
+		sync_rec_r->seq2 = range[ctx->flag_update_idx].seq2;
+		ctx->flag_update_idx++;
+		return TRUE;
+	}
+	if ((_ctx->box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0) {
+		/* hidden flag changes' MODSEQs still need to be returned */
+		range = array_get(&ctx->hidden_updates, &count);
+		if (ctx->hidden_update_idx < count) {
+			sync_rec_r->type = MAILBOX_SYNC_TYPE_MODSEQ;
+			sync_rec_r->seq1 = range[ctx->hidden_update_idx].seq1;
+			sync_rec_r->seq2 = range[ctx->hidden_update_idx].seq2;
+			ctx->hidden_update_idx++;
+			return TRUE;
+		}
+	}
+
+	return index_mailbox_sync_next_expunge(ctx, sync_rec_r);
+}
+
+static void
+index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context *ctx)
+{
+	struct mailbox *box = ctx->ctx.box;
+	struct mail_index_view *view = ctx->ctx.box->view;
+	const struct mail_index_header *hdr;
+	uint32_t seq, start_uid, uid;
+
+	if (!array_is_created(&box->recent_flags))
+		return;
+
+	/* expunges array contained expunges for the messages that were already
+	   visible in this view, but append+expunge would be invisible.
+	   recent_flags may however contain the append UID, so we'll have to
+	   remove it separately */
+	hdr = mail_index_get_header(view);
+	if (ctx->messages_count == 0)
+		uid = 0;
+	else if (ctx->messages_count <= hdr->messages_count)
+		mail_index_lookup_uid(view, ctx->messages_count, &uid);
+	else {
+		i_assert(mail_index_view_is_inconsistent(view));
+		return;
+	}
+
+	for (seq = ctx->messages_count + 1; seq <= hdr->messages_count; seq++) {
+		start_uid = uid;
+		mail_index_lookup_uid(view, seq, &uid);
+		if (start_uid + 1 > uid - 1)
+			continue;
+
+		box->recent_flags_count -=
+			seq_range_array_remove_range(&box->recent_flags,
+						     start_uid + 1, uid - 1);
+	}
+
+	if (uid + 1 < hdr->next_uid) {
+		box->recent_flags_count -=
+			seq_range_array_remove_range(&box->recent_flags,
+						     uid + 1,
+						     hdr->next_uid - 1);
+	}
+#ifdef DEBUG
+	if (!mail_index_view_is_inconsistent(view)) {
+		const struct seq_range *range;
+		unsigned int i, count;
+
+		range = array_get(&box->recent_flags, &count);
+		for (i = 0; i < count; i++) {
+			for (uid = range[i].seq1; uid <= range[i].seq2; uid++) {
+				if (uid >= hdr->next_uid)
+					break;
+				(void)mail_index_lookup_seq(view, uid, &seq);
+				i_assert(seq != 0);
+			}
+		}
+	}
+#endif
+}
+
+void index_sync_update_recent_count(struct mailbox *box)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	const struct mail_index_header *hdr;
+	uint32_t seq1, seq2;
+
+	hdr = mail_index_get_header(box->view);
+	if (hdr->first_recent_uid < ibox->recent_flags_prev_first_recent_uid) {
+		mailbox_set_critical(box,
+			"first_recent_uid unexpectedly shrank: %u -> %u",
+			ibox->recent_flags_prev_first_recent_uid,
+			hdr->first_recent_uid);
+		mailbox_recent_flags_reset(box);
+	}
+
+	if (hdr->first_recent_uid > box->recent_flags_prev_uid ||
+	    hdr->next_uid > ibox->recent_flags_last_check_nextuid) {
+		ibox->recent_flags_prev_first_recent_uid = hdr->first_recent_uid;
+		ibox->recent_flags_last_check_nextuid = hdr->next_uid;
+		if (mail_index_lookup_seq_range(box->view,
+						hdr->first_recent_uid,
+						hdr->next_uid,
+						&seq1, &seq2)) {
+			mailbox_recent_flags_set_seqs(box, box->view,
+						      seq1, seq2);
+		}
+	}
+}
+
+static void index_mailbox_sync_free(struct index_mailbox_sync_context *ctx)
+{
+	if (array_is_created(&ctx->flag_updates))
+		array_free(&ctx->flag_updates);
+	if (array_is_created(&ctx->hidden_updates))
+		array_free(&ctx->hidden_updates);
+	if (array_is_created(&ctx->all_flag_update_uids))
+		array_free(&ctx->all_flag_update_uids);
+	i_free(ctx);
+}
+
+int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx,
+			      struct mailbox_sync_status *status_r)
+{
+	struct index_mailbox_sync_context *ctx =
+		(struct index_mailbox_sync_context *)_ctx;
+	struct mailbox_sync_rec sync_rec;
+	bool delayed_expunges = FALSE;
+	int ret = ctx->failed ? -1 : 0;
+
+	/* finish handling expunges, so we don't break when updating
+	   recent flags */
+	while (index_mailbox_sync_next_expunge(ctx, &sync_rec)) ;
+
+	/* convert sequences to uids before syncing view */
+	index_sync_search_results_uidify(ctx);
+
+	if (ctx->sync_ctx != NULL) {
+		if (mail_index_view_sync_commit(&ctx->sync_ctx,
+						&delayed_expunges) < 0) {
+			mailbox_set_index_error(_ctx->box);
+			ret = -1;
+		}
+	}
+	if (ret < 0) {
+		index_mailbox_sync_free(ctx);
+		return -1;
+	}
+	index_mailbox_expunge_unseen_recent(ctx);
+
+	if ((_ctx->box->flags & MAILBOX_FLAG_DROP_RECENT) == 0 &&
+	    _ctx->box->opened) {
+		/* mailbox syncing didn't necessarily update our recent state */
+		index_sync_update_recent_count(_ctx->box);
+	}
+
+	if (status_r != NULL)
+		status_r->sync_delayed_expunges = delayed_expunges;
+
+	/* update search results after private index is updated */
+	index_sync_search_results_update(ctx);
+	/* update vsize header if wanted */
+	index_mailbox_vsize_update_appends(_ctx->box);
+
+	if (ret == 0 && mail_index_view_is_inconsistent(_ctx->box->view)) {
+		/* we probably had MAILBOX_SYNC_FLAG_FIX_INCONSISTENT set,
+		   but the view got broken in the middle. FIXME: We could
+		   attempt to fix it automatically. In any case now the view
+		   isn't usable and we can't return success. */
+		mailbox_set_index_error(_ctx->box);
+		ret = -1;
+	}
+
+	index_mailbox_sync_free(ctx);
+	return ret;
+}
+
+bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1,
+			     const ARRAY_TYPE(keyword_indexes) *k2)
+{
+	const unsigned int *idx1, *idx2;
+	unsigned int i, j, count1, count2;
+
+	if (!array_is_created(k1))
+		return !array_is_created(k2) || array_count(k2) == 0;
+	if (!array_is_created(k2))
+		return array_count(k1) == 0;
+
+	/* The arrays may not be sorted, but they usually are. Optimize for
+	   the assumption that they are */
+	idx1 = array_get(k1, &count1);
+	idx2 = array_get(k2, &count2);
+
+	if (count1 != count2)
+		return FALSE;
+
+	for (i = 0; i < count1; i++) {
+		if (idx1[i] != idx2[i]) {
+			/* not found / unsorted array. check. */
+			for (j = 0; j < count1; j++) {
+				if (idx1[i] == idx2[j])
+					break;
+			}
+			if (j == count1)
+				return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+enum mailbox_sync_type index_sync_type_convert(enum mail_index_sync_type type)
+{
+	enum mailbox_sync_type ret = 0;
+
+	if ((type & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0)
+		ret |= MAILBOX_SYNC_TYPE_EXPUNGE;
+	if ((type & (MAIL_INDEX_SYNC_TYPE_FLAGS |
+		     MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD |
+		     MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE)) != 0)
+		ret |= MAILBOX_SYNC_TYPE_FLAGS;
+	return ret;
+}
+
+static uint32_t
+index_list_get_ext_id(struct mailbox *box, struct mail_index_view *view)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+
+	if (ibox->list_index_sync_ext_id == (uint32_t)-1) {
+		ibox->list_index_sync_ext_id =
+			mail_index_ext_register(mail_index_view_get_index(view),
+				"index sync", 0,
+				sizeof(struct index_storage_list_index_record),
+				sizeof(uint32_t));
+	}
+	return ibox->list_index_sync_ext_id;
+}
+
+enum index_storage_list_change
+index_storage_list_index_has_changed_full(struct mailbox *box,
+					  struct mail_index_view *list_view,
+					  uint32_t seq, const char **reason_r)
+{
+	const struct index_storage_list_index_record *rec;
+	const void *data;
+	const char *dir, *path;
+	struct stat st;
+	uint32_t ext_id;
+	bool expunged;
+	int ret;
+
+	*reason_r = NULL;
+
+	if (mail_index_is_in_memory(mail_index_view_get_index(list_view))) {
+		*reason_r = "List index is in memory";
+		return INDEX_STORAGE_LIST_CHANGE_INMEMORY;
+	}
+
+	ext_id = index_list_get_ext_id(box, list_view);
+	mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
+	rec = data;
+
+	if (rec == NULL) {
+		*reason_r = "Storage record is missing";
+		return INDEX_STORAGE_LIST_CHANGE_NORECORD;
+	} else if (expunged) {
+		*reason_r = "Storage record is expunged";
+		return INDEX_STORAGE_LIST_CHANGE_NORECORD;
+	} else if (rec->size == 0) {
+		*reason_r = "Storage record size=0";
+		return INDEX_STORAGE_LIST_CHANGE_NORECORD;
+	} else if (rec->mtime == 0) {
+		*reason_r = "Storage record mtime=0";
+		return INDEX_STORAGE_LIST_CHANGE_NORECORD;
+	}
+	if (box->storage->set->mailbox_list_index_very_dirty_syncs)
+		return INDEX_STORAGE_LIST_CHANGE_NONE;
+
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir);
+	if (ret < 0)
+		return INDEX_STORAGE_LIST_CHANGE_ERROR;
+	i_assert(ret > 0);
+
+	path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL);
+	if (stat(path, &st) < 0) {
+		if (errno == ENOENT) {
+			*reason_r = t_strdup_printf("%s not found", path);
+			return INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS;
+		}
+		mailbox_set_critical(box, "stat(%s) failed: %m", path);
+		return INDEX_STORAGE_LIST_CHANGE_ERROR;
+	}
+	uint32_t new_size = st.st_size & 0xffffffffU;
+	if (rec->size != new_size) {
+		*reason_r = t_strdup_printf("Storage size changed %u != %u",
+					    rec->size, new_size);
+		return INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED;
+	}
+	uint32_t new_mtime = st.st_mtime & 0xffffffffU;
+	if (rec->mtime != new_mtime) {
+		*reason_r = t_strdup_printf("Storage mtime changed %u != %u",
+					    rec->mtime, new_mtime);
+		return INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED;
+	}
+	return INDEX_STORAGE_LIST_CHANGE_NONE;
+}
+
+int index_storage_list_index_has_changed(struct mailbox *box,
+					 struct mail_index_view *list_view,
+					 uint32_t seq, bool quick ATTR_UNUSED,
+					 const char **reason_r)
+{
+	switch (index_storage_list_index_has_changed_full(box, list_view, seq,
+							  reason_r)) {
+	case INDEX_STORAGE_LIST_CHANGE_ERROR:
+		return -1;
+	case INDEX_STORAGE_LIST_CHANGE_NONE:
+		return 0;
+	default:
+		return 1;
+	}
+}
+
+void index_storage_list_index_update_sync(struct mailbox *box,
+					  struct mail_index_transaction *trans,
+					  uint32_t seq)
+{
+	struct mail_index_view *list_view;
+	const struct index_storage_list_index_record *old_rec;
+	struct index_storage_list_index_record new_rec;
+	const void *data;
+	const char *dir, *path;
+	struct stat st;
+	uint32_t ext_id;
+	bool expunged;
+	int ret;
+
+	list_view = mail_index_transaction_get_view(trans);
+	if (mail_index_is_in_memory(mail_index_view_get_index(list_view)))
+		return;
+
+	/* get the current record */
+	ext_id = index_list_get_ext_id(box, list_view);
+	mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
+	if (expunged)
+		return;
+	old_rec = data;
+
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir);
+	if (ret < 0)
+		return;
+	i_assert(ret > 0);
+
+	path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL);
+	if (stat(path, &st) < 0) {
+		mailbox_set_critical(box, "stat(%s) failed: %m", path);
+		return;
+	}
+
+	i_zero(&new_rec);
+	new_rec.size = st.st_size & 0xffffffffU;
+	new_rec.mtime = st.st_mtime & 0xffffffffU;
+
+	if (old_rec == NULL ||
+	    memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0)
+		mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL);
+}
diff --git a/src/lib-storage/index/index-thread-finish.c b/src/lib-storage/index/index-thread-finish.c
new file mode 100644
index 0000000..42326b6
--- /dev/null
+++ b/src/lib-storage/index/index-thread-finish.c
@@ -0,0 +1,682 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "imap-base-subject.h"
+#include "mail-storage-private.h"
+#include "index-thread-private.h"
+
+
+struct mail_thread_shadow_node {
+	uint32_t first_child_idx, next_sibling_idx;
+};
+
+struct mail_thread_root_node {
+	/* node.idx usually points to indexes from mail hash. However
+	   REFERENCES step (5) may add temporary dummy roots. They use larger
+	   index numbers than exist in the hash. */
+	struct mail_thread_child_node node;
+
+	/* Used temporarily by (5)(B) base subject gathering.
+	   root_idx1 is node's index in roots[] array + 1.
+	   parent_root_idx points to root_idx1, or 0 for root. */
+	unsigned int root_idx1;
+	uint32_t parent_root_idx1;
+
+	/* subject contained a Re: or Fwd: */
+	bool reply_or_forward:1;
+	/* a dummy node */
+	bool dummy:1;
+	/* ignore this node - it's a dummy without children */
+	bool ignore:1;
+};
+
+struct thread_finish_context {
+	unsigned int refcount;
+
+	struct mail *tmp_mail;
+	struct mail_thread_cache *cache;
+
+	ARRAY(struct mail_thread_root_node) roots;
+	ARRAY(struct mail_thread_shadow_node) shadow_nodes;
+	unsigned int next_new_root_idx;
+
+	bool use_sent_date:1;
+	bool return_seqs:1;
+};
+
+struct mail_thread_iterate_context {
+	struct thread_finish_context *ctx;
+
+	ARRAY_TYPE(mail_thread_child_node) children;
+	unsigned int next_idx;
+	bool failed;
+};
+
+struct subject_gather_context {
+	struct thread_finish_context *ctx;
+
+	pool_t subject_pool;
+	HASH_TABLE(char *, struct mail_thread_root_node *) subject_hash;
+};
+
+static void
+add_base_subject(struct subject_gather_context *ctx, const char *subject,
+		 struct mail_thread_root_node *node)
+{
+	struct mail_thread_root_node *hash_node;
+	char *hash_subject;
+	bool is_reply_or_forward;
+
+	subject = imap_get_base_subject_cased(pool_datastack_create(), subject,
+					      &is_reply_or_forward);
+	/* (ii) If the thread subject is empty, skip this message. */
+	if (*subject == '\0')
+		return;
+
+	/* (iii) Look up the message associated with the thread
+	   subject in the subject table. */
+	if (!hash_table_lookup_full(ctx->subject_hash, subject, &hash_subject,
+				    &hash_node)) {
+		/* (iv) If there is no message in the subject table with the
+		   thread subject, add the current message and the thread
+		   subject to the subject table. */
+		hash_subject = p_strdup(ctx->subject_pool, subject);
+		hash_table_insert(ctx->subject_hash, hash_subject, node);
+	} else {
+		/* Otherwise, if the message in the subject table is not a
+		   dummy, AND either of the following criteria are true:
+
+		     The current message is a dummy, OR
+
+                     The message in the subject table is a reply or forward
+		     and the current message is not.
+
+		   then replace the message in the subject table with the
+		   current message. */
+		if (!hash_node->dummy &&
+		    (node->dummy ||
+		     (hash_node->reply_or_forward && !is_reply_or_forward))) {
+			hash_node->parent_root_idx1 = node->root_idx1;
+			hash_table_update(ctx->subject_hash, hash_subject, node);
+		} else {
+			node->parent_root_idx1 = hash_node->root_idx1;
+		}
+	}
+
+	node->reply_or_forward = is_reply_or_forward;
+}
+
+static int mail_thread_child_node_cmp(const struct mail_thread_child_node *c1,
+				      const struct mail_thread_child_node *c2)
+{
+	if (c1->sort_date < c2->sort_date)
+		return -1;
+	if (c1->sort_date > c2->sort_date)
+		return 1;
+
+	if (c1->uid < c2->uid)
+		return -1;
+	if (c1->uid > c2->uid)
+		return 1;
+	return 0;
+}
+
+static int mail_thread_root_node_cmp(const struct mail_thread_root_node *r1,
+				     const struct mail_thread_root_node *r2)
+{
+	return mail_thread_child_node_cmp(&r1->node, &r2->node);
+}
+
+static uint32_t
+thread_lookup_existing(struct thread_finish_context *ctx, uint32_t idx)
+{
+	const struct mail_thread_node *node;
+
+	node = array_idx(&ctx->cache->thread_nodes, idx);
+	i_assert(MAIL_THREAD_NODE_EXISTS(node));
+	i_assert(node->uid != 0);
+	return node->uid;
+}
+
+static void
+thread_child_node_fill(struct thread_finish_context *ctx,
+		       struct mail_thread_child_node *child)
+{
+	int tz;
+
+	child->uid = thread_lookup_existing(ctx, child->idx);
+
+	if (!mail_set_uid(ctx->tmp_mail, child->uid)) {
+		/* the UID should have existed. we would have rebuild
+		   the thread tree otherwise. */
+		i_unreached();
+	}
+
+	/* get sent date if we want to use it and if it's valid */
+	if (!ctx->use_sent_date)
+		child->sort_date = 0;
+	else if (mail_get_date(ctx->tmp_mail, &child->sort_date, &tz) < 0)
+		child->sort_date = 0;
+
+	if (child->sort_date == 0) {
+		/* fallback to received date */
+		(void)mail_get_received_date(ctx->tmp_mail, &child->sort_date);
+	}
+}
+
+static void
+thread_sort_children(struct thread_finish_context *ctx, uint32_t parent_idx,
+		     ARRAY_TYPE(mail_thread_child_node) *sorted_children)
+{
+	const struct mail_thread_shadow_node *shadows;
+	struct mail_thread_child_node child;
+	unsigned int count;
+
+	i_zero(&child);
+	array_clear(sorted_children);
+
+	/* add all child indexes to the array */
+	shadows = array_get(&ctx->shadow_nodes, &count);
+	child.idx = shadows[parent_idx].first_child_idx;
+	i_assert(child.idx != 0);
+	if (shadows[child.idx].next_sibling_idx == 0) {
+		/* only child - don't bother setting sort date */
+		child.uid = thread_lookup_existing(ctx, child.idx);
+
+		array_push_back(sorted_children, &child);
+		return;
+	}
+	while (child.idx != 0) {
+		thread_child_node_fill(ctx, &child);
+
+		array_push_back(sorted_children, &child);
+		child.idx = shadows[child.idx].next_sibling_idx;
+	}
+
+	/* sort the children */
+	array_sort(sorted_children, mail_thread_child_node_cmp);
+}
+
+static void gather_base_subjects(struct thread_finish_context *ctx)
+{
+	struct subject_gather_context gather_ctx;
+	struct mail_thread_root_node *roots;
+	const char *subject;
+	unsigned int i, count;
+	ARRAY_TYPE(mail_thread_child_node) sorted_children;
+	const struct mail_thread_child_node *children;
+	uint32_t idx, uid;
+
+	i_zero(&gather_ctx);
+	gather_ctx.ctx = ctx;
+
+	roots = array_get_modifiable(&ctx->roots, &count);
+	if (count == 0)
+		return;
+	gather_ctx.subject_pool =
+		pool_alloconly_create(MEMPOOL_GROWING"base subjects",
+				      nearest_power(count * 20));
+	hash_table_create(&gather_ctx.subject_hash, gather_ctx.subject_pool,
+			  count * 2, str_hash, strcmp);
+
+	i_array_init(&sorted_children, 64);
+	for (i = 0; i < count; i++) {
+		roots[i].root_idx1 = i + 1;
+		if (!roots[i].dummy)
+			idx = roots[i].node.idx;
+		else if (!roots[i].ignore) {
+			/* find the oldest child */
+			thread_sort_children(ctx, roots[i].node.idx,
+					     &sorted_children);
+			children = array_front(&sorted_children);
+			idx = children[0].idx;
+		} else {
+			/* dummy without children */
+			continue;
+		}
+
+		uid = thread_lookup_existing(ctx, idx);
+		if (!mail_set_uid(ctx->tmp_mail, uid)) {
+			/* the UID should have existed. we would have rebuild
+			   the thread tree otherwise. */
+			i_unreached();
+		}
+		if (mail_get_first_header(ctx->tmp_mail, HDR_SUBJECT,
+					  &subject) > 0) T_BEGIN {
+			add_base_subject(&gather_ctx, subject, &roots[i]);
+		} T_END;
+	}
+	i_assert(roots[count-1].parent_root_idx1 <= count);
+	array_free(&sorted_children);
+	hash_table_destroy(&gather_ctx.subject_hash);
+	pool_unref(&gather_ctx.subject_pool);
+}
+
+static void thread_add_shadow_child(struct thread_finish_context *ctx,
+				    uint32_t parent_idx, uint32_t child_idx)
+{
+	struct mail_thread_shadow_node *parent_shadow, *child_shadow;
+
+	parent_shadow = array_idx_get_space(&ctx->shadow_nodes, parent_idx);
+	child_shadow = array_idx_modifiable(&ctx->shadow_nodes, child_idx);
+
+	child_shadow->next_sibling_idx = parent_shadow->first_child_idx;
+	parent_shadow->first_child_idx = child_idx;
+}
+
+static void mail_thread_root_thread_merge(struct thread_finish_context *ctx,
+					  struct mail_thread_root_node *cur)
+{
+	struct mail_thread_root_node *roots, *root, new_root;
+	struct mail_thread_shadow_node *shadows;
+	unsigned int count;
+	uint32_t idx, next_idx;
+
+	i_assert(cur->parent_root_idx1 != 0);
+
+	/* The highest parent is the same as the current message in the
+	   subject table. */
+	roots = array_get_modifiable(&ctx->roots, &count);
+	root = cur;
+	do {
+		i_assert(root->parent_root_idx1 <= count);
+		root = &roots[root->parent_root_idx1 - 1];
+	} while (root->parent_root_idx1 != 0);
+	i_assert(!root->ignore);
+
+	shadows = array_front_modifiable(&ctx->shadow_nodes);
+	if (cur->dummy) {
+		/* If both messages are dummies, append the current
+                   message's children to the children of the message in
+		   the subject table (the children of both messages
+		   become siblings), and then delete the current message. */
+		i_assert(root->dummy);
+
+		idx = shadows[cur->node.idx].first_child_idx;
+		while (idx != 0) {
+			next_idx = shadows[idx].next_sibling_idx;
+			thread_add_shadow_child(ctx, root->node.idx, idx);
+			idx = next_idx;
+		}
+
+		shadows[cur->node.idx].first_child_idx = 0;
+		cur->ignore = TRUE;
+	} else if (root->dummy || (cur->reply_or_forward &&
+				   !root->reply_or_forward)) {
+		/* a) If the message in the subject table is a dummy and the
+		   current message is not, make the current message a
+		   child of the message in the subject table (a sibling
+		   of its children).
+
+		   b) If the current message is a reply or forward and the
+		   message in the subject table is not, make the current
+		   message a child of the message in the subject table (a
+		   sibling of its children). */
+		thread_add_shadow_child(ctx, root->node.idx, cur->node.idx);
+		cur->ignore = TRUE;
+	} else  {
+		/* Otherwise, create a new dummy message and make both
+		   the current message and the message in the subject
+		   table children of the dummy.  Then replace the message
+                   in the subject table with the dummy message. */
+		i_zero(&new_root);
+		new_root.root_idx1 = array_count(&ctx->roots) + 1;
+		new_root.node.idx = ctx->next_new_root_idx++;
+		new_root.dummy = TRUE;
+
+		thread_add_shadow_child(ctx, new_root.node.idx, root->node.idx);
+		thread_add_shadow_child(ctx, new_root.node.idx, cur->node.idx);
+
+		root->parent_root_idx1 = new_root.root_idx1;
+		root->ignore = TRUE;
+		cur->ignore = TRUE;
+
+		/* append last, since it breaks root and cur pointers */
+		array_push_back(&ctx->roots, &new_root);
+
+		/* make sure all shadow indexes are accessible directly */
+		(void)array_idx_modifiable(&ctx->shadow_nodes,
+					   new_root.node.idx);
+	}
+}
+
+static bool merge_subject_threads(struct thread_finish_context *ctx)
+{
+	struct mail_thread_root_node *roots;
+	unsigned int i, count;
+	bool changed = FALSE;
+
+	roots = array_get_modifiable(&ctx->roots, &count);
+	for (i = 0; i < count; i++) {
+		if (roots[i].parent_root_idx1 != 0 && !roots[i].ignore) {
+			mail_thread_root_thread_merge(ctx, &roots[i]);
+			/* more roots may have been added */
+			roots = array_front_modifiable(&ctx->roots);
+			changed = TRUE;
+		}
+	}
+
+	return changed;
+}
+
+static void sort_root_nodes(struct thread_finish_context *ctx)
+{
+	ARRAY_TYPE(mail_thread_child_node) sorted_children;
+	const struct mail_thread_child_node *children;
+	const struct mail_thread_shadow_node *shadows;
+	struct mail_thread_root_node *roots;
+	unsigned int i, count, child_count;
+
+	i_array_init(&sorted_children, 64);
+	shadows = array_front(&ctx->shadow_nodes);
+	roots = array_get_modifiable(&ctx->roots, &count);
+	for (i = 0; i < count; i++) {
+		if (roots[i].ignore)
+			continue;
+		if (roots[i].dummy) {
+			/* sort by the first child */
+			if (shadows[roots[i].node.idx].first_child_idx == 0) {
+				/* childless dummy node */
+				roots[i].ignore = TRUE;
+				continue;
+			}
+			thread_sort_children(ctx, roots[i].node.idx,
+					     &sorted_children);
+			children = array_get(&sorted_children, &child_count);
+			if (child_count == 1) {
+				/* only one child - deferred step (3).
+				   promote the child to the root. */
+				roots[i].node = children[0];
+				thread_child_node_fill(ctx, &roots[i].node);
+				roots[i].dummy = FALSE;
+			} else {
+				roots[i].node.uid = children[0].uid;
+				roots[i].node.sort_date = children[0].sort_date;
+			}
+		} else {
+			thread_child_node_fill(ctx, &roots[i].node);
+		}
+	}
+	array_free(&sorted_children);
+	array_sort(&ctx->roots, mail_thread_root_node_cmp);
+}
+
+static int mail_thread_root_node_idx_cmp(const void *key, const void *value)
+{
+	const uint32_t *idx = key;
+	const struct mail_thread_root_node *root = value;
+
+	return *idx < root->node.idx ? -1 :
+		*idx > root->node.idx ? 1 : 0;
+}
+
+static void sort_root_nodes_ref2(struct thread_finish_context *ctx,
+				 uint32_t record_count)
+{
+	const struct mail_thread_node *node;
+	struct mail_thread_root_node *roots, *root;
+	struct mail_thread_child_node child;
+	const struct mail_thread_shadow_node *shadows;
+	unsigned int root_count;
+	uint32_t idx, parent_idx;
+
+	roots = array_get_modifiable(&ctx->roots, &root_count);
+
+	/* drop childless dummy nodes */
+	shadows = array_front(&ctx->shadow_nodes);
+	for (idx = 1; idx < root_count; idx++) {
+		if (roots[idx].dummy &&
+		    shadows[roots[idx].node.idx].first_child_idx == 0)
+			roots[idx].ignore = TRUE;
+	}
+
+	for (idx = 1; idx < record_count; idx++) {
+		node = array_idx(&ctx->cache->thread_nodes, idx);
+		if (!MAIL_THREAD_NODE_EXISTS(node))
+			continue;
+
+		child.idx = idx;
+		thread_child_node_fill(ctx, &child);
+
+		parent_idx = idx;
+		while (node->parent_idx != 0) {
+			parent_idx = node->parent_idx;
+			node = array_idx(&ctx->cache->thread_nodes,
+					 node->parent_idx);
+		}
+		root = bsearch(&parent_idx, roots, root_count, sizeof(*roots),
+			       mail_thread_root_node_idx_cmp);
+		i_assert(root != NULL);
+
+		if (root->node.sort_date < child.sort_date)
+			root->node.sort_date = child.sort_date;
+	}
+	array_sort(&ctx->roots, mail_thread_root_node_cmp);
+}
+
+static void mail_thread_create_shadows(struct thread_finish_context *ctx,
+				       uint32_t record_count)
+{
+	const struct mail_thread_node *node, *parent;
+	struct mail_thread_root_node root;
+	struct mail_thread_child_node child;
+	uint32_t idx, parent_idx;
+
+	ctx->use_sent_date = FALSE;
+
+	i_zero(&root);
+	i_zero(&child);
+
+	/* We may see dummy messages without parents or children. We can't
+	   free them since the nodes are in an array, but they may get reused
+	   later so just leave them be. With the current algorithm when this
+	   happens all the struct fields are always zero at that point, so
+	   we don't even have to try to zero them. */
+	for (idx = 1; idx < record_count; idx++) {
+		node = array_idx(&ctx->cache->thread_nodes, idx);
+
+		if (node->parent_idx == 0) {
+			/* root node - add to roots list */
+			root.node.idx = idx;
+			if (!MAIL_THREAD_NODE_EXISTS(node)) {
+				root.dummy = TRUE;
+				root.node.uid = 0;
+			} else {
+				root.dummy = FALSE;
+				root.node.uid = node->uid;
+			}
+			array_push_back(&ctx->roots, &root);
+			continue;
+		}
+		i_assert(node->parent_idx < record_count);
+
+		if (!MAIL_THREAD_NODE_EXISTS(node)) {
+			/* dummy node */
+			continue;
+		}
+
+		/* Find the node's first non-dummy parent and add the
+		   node as its child. If there are no non-dummy
+		   parents, add it as the highest dummy's child. */
+		parent_idx = node->parent_idx;
+		parent = array_idx(&ctx->cache->thread_nodes, parent_idx);
+		while (!MAIL_THREAD_NODE_EXISTS(parent) &&
+		       parent->parent_idx != 0) {
+			parent_idx = parent->parent_idx;
+			parent = array_idx(&ctx->cache->thread_nodes,
+					   parent_idx);
+		}
+		thread_add_shadow_child(ctx, parent_idx, idx);
+	}
+}
+
+static void mail_thread_finish(struct thread_finish_context *ctx,
+			       enum mail_thread_type thread_type)
+{
+	unsigned int record_count = array_count(&ctx->cache->thread_nodes);
+
+	ctx->next_new_root_idx = record_count + 1;
+
+	/* (2) save root nodes and (3) remove dummy messages */
+	i_array_init(&ctx->roots, I_MIN(128, record_count));
+	i_array_init(&ctx->shadow_nodes, record_count);
+	/* make sure all shadow indexes are accessible directly. */
+	(void)array_idx_get_space(&ctx->shadow_nodes, record_count);
+
+	mail_thread_create_shadows(ctx, record_count);
+
+	/* (4) */
+	ctx->use_sent_date = TRUE;
+	switch (thread_type) {
+	case MAIL_THREAD_REFERENCES:
+		sort_root_nodes(ctx);
+		/* (5) Gather together messages under the root that have
+		   the same base subject text. */
+		gather_base_subjects(ctx);
+		/* (5.C) Merge threads with the same thread subject. */
+		if (merge_subject_threads(ctx)) {
+			/* root ordering may have changed, sort them again. */
+			sort_root_nodes(ctx);
+		}
+		break;
+	case MAIL_THREAD_REFS:
+		sort_root_nodes_ref2(ctx, record_count);
+		break;
+	default:
+		i_unreached();
+	}
+}
+
+static void
+nodes_change_uids_to_seqs(struct mail_thread_iterate_context *iter, bool root)
+{
+	struct mail_thread_child_node *children;
+	struct mailbox *box = iter->ctx->tmp_mail->box;
+	unsigned int i, count;
+	uint32_t uid, seq;
+
+	children = array_get_modifiable(&iter->children, &count);
+	for (i = 0; i < count; i++) {
+		uid = children[i].uid;
+		if (uid == 0) {
+			/* dummy root */
+			if (root)
+				continue;
+			i_unreached();
+		} else {
+			mailbox_get_seq_range(box, uid, uid, &seq, &seq);
+			i_assert(seq != 0);
+		}
+		children[i].uid = seq;
+	}
+}
+
+static void
+mail_thread_iterate_fill_root(struct mail_thread_iterate_context *iter)
+{
+	struct mail_thread_root_node *roots;
+	unsigned int i, count;
+
+	roots = array_get_modifiable(&iter->ctx->roots, &count);
+	i_array_init(&iter->children, count);
+	for (i = 0; i < count; i++) {
+		if (!roots[i].ignore) {
+			if (roots[i].dummy)
+				roots[i].node.uid = 0;
+			array_push_back(&iter->children, &roots[i].node);
+		}
+	}
+}
+
+static struct mail_thread_iterate_context *
+mail_thread_iterate_children(struct mail_thread_iterate_context *parent_iter,
+			     uint32_t parent_idx)
+{
+	struct mail_thread_iterate_context *child_iter;
+
+	child_iter = i_new(struct mail_thread_iterate_context, 1);
+	child_iter->ctx = parent_iter->ctx;
+	child_iter->ctx->refcount++;
+
+	i_array_init(&child_iter->children, 8);
+	struct event_reason *reason = event_reason_begin("mailbox:thread");
+	thread_sort_children(child_iter->ctx, parent_idx,
+			     &child_iter->children);
+	if (child_iter->ctx->return_seqs)
+		nodes_change_uids_to_seqs(child_iter, FALSE);
+	event_reason_end(&reason);
+	return child_iter;
+}
+
+struct mail_thread_iterate_context *
+mail_thread_iterate_init_full(struct mail_thread_cache *cache,
+			      struct mail *tmp_mail,
+			      enum mail_thread_type thread_type,
+			      bool return_seqs)
+{
+	struct mail_thread_iterate_context *iter;
+	struct thread_finish_context *ctx;
+
+	iter = i_new(struct mail_thread_iterate_context, 1);
+	ctx = iter->ctx = i_new(struct thread_finish_context, 1);
+	ctx->refcount = 1;
+	ctx->cache = cache;
+	ctx->tmp_mail = tmp_mail;
+	ctx->return_seqs = return_seqs;
+
+	struct event_reason *reason = event_reason_begin("mailbox:thread");
+	mail_thread_finish(ctx, thread_type);
+
+	mail_thread_iterate_fill_root(iter);
+	if (return_seqs)
+		nodes_change_uids_to_seqs(iter, TRUE);
+	event_reason_end(&reason);
+	return iter;
+}
+
+const struct mail_thread_child_node *
+mail_thread_iterate_next(struct mail_thread_iterate_context *iter,
+			 struct mail_thread_iterate_context **child_iter_r)
+{
+	const struct mail_thread_child_node *children, *child;
+	const struct mail_thread_shadow_node *shadow;
+	unsigned int count;
+
+	children = array_get(&iter->children, &count);
+	if (iter->next_idx >= count)
+		return NULL;
+
+	child = &children[iter->next_idx++];
+	shadow = array_idx(&iter->ctx->shadow_nodes, child->idx);
+	*child_iter_r = shadow->first_child_idx == 0 ? NULL :
+		mail_thread_iterate_children(iter, child->idx);
+	if (child->uid == 0 && *child_iter_r == NULL) {
+		/* this is a dummy node without children,
+		   there's no point in returning it */
+		return mail_thread_iterate_next(iter, child_iter_r);
+	}
+	return child;
+}
+
+unsigned int mail_thread_iterate_count(struct mail_thread_iterate_context *iter)
+{
+	return array_count(&iter->children);
+}
+
+int mail_thread_iterate_deinit(struct mail_thread_iterate_context **_iter)
+{
+	struct mail_thread_iterate_context *iter = *_iter;
+
+	*_iter = NULL;
+
+	if (--iter->ctx->refcount == 0) {
+		array_free(&iter->ctx->roots);
+		array_free(&iter->ctx->shadow_nodes);
+		i_free(iter->ctx);
+	}
+	array_free(&iter->children);
+	i_free(iter);
+	return 0;
+}
diff --git a/src/lib-storage/index/index-thread-links.c b/src/lib-storage/index/index-thread-links.c
new file mode 100644
index 0000000..9affb83
--- /dev/null
+++ b/src/lib-storage/index/index-thread-links.c
@@ -0,0 +1,242 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "message-id.h"
+#include "mail-storage.h"
+#include "index-thread-private.h"
+
+static uint32_t thread_msg_add(struct mail_thread_cache *cache,
+			       uint32_t uid, uint32_t msgid_idx)
+{
+	struct mail_thread_node *node;
+
+	i_assert(msgid_idx != 0);
+	i_assert(msgid_idx < cache->first_invalid_msgid_str_idx);
+
+	node = array_idx_get_space(&cache->thread_nodes, msgid_idx);
+	if (node->uid == 0)
+		node->uid = uid;
+	else {
+		/* duplicate message-id, keep the original.
+		   if the original ever gets expunged, rebuild. */
+		node->expunge_rebuilds = TRUE;
+
+		msgid_idx = cache->next_invalid_msgid_str_idx++;
+		node = array_idx_get_space(&cache->thread_nodes, msgid_idx);
+		node->uid = uid;
+	}
+	return msgid_idx;
+}
+
+static bool thread_node_has_ancestor(struct mail_thread_cache *cache,
+				     const struct mail_thread_node *node,
+				     const struct mail_thread_node *ancestor)
+{
+	while (node != ancestor) {
+		if (node->parent_idx == 0)
+			return FALSE;
+
+		node = array_idx(&cache->thread_nodes, node->parent_idx);
+	}
+	return TRUE;
+}
+
+static void thread_link_reference(struct mail_thread_cache *cache,
+				  uint32_t parent_idx, uint32_t child_idx)
+{
+	struct mail_thread_node *node, *parent, *child;
+	uint32_t idx;
+
+	i_assert(parent_idx < cache->first_invalid_msgid_str_idx);
+
+	/* either child_idx or parent_idx may cause thread_nodes array to
+	   grow. in such situation the other pointer may become invalid if
+	   we don't get the pointers in correct order. */
+	if (child_idx < parent_idx) {
+		parent = array_idx_get_space(&cache->thread_nodes, parent_idx);
+		child = array_idx_modifiable(&cache->thread_nodes, child_idx);
+	} else {
+		child = array_idx_get_space(&cache->thread_nodes, child_idx);
+		parent = array_idx_modifiable(&cache->thread_nodes, parent_idx);
+	}
+
+	child->parent_link_refcount++;
+	if (thread_node_has_ancestor(cache, parent, child)) {
+		if (parent == child) {
+			/* loops to itself - ignore */
+			return;
+		}
+
+		/* child is an ancestor of parent. Adding child -> parent_node
+		   would introduce a loop. If any messages referencing the path
+		   between parent_node's parent and child_node get expunged, we
+		   have to rebuild the tree because the loop might break.
+		   For example:
+
+		     #1: a -> b       (a.ref=1, b.ref=1)
+		     #2: b -> a       (a.ref=2, b.ref=2)
+		     #3: c -> a -> b  (a.ref=3, b.ref=3, c.ref=1)
+
+		   Expunging #3 wouldn't break the loop, but expunging #1
+		   would. */
+		node = parent;
+		do {
+			idx = node->parent_idx;
+			i_assert(idx != 0);
+			node = array_idx_modifiable(&cache->thread_nodes, idx);
+			node->child_unref_rebuilds = TRUE;
+		} while (node != child);
+		return;
+	} else if (child->parent_idx == parent_idx) {
+		/* The same link already exists */
+		return;
+	}
+
+	/* Set parent_node as child_node's parent */
+	if (child->parent_idx == 0) {
+		child->parent_idx = parent_idx;
+	} else {
+		/* Conflicting parent already exists, keep the original */
+		if (MAIL_THREAD_NODE_EXISTS(child)) {
+			/* If this message gets expunged,
+			   the parent is changed. */
+			child->expunge_rebuilds = TRUE;
+		} else {
+			/* Message doesn't exist, so it was one of the node's
+			   children that created the original reference. If
+			   that reference gets dropped, the parent is changed.
+			   We could catch this in one of several ways:
+
+			    a) Link to parent node gets unreferenced
+			    b) Link to this node gets unreferenced
+			    c) Any of the child nodes gets expunged
+
+			   b) is probably the least likely to happen,
+			   so use it */
+			child->child_unref_rebuilds = TRUE;
+		}
+	}
+}
+
+static uint32_t
+thread_link_references(struct mail_thread_cache *cache, uint32_t uid,
+		       const struct mail_index_strmap_rec *msgid_map,
+		       unsigned int *msgid_map_idx)
+{
+	uint32_t parent_idx;
+
+	if (msgid_map->uid != uid)
+		return 0;
+
+	parent_idx = msgid_map->str_idx;
+	msgid_map++;
+	*msgid_map_idx += 1;
+
+	for (; msgid_map->uid == uid; msgid_map++) {
+		thread_link_reference(cache, parent_idx, msgid_map->str_idx);
+		parent_idx = msgid_map->str_idx;
+		*msgid_map_idx += 1;
+	}
+	i_assert(parent_idx < cache->first_invalid_msgid_str_idx);
+	return parent_idx;
+}
+
+void mail_thread_add(struct mail_thread_cache *cache,
+		     const struct mail_index_strmap_rec *msgid_map,
+		     unsigned int *msgid_map_idx)
+{
+	struct mail_thread_node *node;
+	uint32_t idx, parent_idx;
+
+	i_assert(msgid_map->ref_index == MAIL_THREAD_NODE_REF_MSGID);
+	i_assert(cache->last_uid <= msgid_map->uid);
+
+	cache->last_uid = msgid_map->uid;
+
+	idx = thread_msg_add(cache, msgid_map->uid, msgid_map->str_idx);
+	parent_idx = thread_link_references(cache, msgid_map->uid,
+					    msgid_map + 1, msgid_map_idx);
+
+	node = array_idx_modifiable(&cache->thread_nodes, idx);
+	if (node->parent_idx != parent_idx && node->parent_idx != 0) {
+		/* conflicting parent, remove it. */
+		node->parent_idx = 0;
+		/* If this message gets expunged, we have to revert back to
+		   the original parent. */
+		node->expunge_rebuilds = TRUE;
+	}
+	if (parent_idx != 0)
+		thread_link_reference(cache, parent_idx, idx);
+	*msgid_map_idx += 1;
+}
+
+static bool
+mail_thread_unref_link(struct mail_thread_cache *cache,
+		       uint32_t parent_idx, uint32_t child_idx)
+{
+	struct mail_thread_node *parent, *child;
+
+	parent = array_idx_modifiable(&cache->thread_nodes, parent_idx);
+	if (parent->child_unref_rebuilds)
+		return FALSE;
+
+	child = array_idx_modifiable(&cache->thread_nodes, child_idx);
+	i_assert(child->parent_link_refcount > 0);
+
+	child->parent_link_refcount--;
+	if (child->parent_link_refcount == 0) {
+		/* we don't have a root anymore */
+		child->parent_idx = 0;
+	}
+	return TRUE;
+}
+
+bool mail_thread_remove(struct mail_thread_cache *cache,
+			const struct mail_index_strmap_rec *msgid_map,
+			unsigned int *msgid_map_idx)
+{
+	struct mail_thread_node *node;
+	uint32_t idx, parent_idx;
+	unsigned int count = 1;
+
+	idx = msgid_map->str_idx;
+	i_assert(idx != 0);
+
+	if (msgid_map->uid > cache->last_uid) {
+		/* this message was never added to the cache, skip */
+		while (msgid_map[count].uid == msgid_map->uid)
+			count++;
+		*msgid_map_idx += count;
+		return TRUE;
+	}
+
+	node = array_idx_modifiable(&cache->thread_nodes, idx);
+	if (node->expunge_rebuilds) {
+		/* this catches the duplicate message-id case */
+		return FALSE;
+	}
+	i_assert(node->uid == msgid_map->uid);
+
+	/* update link refcounts */
+	if (msgid_map[count].uid == node->uid) {
+		parent_idx = msgid_map[count].str_idx;
+		count++;
+		while (msgid_map[count].uid == node->uid) {
+			if (!mail_thread_unref_link(cache, parent_idx,
+						    msgid_map[count].str_idx))
+				return FALSE;
+			parent_idx = msgid_map[count].str_idx;
+			count++;
+		}
+		if (!mail_thread_unref_link(cache, parent_idx, idx))
+			return FALSE;
+	}
+	/* mark this message as expunged */
+	node->uid = 0;
+
+	/* we don't know (and don't want to waste time figuring out) if other
+	   messages point to this removed message, so don't delete the node */
+	*msgid_map_idx += count;
+	return TRUE;
+}
diff --git a/src/lib-storage/index/index-thread-private.h b/src/lib-storage/index/index-thread-private.h
new file mode 100644
index 0000000..ed2837b
--- /dev/null
+++ b/src/lib-storage/index/index-thread-private.h
@@ -0,0 +1,82 @@
+#ifndef INDEX_THREAD_PRIVATE_H
+#define INDEX_THREAD_PRIVATE_H
+
+#include "crc32.h"
+#include "mail-thread.h"
+#include "mail-index-strmap.h"
+
+#define MAIL_THREAD_INDEX_SUFFIX ".thread"
+
+/* After initially building the index, assign first_invalid_msgid_idx to
+   the next unused index + SKIP_COUNT. When more messages are added and
+   the next valid msgid conflicts with the first invalid msgid, the invalid
+   msgids will be moved forward again this many indexes. */
+#define THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT \
+	(4096 / sizeof(struct mail_thread_node))
+
+#define HDR_MESSAGE_ID "message-id"
+#define HDR_IN_REPLY_TO "in-reply-to"
+#define HDR_REFERENCES "references"
+#define HDR_SUBJECT "subject"
+
+#define MAIL_THREAD_NODE_REF_MSGID 0
+#define MAIL_THREAD_NODE_REF_INREPLYTO 1
+#define MAIL_THREAD_NODE_REF_REFERENCES1 2
+
+struct mail_thread_node {
+	/* UID of the message, or 0 for dummy nodes */
+	uint32_t uid;
+	/* Index for this node's parent node, 0 = this is root */
+	uint32_t parent_idx;
+	/* Number of messages containing "this message" -> "parent message"
+	   link, i.e. "number of links to parent node". However since parents
+	   can change, not all of these references might be from our current
+	   child nodes. When this refcount reaches 0, it means we must detach
+	   from our parent. */
+	unsigned int parent_link_refcount:30;
+	/* If uid is expunged, rebuild the thread tree. */
+	bool expunge_rebuilds:1;
+	/* If a link between this node and its child gets unreferenced,
+	   rebuild the thread tree. */
+	bool child_unref_rebuilds:1;
+};
+ARRAY_DEFINE_TYPE(mail_thread_node, struct mail_thread_node);
+#define MAIL_THREAD_NODE_EXISTS(node) \
+	((node)->uid != 0)
+
+struct mail_thread_cache {
+	uint32_t last_uid;
+	/* indexes used for invalid Message-IDs. that means no other messages
+	   point to them and they can safely be moved around whenever
+	   necessary. */
+	uint32_t first_invalid_msgid_str_idx;
+	uint32_t next_invalid_msgid_str_idx;
+
+	struct mail_search_result *search_result;
+
+	/* indexed by mail_index_strmap_rec.str_idx */
+	ARRAY_TYPE(mail_thread_node) thread_nodes;
+};
+
+static inline uint32_t crc32_str_nonzero(const char *str)
+{
+	uint32_t value = crc32_str(str);
+	return value == 0 ? 1 : value;
+}
+
+void mail_thread_add(struct mail_thread_cache *cache,
+		     const struct mail_index_strmap_rec *msgid_map,
+		     unsigned int *msgid_map_idx);
+bool mail_thread_remove(struct mail_thread_cache *cache,
+			const struct mail_index_strmap_rec *msgid_map,
+			unsigned int *msgid_map_idx);
+
+struct mail_thread_iterate_context *
+mail_thread_iterate_init_full(struct mail_thread_cache *cache,
+			      struct mail *tmp_mail,
+			      enum mail_thread_type thread_type,
+			      bool return_seqs);
+
+void index_thread_mailbox_opened(struct mailbox *box);
+
+#endif
diff --git a/src/lib-storage/index/index-thread.c b/src/lib-storage/index/index-thread.c
new file mode 100644
index 0000000..2b1ba04
--- /dev/null
+++ b/src/lib-storage/index/index-thread.c
@@ -0,0 +1,670 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+/* doc/thread-refs.txt describes the incremental algorithm we use here. */
+
+#include "lib.h"
+#include "array.h"
+#include "bsearch-insert-pos.h"
+#include "hash2.h"
+#include "message-id.h"
+#include "mail-search.h"
+#include "mail-search-build.h"
+#include "mailbox-search-result-private.h"
+#include "index-storage.h"
+#include "index-thread-private.h"
+
+
+#define MAIL_THREAD_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, mail_thread_storage_module)
+#define MAIL_THREAD_CONTEXT_REQUIRE(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, mail_thread_storage_module)
+
+struct mail_thread_context {
+	struct mailbox *box;
+	struct mailbox_transaction_context *t;
+	struct mail_index_strmap_view_sync *strmap_sync;
+
+	struct mail *tmp_mail;
+	struct mail_search_args *search_args;
+	ARRAY_TYPE(seq_range) added_uids;
+
+	bool failed:1;
+	bool corrupted:1;
+};
+
+struct mail_thread_mailbox {
+	union mailbox_module_context module_ctx;
+
+	unsigned int next_msgid_idx;
+	struct mail_thread_cache *cache;
+
+	struct mail_index_strmap *strmap;
+	struct mail_index_strmap_view *strmap_view;
+	/* sorted by UID, ref_index */
+	const ARRAY_TYPE(mail_index_strmap_rec) *msgid_map;
+	const struct hash2_table *msgid_hash;
+
+	/* set only temporarily while needed */
+	struct mail_thread_context *ctx;
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(mail_thread_storage_module,
+				  &mail_storage_module_register);
+
+static void mail_thread_clear(struct mail_thread_context *ctx);
+
+static int
+mail_strmap_rec_get_msgid(struct mail_thread_context *ctx,
+			  const struct mail_index_strmap_rec *rec,
+			  const char **msgid_r)
+{
+	struct mail *mail = ctx->tmp_mail;
+	const char *msgids = NULL, *msgid;
+	unsigned int n = 0;
+	int ret;
+
+	if (!mail_set_uid(mail, rec->uid))
+		return 0;
+
+	switch (rec->ref_index) {
+	case MAIL_THREAD_NODE_REF_MSGID:
+		/* Message-ID: header */
+		ret = mail_get_first_header(mail, HDR_MESSAGE_ID, &msgids);
+		break;
+	case MAIL_THREAD_NODE_REF_INREPLYTO:
+		/* In-Reply-To: header */
+		ret = mail_get_first_header(mail, HDR_IN_REPLY_TO, &msgids);
+		break;
+	default:
+		/* References: header */
+		ret = mail_get_first_header(mail, HDR_REFERENCES, &msgids);
+		n = rec->ref_index - MAIL_THREAD_NODE_REF_REFERENCES1;
+		break;
+	}
+
+	if (ret < 0) {
+		if (mail->expunged) {
+			/* treat it as if it didn't exist. trying to add it
+			   again will result in failure. */
+			return 0;
+		}
+		return -1;
+	}
+
+	/* get the nth message-id */
+	msgid = message_id_get_next(&msgids);
+	if (msgid != NULL) {
+		for (; n > 0; n--)
+			msgid = message_id_get_next(&msgids);
+	}
+
+	if (msgid == NULL) {
+		/* shouldn't have happened, probably corrupted */
+		mail_set_critical(mail,
+			"Corrupted thread index: lost Message ID %u",
+			rec->ref_index);
+		ctx->failed = TRUE;
+		ctx->corrupted = TRUE;
+		return -1;
+	}
+	*msgid_r = msgid;
+	return 1;
+}
+
+static bool
+mail_thread_hash_key_cmp(const char *key,
+			 const struct mail_index_strmap_rec *rec,
+			 void *context)
+{
+	struct mail_thread_mailbox *tbox = context;
+	struct mail_thread_context *ctx = tbox->ctx;
+	const char *msgid;
+	bool cmp_ret;
+	int ret;
+
+	/* either a match or a collision, need to look closer */
+	T_BEGIN {
+		ret = mail_strmap_rec_get_msgid(ctx, rec, &msgid);
+		if (ret <= 0) {
+			if (ret < 0)
+				ctx->failed = TRUE;
+			cmp_ret = FALSE;
+		} else {
+			cmp_ret = strcmp(msgid, key) == 0;
+		}
+	} T_END;
+	return cmp_ret;
+}
+
+static int
+mail_thread_hash_rec_cmp(const struct mail_index_strmap_rec *rec1,
+			 const struct mail_index_strmap_rec *rec2,
+			 void *context)
+{
+	struct mail_thread_mailbox *tbox = context;
+	struct mail_thread_context *ctx = tbox->ctx;
+	const char *msgid1, *msgid2;
+	int ret;
+
+	T_BEGIN {
+		ret = mail_strmap_rec_get_msgid(ctx, rec1, &msgid1);
+		if (ret > 0) {
+			msgid1 = t_strdup(msgid1);
+			ret = mail_strmap_rec_get_msgid(ctx, rec2, &msgid2);
+		}
+		ret = ret <= 0 ? -1 :
+			strcmp(msgid1, msgid2) == 0;
+	} T_END;
+	return ret;
+}
+
+static void mail_thread_strmap_remap(const uint32_t *idx_map,
+				     unsigned int old_count,
+				     unsigned int new_count, void *context)
+{
+	struct mail_thread_mailbox *tbox = context;
+	struct mail_thread_cache *cache = tbox->cache;
+	ARRAY_TYPE(mail_thread_node) new_nodes;
+	const struct mail_thread_node *old_nodes;
+	struct mail_thread_node *node;
+	unsigned int i, nodes_count, max, new_first_invalid, invalid_count;
+
+	if (cache->search_result == NULL)
+		return;
+
+	if (new_count == 0) {
+		/* strmap was reset, we'll need to rebuild thread */
+		mailbox_search_result_free(&cache->search_result);
+		return;
+	}
+
+	invalid_count = cache->next_invalid_msgid_str_idx -
+		cache->first_invalid_msgid_str_idx;
+
+	old_nodes = array_get(&cache->thread_nodes, &nodes_count);
+	i_array_init(&new_nodes, new_count + invalid_count + 32);
+
+	/* optimization: allocate all nodes initially */
+	(void)array_idx_modifiable(&new_nodes, new_count-1);
+
+	/* renumber existing valid nodes. all existing records in old_nodes
+	   should also exist in idx_map since we've removed expunged messages
+	   from the cache before committing the sync. */
+	max = I_MIN(I_MIN(old_count, nodes_count),
+		    cache->first_invalid_msgid_str_idx);
+	for (i = 0; i < max; i++) {
+		if (idx_map[i] == 0) {
+			/* expunged record. */
+			i_assert(old_nodes[i].uid == 0);
+		} else {
+			node = array_idx_modifiable(&new_nodes, idx_map[i]);
+			*node = old_nodes[i];
+			if (node->parent_idx != 0) {
+				node->parent_idx = idx_map[node->parent_idx];
+				i_assert(node->parent_idx != 0);
+			}
+		}
+	}
+
+	/* copy invalid nodes, if any. no other messages point to them,
+	   so this is safe. we still need to update their parent_idx
+	   pointers though. */
+	new_first_invalid = new_count + 1 +
+		THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT;
+	for (i = 0; i < invalid_count; i++) {
+		node = array_idx_modifiable(&new_nodes, new_first_invalid + i);
+		*node = old_nodes[cache->first_invalid_msgid_str_idx + i];
+		if (node->parent_idx != 0) {
+			node->parent_idx = idx_map[node->parent_idx];
+			i_assert(node->parent_idx != 0);
+		}
+	}
+	cache->first_invalid_msgid_str_idx = new_first_invalid;
+	cache->next_invalid_msgid_str_idx = new_first_invalid + invalid_count;
+
+	/* replace the old nodes with the renumbered ones */
+	array_free(&cache->thread_nodes);
+	cache->thread_nodes = new_nodes;
+}
+
+static int thread_get_mail_header(struct mail *mail, const char *name,
+				  const char **value_r)
+{
+	if (mail_get_first_header(mail, name, value_r) < 0) {
+		if (!mail->expunged)
+			return -1;
+
+		/* Message is expunged. Instead of failing the entire THREAD
+		   command, just treat the header as nonexistent. */
+		*value_r = NULL;
+	}
+	return 0;
+}
+
+static int
+mail_thread_map_add_mail(struct mail_thread_context *ctx, struct mail *mail)
+{
+	const char *message_id, *in_reply_to, *references, *msgid;
+	uint32_t ref_index;
+
+	if (thread_get_mail_header(mail, HDR_MESSAGE_ID, &message_id) < 0 ||
+	    thread_get_mail_header(mail, HDR_REFERENCES, &references) < 0)
+		return -1;
+
+	/* add Message-ID: */
+	msgid = message_id_get_next(&message_id);
+	if (msgid != NULL) {
+		mail_index_strmap_view_sync_add(ctx->strmap_sync, mail->uid,
+						MAIL_THREAD_NODE_REF_MSGID,
+						msgid);
+	} else {
+		mail_index_strmap_view_sync_add_unique(ctx->strmap_sync,
+					mail->uid, MAIL_THREAD_NODE_REF_MSGID);
+	}
+
+	/* add References: if there are any valid ones */
+	msgid = message_id_get_next(&references);
+	if (msgid != NULL) {
+		ref_index = MAIL_THREAD_NODE_REF_REFERENCES1;
+		do {
+			mail_index_strmap_view_sync_add(ctx->strmap_sync,
+							mail->uid,
+							ref_index, msgid);
+			ref_index++;
+			msgid = message_id_get_next(&references);
+		} while (msgid != NULL);
+	} else {
+		/* no References:, use In-Reply-To: */
+		if (thread_get_mail_header(mail, HDR_IN_REPLY_TO,
+					   &in_reply_to) < 0)
+			return -1;
+
+		msgid = message_id_get_next(&in_reply_to);
+		if (msgid != NULL) {
+			mail_index_strmap_view_sync_add(ctx->strmap_sync,
+				mail->uid, MAIL_THREAD_NODE_REF_INREPLYTO,
+				msgid);
+		}
+	}
+	if (ctx->failed) {
+		/* message-id lookup failed in hash compare */
+		return -1;
+	}
+	return 0;
+}
+
+static int mail_thread_index_map_build(struct mail_thread_context *ctx)
+{
+	static const char *wanted_headers[] = {
+		HDR_MESSAGE_ID, HDR_IN_REPLY_TO, HDR_REFERENCES, HDR_SUBJECT,
+		NULL
+	};
+	struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(ctx->box);
+	struct mailbox_header_lookup_ctx *headers_ctx;
+	struct mail_search_args *search_args;
+	struct mail_search_context *search_ctx;
+	struct mail *mail;
+	uint32_t last_uid, seq1, seq2;
+	int ret = 0;
+
+	if (tbox->strmap_view == NULL) {
+		/* first time we're threading this mailbox */
+		tbox->strmap_view =
+			mail_index_strmap_view_open(tbox->strmap,
+						    ctx->box->view,
+						    mail_thread_hash_key_cmp,
+						    mail_thread_hash_rec_cmp,
+						    mail_thread_strmap_remap,
+						    tbox, &tbox->msgid_map,
+						    &tbox->msgid_hash);
+	}
+
+	headers_ctx = mailbox_header_lookup_init(ctx->box, wanted_headers);
+	ctx->tmp_mail = mail_alloc(ctx->t, MAIL_FETCH_DATE |
+				   MAIL_FETCH_RECEIVED_DATE, headers_ctx);
+
+	/* add all missing UIDs */
+	ctx->strmap_sync = mail_index_strmap_view_sync_init(tbox->strmap_view,
+							    &last_uid);
+	mailbox_get_seq_range(ctx->box, last_uid + 1, (uint32_t)-1,
+			      &seq1, &seq2);
+	if (seq1 == 0) {
+		/* nothing is missing */
+		mail_index_strmap_view_sync_commit(&ctx->strmap_sync);
+		mailbox_header_lookup_unref(&headers_ctx);
+		return 0;
+	}
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_seqset(search_args, seq1, seq2);
+	search_ctx = mailbox_search_init(ctx->t, search_args, NULL,
+					 MAIL_FETCH_DATE |
+					 MAIL_FETCH_RECEIVED_DATE,
+					 headers_ctx);
+	mailbox_header_lookup_unref(&headers_ctx);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(search_ctx, &mail)) {
+		if (mail_thread_map_add_mail(ctx, mail) < 0) {
+			ret = -1;
+			break;
+		}
+	}
+	if (mailbox_search_deinit(&search_ctx) < 0)
+		ret = -1;
+
+	if (ret < 0)
+		mail_index_strmap_view_sync_rollback(&ctx->strmap_sync);
+	else
+		mail_index_strmap_view_sync_commit(&ctx->strmap_sync);
+	return ret;
+}
+
+static int msgid_map_cmp(const uint32_t *uid,
+			 const struct mail_index_strmap_rec *rec)
+{
+	return *uid < rec->uid ? -1 :
+		(*uid > rec->uid ? 1 : 0);
+}
+
+static bool mail_thread_cache_update_removes(struct mail_thread_mailbox *tbox,
+					     ARRAY_TYPE(seq_range) *added_uids)
+{
+	struct mail_thread_cache *cache = tbox->cache;
+	ARRAY_TYPE(seq_range) removed_uids;
+	const struct seq_range *uids;
+	const struct mail_index_strmap_rec *msgid_map;
+	unsigned int i, j, idx, map_count, uid_count;
+	uint32_t uid;
+
+	t_array_init(&removed_uids, 64);
+	mailbox_search_result_sync(cache->search_result,
+				   &removed_uids, added_uids);
+
+	/* first check that we're not inserting any messages in the middle */
+	uids = array_get(added_uids, &uid_count);
+	if (uid_count > 0 && uids[0].seq1 <= cache->last_uid)
+		return FALSE;
+
+	/* next remove messages so we'll see early if we have to rebuild.
+	   we expect to find all removed UIDs from msgid_map that are <= max
+	   UID in msgid_map */
+	msgid_map = array_get(tbox->msgid_map, &map_count);
+	uids = array_get(&removed_uids, &uid_count);
+	for (i = j = 0; i < uid_count; i++) {
+		/* find and remove from the map */
+		bsearch_insert_pos(&uids[i].seq1, &msgid_map[j],
+				   map_count - j, sizeof(*msgid_map),
+				   msgid_map_cmp, &idx);
+		j += idx;
+		if (j == map_count) {
+			/* all removals after this are about messages we never
+			   even added to the cache */
+			i_assert(uids[i].seq1 > cache->last_uid);
+			break;
+		}
+		while (j > 0 && msgid_map[j-1].uid == msgid_map[j].uid)
+			j--;
+
+		/* remove the messages from cache */
+		for (uid = uids[i].seq1; uid <= uids[i].seq2; uid++) {
+			if (j == map_count) {
+				i_assert(uid > cache->last_uid);
+				break;
+			}
+			i_assert(msgid_map[j].uid == uid);
+			if (!mail_thread_remove(cache, msgid_map + j, &j))
+				return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+static void mail_thread_cache_update_adds(struct mail_thread_mailbox *tbox,
+					  ARRAY_TYPE(seq_range) *added_uids)
+{
+	struct mail_thread_cache *cache = tbox->cache;
+	const struct seq_range *uids;
+	const struct mail_index_strmap_rec *msgid_map;
+	unsigned int i, j, map_count, uid_count;
+	uint32_t uid;
+
+	/* everything removed successfully, add the new messages. all of them
+	   should already be in msgid_map. */
+	uids = array_get(added_uids, &uid_count);
+	if (uid_count == 0)
+		return;
+
+	(void)array_bsearch_insert_pos(tbox->msgid_map, &uids[0].seq1,
+				       msgid_map_cmp, &j);
+	msgid_map = array_get(tbox->msgid_map, &map_count);
+	i_assert(j < map_count);
+	while (j > 0 && msgid_map[j-1].uid == msgid_map[j].uid)
+		j--;
+
+	for (i = 0; i < uid_count; i++) {
+		for (uid = uids[i].seq1; uid <= uids[i].seq2; uid++) {
+			while (j < map_count && msgid_map[j].uid < uid)
+				j++;
+			i_assert(j < map_count && msgid_map[j].uid == uid);
+			mail_thread_add(cache, msgid_map+j, &j);
+		}
+	}
+}
+
+static void
+mail_thread_cache_fix_invalid_indexes(struct mail_thread_mailbox *tbox)
+{
+	struct mail_thread_cache *cache = tbox->cache;
+	uint32_t highest_idx, new_first_idx, count;
+
+	highest_idx = mail_index_strmap_view_get_highest_idx(tbox->strmap_view);
+	new_first_idx = highest_idx + 1 +
+		THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT;
+	count = cache->next_invalid_msgid_str_idx -
+		cache->first_invalid_msgid_str_idx;
+
+	if (count == 0) {
+		/* there are no invalid indexes yet, we can update the first
+		   invalid index position to delay conflicts. */
+		cache->first_invalid_msgid_str_idx =
+			cache->next_invalid_msgid_str_idx = new_first_idx;
+	} else if (highest_idx >= cache->first_invalid_msgid_str_idx) {
+		/* conflict - move the invalid indexes forward */
+		array_copy(&cache->thread_nodes.arr, new_first_idx,
+			   &cache->thread_nodes.arr,
+			   cache->first_invalid_msgid_str_idx, count);
+		cache->first_invalid_msgid_str_idx = new_first_idx;
+		cache->next_invalid_msgid_str_idx = new_first_idx + count;
+	}
+}
+
+static void mail_thread_cache_sync_remove(struct mail_thread_mailbox *tbox,
+					  struct mail_thread_context *ctx)
+{
+	struct mail_thread_cache *cache = tbox->cache;
+
+	if (cache->search_result == NULL)
+		return;
+
+	if (mail_search_args_equal(ctx->search_args,
+				   cache->search_result->search_args)) {
+		t_array_init(&ctx->added_uids, 64);
+		if (mail_thread_cache_update_removes(tbox, &ctx->added_uids)) {
+			/* successfully updated the cache */
+			return;
+		}
+	}
+	/* failed to use the cache, rebuild */
+	mailbox_search_result_free(&cache->search_result);
+}
+
+static void mail_thread_cache_sync_add(struct mail_thread_mailbox *tbox,
+				       struct mail_thread_context *ctx,
+				       struct mail_search_context *search_ctx)
+{
+	struct mail_thread_cache *cache = tbox->cache;
+	struct mail *mail;
+	const struct mail_index_strmap_rec *msgid_map;
+	unsigned int i, count;
+
+	mail_thread_cache_fix_invalid_indexes(tbox);
+
+	if (cache->search_result != NULL) {
+		/* we already checked at sync_remove that we can use this
+		   search result. */
+		mail_thread_cache_update_adds(tbox, &ctx->added_uids);
+		return;
+	}
+
+	cache->last_uid = 0;
+	cache->first_invalid_msgid_str_idx = cache->next_invalid_msgid_str_idx =
+		mail_index_strmap_view_get_highest_idx(tbox->strmap_view) + 1 +
+		THREAD_INVALID_MSGID_STR_IDX_SKIP_COUNT;
+	array_clear(&cache->thread_nodes);
+
+	cache->search_result =
+		mailbox_search_result_save(search_ctx,
+			MAILBOX_SEARCH_RESULT_FLAG_UPDATE |
+			MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC);
+
+	msgid_map = array_get(tbox->msgid_map, &count);
+	/* we're relying on the array being zero-terminated (outside used
+	   count - kind of kludgy) */
+	i_assert(msgid_map[count].uid == 0);
+	i = 0;
+	while (i < count && mailbox_search_next(search_ctx, &mail)) {
+		while (msgid_map[i].uid < mail->uid)
+			i++;
+		i_assert(i < count);
+		mail_thread_add(cache, msgid_map+i, &i);
+	}
+}
+
+int mail_thread_init(struct mailbox *box, struct mail_search_args *args,
+		     struct mail_thread_context **ctx_r)
+{
+	struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(box);
+	struct mail_thread_context *ctx;
+	struct mail_search_context *search_ctx;
+	int ret;
+
+	i_assert(tbox->ctx == NULL);
+
+	struct event_reason *reason = event_reason_begin("mailbox:thread");
+	if (args != NULL)
+		mail_search_args_ref(args);
+	else {
+		args = mail_search_build_init();
+		mail_search_build_add_all(args);
+		mail_search_args_init(args, box, FALSE, NULL);
+	}
+
+	ctx = i_new(struct mail_thread_context, 1);
+	ctx->box = box;
+	ctx->search_args = args;
+	ctx->t = mailbox_transaction_begin(ctx->box, 0, __func__);
+	/* perform search first, so we don't break if there are INTHREAD keys */
+	search_ctx = mailbox_search_init(ctx->t, args, NULL, 0, NULL);
+
+	tbox->ctx = ctx;
+
+	mail_thread_cache_sync_remove(tbox, ctx);
+	ret = mail_thread_index_map_build(ctx);
+	if (ret == 0)
+		mail_thread_cache_sync_add(tbox, ctx, search_ctx);
+	if (mailbox_search_deinit(&search_ctx) < 0)
+		ret = -1;
+	if (ctx->failed) {
+		ret = -1;
+		if (ctx->corrupted)
+			mail_index_strmap_view_set_corrupted(tbox->strmap_view);
+	}
+	event_reason_end(&reason);
+	if (ret < 0) {
+		mail_thread_deinit(&ctx);
+		return -1;
+	} else {
+		i_zero(&ctx->added_uids);
+		*ctx_r = ctx;
+		return 0;
+	}
+}
+
+static void mail_thread_clear(struct mail_thread_context *ctx)
+{
+	mail_free(&ctx->tmp_mail);
+	(void)mailbox_transaction_commit(&ctx->t);
+}
+
+void mail_thread_deinit(struct mail_thread_context **_ctx)
+{
+	struct mail_thread_context *ctx = *_ctx;
+	struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(ctx->box);
+
+	*_ctx = NULL;
+
+	mail_thread_clear(ctx);
+	mail_search_args_unref(&ctx->search_args);
+	tbox->ctx = NULL;
+	i_free(ctx);
+}
+
+struct mail_thread_iterate_context *
+mail_thread_iterate_init(struct mail_thread_context *ctx,
+			 enum mail_thread_type thread_type, bool write_seqs)
+{
+	struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(ctx->box);
+
+	return mail_thread_iterate_init_full(tbox->cache, ctx->tmp_mail,
+					     thread_type, write_seqs);
+}
+
+static void mail_thread_mailbox_close(struct mailbox *box)
+{
+	struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(box);
+
+	i_assert(tbox->ctx == NULL);
+
+	if (tbox->strmap_view != NULL)
+		mail_index_strmap_view_close(&tbox->strmap_view);
+	if (tbox->cache->search_result != NULL)
+		mailbox_search_result_free(&tbox->cache->search_result);
+	tbox->module_ctx.super.close(box);
+}
+
+static void mail_thread_mailbox_free(struct mailbox *box)
+{
+	struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT_REQUIRE(box);
+
+	mail_index_strmap_deinit(&tbox->strmap);
+	tbox->module_ctx.super.free(box);
+
+	array_free(&tbox->cache->thread_nodes);
+	i_free(tbox->cache);
+	i_free(tbox);
+}
+
+void index_thread_mailbox_opened(struct mailbox *box)
+{
+	struct mail_thread_mailbox *tbox = MAIL_THREAD_CONTEXT(box);
+
+	if (tbox != NULL) {
+		/* mailbox was already opened+closed once. */
+		return;
+	}
+
+	tbox = i_new(struct mail_thread_mailbox, 1);
+	tbox->module_ctx.super = box->v;
+	box->v.close = mail_thread_mailbox_close;
+	box->v.free = mail_thread_mailbox_free;
+
+	tbox->strmap = mail_index_strmap_init(box->index,
+					      MAIL_THREAD_INDEX_SUFFIX);
+	tbox->next_msgid_idx = 1;
+
+	tbox->cache = i_new(struct mail_thread_cache, 1);
+	i_array_init(&tbox->cache->thread_nodes, 128);
+
+	MODULE_CONTEXT_SET(box, mail_thread_storage_module, tbox);
+}
diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c
new file mode 100644
index 0000000..edd6c3f
--- /dev/null
+++ b/src/lib-storage/index/index-transaction.c
@@ -0,0 +1,227 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "dict.h"
+#include "index-storage.h"
+#include "index-sync-private.h"
+#include "index-pop3-uidl.h"
+#include "index-mail.h"
+
+static void index_transaction_free(struct mailbox_transaction_context *t)
+{
+	if (t->view_pvt != NULL)
+		mail_index_view_close(&t->view_pvt);
+	mail_cache_view_close(&t->cache_view);
+	mail_index_view_close(&t->view);
+	if (array_is_created(&t->pvt_saves))
+		array_free(&t->pvt_saves);
+	array_free(&t->module_contexts);
+	i_free(t->reason);
+	i_free(t);
+}
+
+static int
+index_transaction_index_commit(struct mail_index_transaction *index_trans,
+			       struct mail_index_transaction_commit_result *result_r)
+{
+	struct mailbox_transaction_context *t =
+		MAIL_STORAGE_CONTEXT_REQUIRE(index_trans);
+	struct index_mailbox_sync_pvt_context *pvt_sync_ctx = NULL;
+	const char *error;
+	int ret = 0;
+
+	index_pop3_uidl_update_exists_finish(t);
+
+	if (t->attr_pvt_trans != NULL) {
+		if (dict_transaction_commit(&t->attr_pvt_trans, &error) < 0) {
+			mailbox_set_critical(t->box,
+				"Dict private transaction commit failed: %s", error);
+			ret = -1;
+		}
+	}
+	if (t->attr_shared_trans != NULL) {
+		if (dict_transaction_commit(&t->attr_shared_trans, &error) < 0) {
+			mailbox_set_critical(t->box,
+				"Dict shared transaction commit failed: %s", error);
+			ret = -1;
+		}
+	}
+
+	if (t->save_ctx != NULL) {
+		mailbox_save_context_deinit(t->save_ctx);
+		if (ret < 0) {
+			t->box->v.transaction_save_rollback(t->save_ctx);
+			t->save_ctx = NULL;
+		} else if (t->box->v.transaction_save_commit_pre(t->save_ctx) < 0) {
+			t->save_ctx = NULL;
+			ret = -1;
+		}
+	}
+
+	if (array_is_created(&t->pvt_saves)) {
+		if (index_mailbox_sync_pvt_init(t->box, TRUE, 0, &pvt_sync_ctx) < 0)
+			ret = -1;
+	}
+
+	i_assert(t->mail_ref_count == 0);
+	if (ret < 0)
+		t->super.rollback(index_trans);
+	else {
+		if (t->super.commit(index_trans, result_r) < 0) {
+			mailbox_set_index_error(t->box);
+			ret = -1;
+		} else {
+			t->changes->changes_mask = result_r->changes_mask;
+		}
+	}
+
+	if (t->save_ctx != NULL) {
+		i_assert(t->save_ctx->dest_mail == NULL);
+		t->box->v.transaction_save_commit_post(t->save_ctx, result_r);
+	}
+
+	if (pvt_sync_ctx != NULL) {
+		if (index_mailbox_sync_pvt_newmails(pvt_sync_ctx, t) < 0) {
+			/* failed to add private flags. a bit too late to
+			   return failure though, so just ignore silently */
+		}
+		index_mailbox_sync_pvt_deinit(&pvt_sync_ctx);
+	}
+
+	if (ret < 0)
+		mail_index_set_error_nolog(t->box->index, mailbox_get_last_error(t->box, NULL));
+	index_transaction_free(t);
+	return ret;
+}
+
+static void
+index_transaction_index_rollback(struct mail_index_transaction *index_trans)
+{
+	struct mailbox_transaction_context *t =
+		MAIL_STORAGE_CONTEXT_REQUIRE(index_trans);
+
+	dict_transaction_rollback(&t->attr_pvt_trans);
+	dict_transaction_rollback(&t->attr_shared_trans);
+
+	if (t->save_ctx != NULL) {
+		mailbox_save_context_deinit(t->save_ctx);
+		t->box->v.transaction_save_rollback(t->save_ctx);
+	}
+
+	i_assert(t->mail_ref_count == 0);
+	t->super.rollback(index_trans);
+	index_transaction_free(t);
+}
+
+static enum mail_index_transaction_flags
+index_transaction_flags_get(enum mailbox_transaction_flags flags)
+{
+	enum mail_index_transaction_flags itrans_flags;
+
+	itrans_flags = MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES;
+	if ((flags & MAILBOX_TRANSACTION_FLAG_HIDE) != 0)
+		itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_HIDE;
+	if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0)
+		itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
+	if ((flags & MAILBOX_TRANSACTION_FLAG_SYNC) != 0)
+		itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_SYNC;
+	return itrans_flags;
+}
+
+void index_transaction_init_pvt(struct mailbox_transaction_context *t)
+{
+	enum mail_index_transaction_flags itrans_flags;
+
+	if (t->box->view_pvt == NULL || t->itrans_pvt != NULL)
+		return;
+
+	itrans_flags = index_transaction_flags_get(t->flags);
+	t->itrans_pvt = mail_index_transaction_begin(t->box->view_pvt,
+						     itrans_flags);
+	t->view_pvt = mail_index_transaction_open_updated_view(t->itrans_pvt);
+}
+
+void index_transaction_init(struct mailbox_transaction_context *t,
+			    struct mailbox *box,
+			    enum mailbox_transaction_flags flags,
+			    const char *reason)
+{
+	enum mail_index_transaction_flags itrans_flags;
+
+	i_assert(box->opened);
+
+	itrans_flags = index_transaction_flags_get(flags);
+	if ((flags & MAILBOX_TRANSACTION_FLAG_REFRESH) != 0)
+		mail_index_refresh(box->index);
+
+	t->flags = flags;
+	t->box = box;
+	t->reason = i_strdup(reason);
+	t->itrans = mail_index_transaction_begin(box->view, itrans_flags);
+	t->view = mail_index_transaction_open_updated_view(t->itrans);
+
+	array_create(&t->module_contexts, default_pool,
+		     sizeof(void *), 5);
+
+	t->cache_view = mail_cache_view_open(box->cache, t->view);
+	t->cache_trans = mail_cache_get_transaction(t->cache_view, t->itrans);
+
+	if ((flags & MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC) != 0)
+		mail_cache_view_update_cache_decisions(t->cache_view, FALSE);
+
+	/* set up after mail_cache_get_transaction(), so that we'll still
+	   have the cache_trans available in _index_commit() */
+	t->super = t->itrans->v;
+	t->itrans->v.commit = index_transaction_index_commit;
+	t->itrans->v.rollback = index_transaction_index_rollback;
+	MODULE_CONTEXT_SET(t->itrans, mail_storage_mail_index_module, t);
+}
+
+struct mailbox_transaction_context *
+index_transaction_begin(struct mailbox *box,
+			enum mailbox_transaction_flags flags,
+			const char *reason)
+{
+	struct mailbox_transaction_context *t;
+
+	t = i_new(struct mailbox_transaction_context, 1);
+	index_transaction_init(t, box, flags, reason);
+	return t;
+}
+
+int index_transaction_commit(struct mailbox_transaction_context *t,
+			     struct mail_transaction_commit_changes *changes_r)
+{
+	struct mailbox *box = t->box;
+	struct mail_index_transaction *itrans = t->itrans;
+	struct mail_index_transaction_commit_result result;
+	int ret = 0;
+
+	i_zero(changes_r);
+	changes_r->pool = pool_alloconly_create(MEMPOOL_GROWING
+						"transaction changes", 512);
+	p_array_init(&changes_r->saved_uids, changes_r->pool, 32);
+	t->changes = changes_r;
+
+	if (t->itrans_pvt != NULL)
+		ret = mail_index_transaction_commit(&t->itrans_pvt);
+	if (mail_index_transaction_commit_full(&itrans, &result) < 0)
+		ret = -1;
+	t = NULL;
+
+	if (ret < 0 && mail_index_is_deleted(box->index))
+		mailbox_set_deleted(box);
+
+	changes_r->ignored_modseq_changes = result.ignored_modseq_changes;
+	return ret;
+}
+
+void index_transaction_rollback(struct mailbox_transaction_context *t)
+{
+	struct mail_index_transaction *itrans = t->itrans;
+
+	if (t->itrans_pvt != NULL)
+		mail_index_transaction_rollback(&t->itrans_pvt);
+	mail_index_transaction_rollback(&itrans);
+}
diff --git a/src/lib-storage/index/istream-mail.c b/src/lib-storage/index/istream-mail.c
new file mode 100644
index 0000000..1477823
--- /dev/null
+++ b/src/lib-storage/index/istream-mail.c
@@ -0,0 +1,173 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-storage-private.h"
+#include "istream-private.h"
+#include "index-mail.h"
+#include "istream-mail.h"
+
+struct mail_istream {
+	struct istream_private istream;
+
+	struct mail *mail;
+	uoff_t expected_size;
+	bool files_read_increased:1;
+	bool input_has_body:1;
+};
+
+static bool i_stream_mail_try_get_cached_size(struct mail_istream *mstream)
+{
+	struct mail *mail = mstream->mail;
+	enum mail_lookup_abort orig_lookup_abort;
+
+	if (mstream->expected_size != UOFF_T_MAX)
+		return TRUE;
+
+	/* make sure this call doesn't change any existing error message,
+	   just in case there's already something important in it. */
+	mail_storage_last_error_push(mail->box->storage);
+	orig_lookup_abort = mail->lookup_abort;
+	mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+	if (mail_get_physical_size(mail, &mstream->expected_size) < 0)
+		mstream->expected_size = UOFF_T_MAX;
+	mail->lookup_abort = orig_lookup_abort;
+	mail_storage_last_error_pop(mail->box->storage);
+	return mstream->expected_size != UOFF_T_MAX;
+}
+
+static const char *
+i_stream_mail_get_cached_mail_id(struct mail_istream *mstream ATTR_UNUSED)
+{
+#if 0
+	/* FIXME: This function may get called in the middle of header parsing,
+	   which then goes into parsing cached headers and causes crashes.
+	   So disable this for now. Eventually it would be nice if recursion
+	   was possible by each parser using its own private struct. */
+	static const char *headers[] = {
+		"Message-Id",
+		"Date",
+		"Subject"
+	};
+	struct mail *mail = mstream->mail;
+	enum mail_lookup_abort orig_lookup_abort;
+	const char *value, *ret = "";
+	unsigned int i;
+
+	orig_lookup_abort = mail->lookup_abort;
+	mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+	for (i = 0; i < N_ELEMENTS(headers); i++) {
+		if (mail_get_first_header(mail, headers[i], &value) > 0) {
+			ret = t_strdup_printf("%s=%s", headers[i], value);
+			break;
+		}
+	}
+	mail->lookup_abort = orig_lookup_abort;
+	return ret;
+#else
+	return "";
+#endif
+}
+
+static void
+i_stream_mail_set_size_corrupted(struct mail_istream *mstream, size_t size)
+{
+	uoff_t cur_size = mstream->istream.istream.v_offset + size;
+	const char *str, *mail_id;
+	char chr;
+
+	if (mstream->expected_size < cur_size) {
+		/* input stream is larger than cached message size */
+		str = "smaller";
+		chr = '<';
+		mstream->istream.istream.stream_errno = EINVAL;
+	} else {
+		/* input stream is smaller than cached message size */
+		str = "larger";
+		chr = '>';
+		mstream->istream.istream.stream_errno = EPIPE;
+	}
+
+	mail_id = i_stream_mail_get_cached_mail_id(mstream);
+	if (mail_id[0] != '\0')
+		mail_id = t_strconcat(", cached ", mail_id, NULL);
+	io_stream_set_error(&mstream->istream.iostream,
+		"Cached message size %s than expected "
+		"(%"PRIuUOFF_T" %c %"PRIuUOFF_T", box=%s, UID=%u%s)", str,
+		mstream->expected_size, chr, cur_size,
+		mailbox_get_vname(mstream->mail->box),
+		mstream->mail->uid, mail_id);
+	mail_set_cache_corrupted(mstream->mail, MAIL_FETCH_PHYSICAL_SIZE,
+		t_strdup_printf("read(%s) failed: %s",
+				i_stream_get_name(&mstream->istream.istream),
+				mstream->istream.iostream.error));
+}
+
+static ssize_t
+i_stream_mail_read(struct istream_private *stream)
+{
+	struct mail_istream *mstream = (struct mail_istream *)stream;
+	size_t size;
+	ssize_t ret;
+
+	i_stream_seek(stream->parent, stream->parent_start_offset +
+		      stream->istream.v_offset);
+
+	ret = i_stream_read_copy_from_parent(&stream->istream);
+	size = i_stream_get_data_size(&stream->istream);
+	if (ret > 0) {
+		mstream->mail->transaction->stats.files_read_bytes += ret;
+		if (!mstream->files_read_increased) {
+			mstream->files_read_increased = TRUE;
+			mstream->mail->transaction->stats.files_read_count++;
+		}
+		if (mstream->expected_size < stream->istream.v_offset + size) {
+			i_stream_mail_set_size_corrupted(mstream, size);
+			/* istream code expects that the position has not changed
+			   when read error occurs, so move pos back. */
+			i_assert(stream->pos >= (size_t)ret);
+			stream->pos -= ret;
+			return -1;
+		}
+	} else if (ret == -1 && stream->istream.eof) {
+		if (!mstream->input_has_body) {
+			/* trying to read past the header, but this stream
+			   doesn't have the body */
+			return -1;
+		}
+		if (stream->istream.stream_errno != 0) {
+			if (stream->istream.stream_errno == ENOENT) {
+				/* update mail's expunged-flag if needed */
+				index_mail_refresh_expunged(mstream->mail);
+			}
+			return -1;
+		}
+		if (i_stream_mail_try_get_cached_size(mstream) &&
+		    mstream->expected_size > stream->istream.v_offset + size) {
+			i_stream_mail_set_size_corrupted(mstream, size);
+			return -1;
+		}
+	}
+	return ret;
+}
+
+struct istream *i_stream_create_mail(struct mail *mail, struct istream *input,
+				     bool input_has_body)
+{
+	struct mail_istream *mstream;
+
+	mstream = i_new(struct mail_istream, 1);
+	mstream->mail = mail;
+	mstream->input_has_body = input_has_body;
+	mstream->expected_size = UOFF_T_MAX;
+	(void)i_stream_mail_try_get_cached_size(mstream);
+	mstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+	mstream->istream.stream_size_passthrough = TRUE;
+
+	mstream->istream.read = i_stream_mail_read;
+
+	mstream->istream.istream.readable_fd = input->readable_fd;
+	mstream->istream.istream.blocking = input->blocking;
+	mstream->istream.istream.seekable = input->seekable;
+	return i_stream_create(&mstream->istream, input,
+			       i_stream_get_fd(input), 0);
+}
diff --git a/src/lib-storage/index/istream-mail.h b/src/lib-storage/index/istream-mail.h
new file mode 100644
index 0000000..24bd567
--- /dev/null
+++ b/src/lib-storage/index/istream-mail.h
@@ -0,0 +1,7 @@
+#ifndef ISTREAM_MAIL_H
+#define ISTREAM_MAIL_H
+
+struct istream *i_stream_create_mail(struct mail *mail, struct istream *input,
+				     bool input_has_body);
+
+#endif
diff --git a/src/lib-storage/index/maildir/Makefile.am b/src/lib-storage/index/maildir/Makefile.am
new file mode 100644
index 0000000..7acf249
--- /dev/null
+++ b/src/lib-storage/index/maildir/Makefile.am
@@ -0,0 +1,36 @@
+noinst_LTLIBRARIES = libstorage_maildir.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_maildir_la_SOURCES = \
+	maildir-copy.c \
+	maildir-filename.c \
+	maildir-filename-flags.c \
+	maildir-keywords.c \
+	maildir-mail.c \
+	maildir-save.c \
+	maildir-settings.c \
+	maildir-storage.c \
+	maildir-sync.c \
+	maildir-sync-index.c \
+	maildir-uidlist.c \
+	maildir-util.c
+
+headers = \
+	maildir-filename.h \
+	maildir-filename-flags.h \
+	maildir-keywords.h \
+	maildir-storage.h \
+	maildir-settings.h \
+	maildir-sync.h \
+	maildir-uidlist.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/maildir/Makefile.in b/src/lib-storage/index/maildir/Makefile.in
new file mode 100644
index 0000000..7210c54
--- /dev/null
+++ b/src/lib-storage/index/maildir/Makefile.in
@@ -0,0 +1,875 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/maildir
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_maildir_la_LIBADD =
+am_libstorage_maildir_la_OBJECTS = maildir-copy.lo maildir-filename.lo \
+	maildir-filename-flags.lo maildir-keywords.lo maildir-mail.lo \
+	maildir-save.lo maildir-settings.lo maildir-storage.lo \
+	maildir-sync.lo maildir-sync-index.lo maildir-uidlist.lo \
+	maildir-util.lo
+libstorage_maildir_la_OBJECTS = $(am_libstorage_maildir_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/maildir-copy.Plo \
+	./$(DEPDIR)/maildir-filename-flags.Plo \
+	./$(DEPDIR)/maildir-filename.Plo \
+	./$(DEPDIR)/maildir-keywords.Plo ./$(DEPDIR)/maildir-mail.Plo \
+	./$(DEPDIR)/maildir-save.Plo ./$(DEPDIR)/maildir-settings.Plo \
+	./$(DEPDIR)/maildir-storage.Plo \
+	./$(DEPDIR)/maildir-sync-index.Plo \
+	./$(DEPDIR)/maildir-sync.Plo ./$(DEPDIR)/maildir-uidlist.Plo \
+	./$(DEPDIR)/maildir-util.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_maildir_la_SOURCES)
+DIST_SOURCES = $(libstorage_maildir_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_maildir.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_maildir_la_SOURCES = \
+	maildir-copy.c \
+	maildir-filename.c \
+	maildir-filename-flags.c \
+	maildir-keywords.c \
+	maildir-mail.c \
+	maildir-save.c \
+	maildir-settings.c \
+	maildir-storage.c \
+	maildir-sync.c \
+	maildir-sync-index.c \
+	maildir-uidlist.c \
+	maildir-util.c
+
+headers = \
+	maildir-filename.h \
+	maildir-filename-flags.h \
+	maildir-keywords.h \
+	maildir-storage.h \
+	maildir-settings.h \
+	maildir-sync.h \
+	maildir-uidlist.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/maildir/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/maildir/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_maildir.la: $(libstorage_maildir_la_OBJECTS) $(libstorage_maildir_la_DEPENDENCIES) $(EXTRA_libstorage_maildir_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_maildir_la_OBJECTS) $(libstorage_maildir_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-copy.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-filename-flags.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-filename.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-keywords.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-save.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-settings.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-sync-index.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-sync.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-uidlist.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/maildir-util.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/maildir-copy.Plo
+	-rm -f ./$(DEPDIR)/maildir-filename-flags.Plo
+	-rm -f ./$(DEPDIR)/maildir-filename.Plo
+	-rm -f ./$(DEPDIR)/maildir-keywords.Plo
+	-rm -f ./$(DEPDIR)/maildir-mail.Plo
+	-rm -f ./$(DEPDIR)/maildir-save.Plo
+	-rm -f ./$(DEPDIR)/maildir-settings.Plo
+	-rm -f ./$(DEPDIR)/maildir-storage.Plo
+	-rm -f ./$(DEPDIR)/maildir-sync-index.Plo
+	-rm -f ./$(DEPDIR)/maildir-sync.Plo
+	-rm -f ./$(DEPDIR)/maildir-uidlist.Plo
+	-rm -f ./$(DEPDIR)/maildir-util.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/maildir-copy.Plo
+	-rm -f ./$(DEPDIR)/maildir-filename-flags.Plo
+	-rm -f ./$(DEPDIR)/maildir-filename.Plo
+	-rm -f ./$(DEPDIR)/maildir-keywords.Plo
+	-rm -f ./$(DEPDIR)/maildir-mail.Plo
+	-rm -f ./$(DEPDIR)/maildir-save.Plo
+	-rm -f ./$(DEPDIR)/maildir-settings.Plo
+	-rm -f ./$(DEPDIR)/maildir-storage.Plo
+	-rm -f ./$(DEPDIR)/maildir-sync-index.Plo
+	-rm -f ./$(DEPDIR)/maildir-sync.Plo
+	-rm -f ./$(DEPDIR)/maildir-uidlist.Plo
+	-rm -f ./$(DEPDIR)/maildir-util.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/maildir/maildir-copy.c b/src/lib-storage/index/maildir/maildir-copy.c
new file mode 100644
index 0000000..1645ddb
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-copy.c
@@ -0,0 +1,149 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "nfs-workarounds.h"
+#include "maildir-storage.h"
+#include "maildir-uidlist.h"
+#include "maildir-filename.h"
+#include "maildir-keywords.h"
+#include "maildir-sync.h"
+#include "index-mail.h"
+#include "mail-copy.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+
+struct hardlink_ctx {
+	const char *dest_path;
+	bool success:1;
+};
+
+static int do_hardlink(struct maildir_mailbox *mbox, const char *path,
+		       struct hardlink_ctx *ctx)
+{
+	int ret;
+
+	if (mbox->storage->storage.set->mail_nfs_storage)
+		ret = nfs_safe_link(path, ctx->dest_path, FALSE);
+	else
+		ret = link(path, ctx->dest_path);
+	if (ret < 0) {
+		if (errno == ENOENT)
+			return 0;
+
+		if (ENOQUOTA(errno)) {
+			mail_storage_set_error(&mbox->storage->storage,
+				MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA);
+			return -1;
+		}
+
+		/* we could handle the EEXIST condition by changing the
+		   filename, but it practically never happens so just fallback
+		   to standard copying for the rare cases when it does. */
+		if (errno == EACCES || ECANTLINK(errno) || errno == EEXIST)
+			return 1;
+
+		mailbox_set_critical(&mbox->box, "link(%s, %s) failed: %m",
+				     path, ctx->dest_path);
+		return -1;
+	}
+
+	ctx->success = TRUE;
+	return 1;
+}
+
+static int
+maildir_copy_hardlink(struct mail_save_context *ctx, struct mail *mail)
+{
+	struct maildir_mailbox *dest_mbox = MAILDIR_MAILBOX(ctx->transaction->box);
+	struct maildir_mailbox *src_mbox;
+	struct maildir_filename *mf;
+	struct hardlink_ctx do_ctx;
+	const char *path, *guid, *dest_fname;
+	uoff_t vsize, size;
+	enum mail_lookup_abort old_abort;
+
+	if (strcmp(mail->box->storage->name, MAILDIR_STORAGE_NAME) == 0)
+		src_mbox = MAILDIR_MAILBOX(mail->box);
+	else if (strcmp(mail->box->storage->name, "raw") == 0) {
+		/* lda uses raw format */
+		src_mbox = NULL;
+	} else {
+		/* Can't hard link files from the source storage */
+		return 0;
+	}
+
+	/* hard link to tmp/ with a newly generated filename and later when we
+	   have uidlist locked, move it to new/cur. */
+	dest_fname = maildir_filename_generate();
+	i_zero(&do_ctx);
+	do_ctx.dest_path =
+		t_strdup_printf("%s/tmp/%s", mailbox_get_path(&dest_mbox->box),
+				dest_fname);
+	if (src_mbox != NULL) {
+		/* maildir */
+		if (maildir_file_do(src_mbox, mail->uid,
+				    do_hardlink, &do_ctx) < 0)
+			return -1;
+	} else {
+		/* raw / lda */
+		if (mail_get_special(mail, MAIL_FETCH_STORAGE_ID,
+				     &path) < 0 || *path == '\0')
+			return 0;
+		if (do_hardlink(dest_mbox, path, &do_ctx) < 0)
+			return -1;
+	}
+
+	if (!do_ctx.success) {
+		/* couldn't copy with hardlinking, fallback to copying */
+		return 0;
+	}
+
+	/* hardlinked to tmp/, treat as normal copied mail */
+	mf = maildir_save_add(ctx, dest_fname, mail);
+	if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) == 0) {
+		if (*guid != '\0')
+			maildir_save_set_dest_basename(ctx, mf, guid);
+	}
+
+	/* finish copying keywords */
+	maildir_save_finish_keywords(ctx);
+
+	/* remember size/vsize if possible */
+	old_abort = mail->lookup_abort;
+	mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
+	if (mail_get_physical_size(mail, &size) < 0)
+		size = UOFF_T_MAX;
+	if (mail_get_virtual_size(mail, &vsize) < 0)
+		vsize = UOFF_T_MAX;
+	maildir_save_set_sizes(mf, size, vsize);
+	mail->lookup_abort = old_abort;
+	return 1;
+}
+
+int maildir_copy(struct mail_save_context *ctx, struct mail *mail)
+{
+	struct mailbox_transaction_context *_t = ctx->transaction;
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_t->box);
+	int ret;
+
+	i_assert((_t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	if (mbox->storage->set->maildir_copy_with_hardlinks &&
+	    mail_storage_copy_can_use_hardlink(mail->box, &mbox->box)) {
+		T_BEGIN {
+			ret = maildir_copy_hardlink(ctx, mail);
+		} T_END;
+
+		if (ret != 0) {
+			index_save_context_free(ctx);
+			return ret > 0 ? 0 : -1;
+		}
+
+		/* non-fatal hardlinking failure, try the slow way */
+	}
+
+	return mail_storage_copy(ctx, mail);
+}
diff --git a/src/lib-storage/index/maildir/maildir-filename-flags.c b/src/lib-storage/index/maildir/maildir-filename-flags.c
new file mode 100644
index 0000000..ac68908
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-filename-flags.c
@@ -0,0 +1,185 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "maildir-storage.h"
+#include "maildir-keywords.h"
+#include "maildir-filename-flags.h"
+
+
+void maildir_filename_flags_get(struct maildir_keywords_sync_ctx *ctx,
+				const char *fname, enum mail_flags *flags_r,
+				ARRAY_TYPE(keyword_indexes) *keywords_r)
+{
+	const char *info;
+
+	array_clear(keywords_r);
+	*flags_r = 0;
+
+	info = strrchr(fname, MAILDIR_INFO_SEP);
+	if (info == NULL || info[1] != '2' || info[2] != MAILDIR_FLAGS_SEP)
+		return;
+
+	for (info += 3; *info != '\0' && *info != MAILDIR_FLAGS_SEP; info++) {
+		switch (*info) {
+		case 'R': /* replied */
+			*flags_r |= MAIL_ANSWERED;
+			break;
+		case 'S': /* seen */
+			*flags_r |= MAIL_SEEN;
+			break;
+		case 'T': /* trashed */
+			*flags_r |= MAIL_DELETED;
+			break;
+		case 'D': /* draft */
+			*flags_r |= MAIL_DRAFT;
+			break;
+		case 'F': /* flagged */
+			*flags_r |= MAIL_FLAGGED;
+			break;
+		default:
+			if (*info >= MAILDIR_KEYWORD_FIRST &&
+			    *info <= MAILDIR_KEYWORD_LAST) {
+				int idx;
+
+				idx = maildir_keywords_char_idx(ctx, *info);
+				if (idx < 0) {
+					/* unknown keyword. */
+					break;
+				}
+
+				array_push_back(keywords_r,
+						(unsigned int *)&idx);
+				break;
+			}
+
+			/* unknown flag - ignore */
+			break;
+		}
+	}
+}
+
+static int char_cmp(const void *p1, const void *p2)
+{
+	const unsigned char *c1 = p1, *c2 = p2;
+
+	return *c1 - *c2;
+}
+
+static void
+maildir_filename_append_keywords(struct maildir_keywords_sync_ctx *ctx,
+				 ARRAY_TYPE(keyword_indexes) *keywords,
+				 string_t *fname)
+{
+	const unsigned int *indexes;
+	unsigned int i, count;
+	size_t start = str_len(fname);
+	char chr;
+
+	indexes = array_get(keywords, &count);
+	for (i = 0; i < count; i++) {
+		chr = maildir_keywords_idx_char(ctx, indexes[i]);
+		if (chr != '\0')
+			str_append_c(fname, chr);
+	}
+
+	qsort(str_c_modifiable(fname) + start, str_len(fname) - start, 1,
+	      char_cmp);
+}
+
+static const char * ATTR_NULL(1, 4)
+maildir_filename_flags_full_set(struct maildir_keywords_sync_ctx *ctx,
+				const char *fname, enum mail_flags flags,
+				ARRAY_TYPE(keyword_indexes) *keywords)
+{
+	string_t *flags_str;
+	enum mail_flags flags_left;
+	const char *info, *oldflags;
+	int nextflag;
+
+	/* remove the old :info from file name, and get the old flags */
+	info = strrchr(fname, MAILDIR_INFO_SEP);
+	if (info != NULL && strrchr(fname, '/') > info)
+		info = NULL;
+
+	oldflags = "";
+	if (info != NULL) {
+		fname = t_strdup_until(fname, info);
+		if (info[1] == '2' && info[2] == MAILDIR_FLAGS_SEP)
+			oldflags = info+3;
+	}
+
+	/* insert the new flags between old flags. flags must be sorted by
+	   their ASCII code. unknown flags are kept. */
+	flags_str = t_str_new(256);
+	str_append(flags_str, fname);
+	str_append(flags_str, MAILDIR_FLAGS_FULL_SEP);
+	flags_left = flags;
+	for (;;) {
+		/* skip all known flags */
+		while (*oldflags == 'D' || *oldflags == 'F' ||
+		       *oldflags == 'R' || *oldflags == 'S' ||
+		       *oldflags == 'T' ||
+		       (*oldflags >= MAILDIR_KEYWORD_FIRST &&
+			*oldflags <= MAILDIR_KEYWORD_LAST))
+			oldflags++;
+
+		nextflag = *oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP ?
+			256 : (unsigned char) *oldflags;
+
+		if ((flags_left & MAIL_DRAFT) != 0 && nextflag > 'D') {
+			str_append_c(flags_str, 'D');
+			flags_left &= ENUM_NEGATE(MAIL_DRAFT);
+		}
+		if ((flags_left & MAIL_FLAGGED) != 0 && nextflag > 'F') {
+			str_append_c(flags_str, 'F');
+			flags_left &= ENUM_NEGATE(MAIL_FLAGGED);
+		}
+		if ((flags_left & MAIL_ANSWERED) != 0 && nextflag > 'R') {
+			str_append_c(flags_str, 'R');
+			flags_left &= ENUM_NEGATE(MAIL_ANSWERED);
+		}
+		if ((flags_left & MAIL_SEEN) != 0 && nextflag > 'S') {
+			str_append_c(flags_str, 'S');
+			flags_left &= ENUM_NEGATE(MAIL_SEEN);
+		}
+		if ((flags_left & MAIL_DELETED) != 0 && nextflag > 'T') {
+			str_append_c(flags_str, 'T');
+			flags_left &= ENUM_NEGATE(MAIL_DELETED);
+		}
+
+		if (keywords != NULL && array_is_created(keywords) &&
+		    nextflag > MAILDIR_KEYWORD_FIRST) {
+			maildir_filename_append_keywords(ctx, keywords,
+							 flags_str);
+			keywords = NULL;
+		}
+
+		if (*oldflags == '\0' || *oldflags == MAILDIR_FLAGS_SEP)
+			break;
+
+		str_append_c(flags_str, *oldflags);
+		oldflags++;
+	}
+
+	if (*oldflags == MAILDIR_FLAGS_SEP) {
+		/* another flagset, we don't know about these, just keep them */
+		while (*oldflags != '\0')
+			str_append_c(flags_str, *oldflags++);
+	}
+
+	return str_c(flags_str);
+}
+
+const char *maildir_filename_flags_set(const char *fname, enum mail_flags flags)
+{
+	return maildir_filename_flags_full_set(NULL, fname, flags, NULL);
+}
+
+const char *maildir_filename_flags_kw_set(struct maildir_keywords_sync_ctx *ctx,
+					  const char *fname, enum mail_flags flags,
+					  ARRAY_TYPE(keyword_indexes) *keywords)
+{
+	return maildir_filename_flags_full_set(ctx, fname, flags, keywords);
+}
diff --git a/src/lib-storage/index/maildir/maildir-filename-flags.h b/src/lib-storage/index/maildir/maildir-filename-flags.h
new file mode 100644
index 0000000..2676e5c
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-filename-flags.h
@@ -0,0 +1,13 @@
+#ifndef MAILDIR_FILENAME_FLAGS_H
+#define MAILDIR_FILENAME_FLAGS_H
+
+void maildir_filename_flags_get(struct maildir_keywords_sync_ctx *ctx,
+			       const char *fname, enum mail_flags *flags_r,
+                               ARRAY_TYPE(keyword_indexes) *keywords_r);
+
+const char *maildir_filename_flags_set(const char *fname, enum mail_flags flags);
+const char *maildir_filename_flags_kw_set(struct maildir_keywords_sync_ctx *ctx,
+					  const char *fname, enum mail_flags flags,
+					  ARRAY_TYPE(keyword_indexes) *keywords);
+
+#endif
diff --git a/src/lib-storage/index/maildir/maildir-filename.c b/src/lib-storage/index/maildir/maildir-filename.c
new file mode 100644
index 0000000..9deba82
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-filename.c
@@ -0,0 +1,143 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "time-util.h"
+#include "hostpid.h"
+#include "maildir-storage.h"
+#include "maildir-filename.h"
+
+const char *maildir_filename_generate(void)
+{
+	static struct timeval last_tv = { 0, 0 };
+	struct timeval tv;
+
+	/* use secs + usecs to guarantee uniqueness within this process. */
+	if (timeval_cmp(&ioloop_timeval, &last_tv) > 0)
+		tv = ioloop_timeval;
+	else {
+		tv = last_tv;
+		if (++tv.tv_usec == 1000000) {
+			tv.tv_sec++;
+			tv.tv_usec = 0;
+		}
+	}
+	last_tv = tv;
+
+	return t_strdup_printf("%s.M%sP%s.%s",
+			       dec2str(tv.tv_sec), dec2str(tv.tv_usec),
+			       my_pid, my_hostname);
+}
+
+bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r)
+{
+	uoff_t size = 0;
+
+	for (; *fname != '\0'; fname++) {
+		i_assert(*fname != '/');
+		if (*fname == ',' && fname[1] == type && fname[2] == '=') {
+			fname += 3;
+			break;
+		}
+	}
+
+	if (*fname == '\0')
+		return FALSE;
+
+	while (*fname >= '0' && *fname <= '9') {
+		size = size * 10 + (*fname - '0');
+		fname++;
+	}
+
+	if (*fname != MAILDIR_INFO_SEP &&
+	    *fname != MAILDIR_EXTRA_SEP &&
+	    *fname != '\0')
+		return FALSE;
+
+	*size_r = size;
+	return TRUE;
+}
+
+/* a char* hash function from ASU -- from glib */
+unsigned int ATTR_NO_SANITIZE_INTEGER
+maildir_filename_base_hash(const char *s)
+{
+	unsigned int g, h = 0;
+
+	while (*s != MAILDIR_INFO_SEP && *s != '\0') {
+		i_assert(*s != '/');
+		h = (h << 4) + *s;
+		if ((g = h & 0xf0000000UL) != 0) {
+			h = h ^ (g >> 24);
+			h = h ^ g;
+		}
+
+		s++;
+	}
+
+	return h;
+}
+
+int maildir_filename_base_cmp(const char *fname1, const char *fname2)
+{
+	while (*fname1 == *fname2 && *fname1 != MAILDIR_INFO_SEP &&
+	       *fname1 != '\0') {
+		i_assert(*fname1 != '/');
+		fname1++; fname2++;
+	}
+
+	if ((*fname1 == '\0' || *fname1 == MAILDIR_INFO_SEP) &&
+	    (*fname2 == '\0' || *fname2 == MAILDIR_INFO_SEP))
+		return 0;
+	return *fname1 - *fname2;
+}
+
+static bool maildir_fname_get_usecs(const char *fname, int *usecs_r)
+{
+	int usecs = 0;
+
+	/* Assume we already read the timestamp. Next up is
+	   ".<uniqueness>.<host>". Find usecs inside the uniqueness. */
+	if (*fname != '.')
+		return FALSE;
+
+	fname++;
+	while (*fname != '\0' && *fname != '.' && *fname != MAILDIR_INFO_SEP) {
+		if (*fname++ == 'M') {
+			while (*fname >= '0' && *fname <= '9') {
+				usecs = usecs * 10 + (*fname - '0');
+				fname++;
+			}
+			*usecs_r = usecs;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+int maildir_filename_sort_cmp(const char *fname1, const char *fname2)
+{
+	const char *s1, *s2;
+	time_t secs1 = 0, secs2 = 0;
+	int ret, usecs1, usecs2;
+
+	/* sort primarily by the timestamp in file name */
+	for (s1 = fname1; *s1 >= '0' && *s1 <= '9'; s1++)
+		secs1 = secs1 * 10 + (*s1 - '0');
+	for (s2 = fname2; *s2 >= '0' && *s2 <= '9'; s2++)
+		secs2 = secs2 * 10 + (*s2 - '0');
+
+	ret = (int)((long)secs1 - (long)secs2);
+	if (ret == 0) {
+		/* sort secondarily by microseconds, if they exist */
+		if (maildir_fname_get_usecs(s1, &usecs1) &&
+		    maildir_fname_get_usecs(s2, &usecs2))
+			ret = usecs1 - usecs2;
+
+		if (ret == 0) {
+			/* fallback to comparing the base file name */
+			ret = maildir_filename_base_cmp(s1, s2);
+		}
+	}
+	return ret;
+}
diff --git a/src/lib-storage/index/maildir/maildir-filename.h b/src/lib-storage/index/maildir/maildir-filename.h
new file mode 100644
index 0000000..a691dea
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-filename.h
@@ -0,0 +1,14 @@
+#ifndef MAILDIR_FILENAME_H
+#define MAILDIR_FILENAME_H
+
+struct maildir_keywords_sync_ctx;
+
+const char *maildir_filename_generate(void);
+
+bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r);
+
+unsigned int maildir_filename_base_hash(const char *fname);
+int maildir_filename_base_cmp(const char *fname1, const char *fname2);
+int maildir_filename_sort_cmp(const char *fname1, const char *fname2);
+
+#endif
diff --git a/src/lib-storage/index/maildir/maildir-keywords.c b/src/lib-storage/index/maildir/maildir-keywords.c
new file mode 100644
index 0000000..a25d112
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-keywords.c
@@ -0,0 +1,499 @@
+/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */
+
+/* note that everything here depends on uidlist file being locked the whole
+   time. that's why we don't have any locking of our own, or that we do things
+   that would be racy otherwise. */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "hash.h"
+#include "str.h"
+#include "istream.h"
+#include "eacces-error.h"
+#include "file-dotlock.h"
+#include "write-full.h"
+#include "nfs-workarounds.h"
+#include "maildir-storage.h"
+#include "maildir-uidlist.h"
+#include "maildir-keywords.h"
+
+#include <sys/stat.h>
+#include <utime.h>
+
+/* how many seconds to wait before overriding dovecot-keywords.lock */
+#define KEYWORDS_LOCK_STALE_TIMEOUT (60*2)
+
+struct maildir_keywords {
+	struct maildir_mailbox *mbox;
+	struct mail_storage *storage;
+	char *path;
+
+	pool_t pool;
+	ARRAY_TYPE(keywords) list;
+	HASH_TABLE(char *, void *) hash; /* name -> idx+1 */
+
+        struct dotlock_settings dotlock_settings;
+
+	time_t synced_mtime;
+	bool synced:1;
+	bool changed:1;
+};
+
+struct maildir_keywords_sync_ctx {
+	struct maildir_keywords *mk;
+	struct mail_index *index;
+
+	const ARRAY_TYPE(keywords) *keywords;
+	ARRAY(char) idx_to_chr;
+	unsigned int chridx_to_idx[MAILDIR_MAX_KEYWORDS];
+	bool readonly;
+};
+
+struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox)
+{
+	struct maildir_keywords *mk;
+
+	mk = maildir_keywords_init_readonly(&mbox->box);
+	mk->mbox = mbox;
+	return mk;
+}
+
+struct maildir_keywords *
+maildir_keywords_init_readonly(struct mailbox *box)
+{
+	struct maildir_keywords *mk;
+	const char *dir;
+
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, &dir) <= 0)
+		i_unreached();
+
+	mk = i_new(struct maildir_keywords, 1);
+	mk->storage = box->storage;
+	mk->path = i_strconcat(dir, "/" MAILDIR_KEYWORDS_NAME, NULL);
+	mk->pool = pool_alloconly_create("maildir keywords", 512);
+	i_array_init(&mk->list, MAILDIR_MAX_KEYWORDS);
+	hash_table_create(&mk->hash, mk->pool, 0, strcase_hash, strcasecmp);
+
+	mk->dotlock_settings.use_excl_lock =
+		box->storage->set->dotlock_use_excl;
+	mk->dotlock_settings.nfs_flush =
+		box->storage->set->mail_nfs_storage;
+	mk->dotlock_settings.timeout =
+		mail_storage_get_lock_timeout(box->storage,
+					      KEYWORDS_LOCK_STALE_TIMEOUT + 2);
+	mk->dotlock_settings.stale_timeout = KEYWORDS_LOCK_STALE_TIMEOUT;
+	mk->dotlock_settings.temp_prefix =
+		mailbox_list_get_temp_prefix(box->list);
+	return mk;
+}
+
+void maildir_keywords_deinit(struct maildir_keywords **_mk)
+{
+	struct maildir_keywords *mk = *_mk;
+
+	*_mk = NULL;
+	hash_table_destroy(&mk->hash);
+	array_free(&mk->list);
+	pool_unref(&mk->pool);
+	i_free(mk->path);
+	i_free(mk);
+}
+
+static void maildir_keywords_clear(struct maildir_keywords *mk)
+{
+	array_clear(&mk->list);
+	hash_table_clear(mk->hash, TRUE);
+	p_clear(mk->pool);
+}
+
+static int maildir_keywords_sync(struct maildir_keywords *mk)
+{
+	struct istream *input;
+	struct stat st;
+	char *line, *p, *new_name;
+	const char **strp;
+	unsigned int idx;
+	int fd;
+
+        /* Remember that we rely on uidlist file locking in here. That's why
+           we rely on stat()'s timestamp and don't bother handling ESTALE
+           errors. */
+
+	if (mk->storage->set->mail_nfs_storage) {
+		/* file is updated only by replacing it, no need to flush
+		   attribute cache */
+		nfs_flush_file_handle_cache(mk->path);
+	}
+
+	if (nfs_safe_stat(mk->path, &st) < 0) {
+		if (errno == ENOENT) {
+			maildir_keywords_clear(mk);
+			mk->synced = TRUE;
+			return 0;
+		}
+                mailbox_set_critical(&mk->mbox->box,
+				     "stat(%s) failed: %m", mk->path);
+		return -1;
+	}
+
+	if (st.st_mtime == mk->synced_mtime) {
+		/* hasn't changed */
+		mk->synced = TRUE;
+		return 0;
+	}
+	mk->synced_mtime = st.st_mtime;
+
+	fd = open(mk->path, O_RDONLY);
+	if (fd == -1) {
+		if (errno == ENOENT) {
+			maildir_keywords_clear(mk);
+			mk->synced = TRUE;
+			return 0;
+		}
+                mailbox_set_critical(&mk->mbox->box,
+				     "open(%s) failed: %m", mk->path);
+		return -1;
+	}
+
+	maildir_keywords_clear(mk);
+	input = i_stream_create_fd(fd, 1024);
+	while ((line = i_stream_read_next_line(input)) != NULL) {
+		p = strchr(line, ' ');
+		if (p == NULL) {
+			/* note that when converting .customflags file this
+			   case happens in the first line. */
+			continue;
+		}
+		*p++ = '\0';
+
+		if (str_to_uint(line, &idx) < 0 ||
+		    idx >= MAILDIR_MAX_KEYWORDS || *p == '\0' ||
+		    hash_table_lookup(mk->hash, p) != NULL) {
+			/* shouldn't happen */
+			continue;
+		}
+
+		/* save it */
+		new_name = p_strdup(mk->pool, p);
+		hash_table_insert(mk->hash, new_name, POINTER_CAST(idx + 1));
+
+		strp = array_idx_get_space(&mk->list, idx);
+		*strp = new_name;
+	}
+	i_stream_destroy(&input);
+
+	if (close(fd) < 0) {
+                mailbox_set_critical(&mk->mbox->box,
+				     "close(%s) failed: %m", mk->path);
+		return -1;
+	}
+
+	mk->synced = TRUE;
+	return 0;
+}
+
+static int
+maildir_keywords_lookup(struct maildir_keywords *mk, const char *name,
+			unsigned int *chridx_r)
+{
+	void *value;
+
+	value = hash_table_lookup(mk->hash, name);
+	if (value == NULL) {
+		if (mk->synced)
+			return 0;
+
+		if (maildir_keywords_sync(mk) < 0)
+			return -1;
+		i_assert(mk->synced);
+
+		value = hash_table_lookup(mk->hash, name);
+		if (value == NULL)
+			return 0;
+	}
+
+	*chridx_r = POINTER_CAST_TO(value, unsigned int)-1;
+	return 1;
+}
+
+static void
+maildir_keywords_create(struct maildir_keywords *mk, const char *name,
+			unsigned int chridx)
+{
+	const char **strp;
+	char *new_name;
+
+	i_assert(chridx < MAILDIR_MAX_KEYWORDS);
+
+	new_name = p_strdup(mk->pool, name);
+	hash_table_insert(mk->hash, new_name, POINTER_CAST(chridx + 1));
+
+	strp = array_idx_get_space(&mk->list, chridx);
+	*strp = new_name;
+
+	mk->changed = TRUE;
+}
+
+static int
+maildir_keywords_lookup_or_create(struct maildir_keywords *mk, const char *name,
+				  unsigned int *chridx_r)
+{
+	const char *const *keywords;
+	unsigned int i, count;
+	int ret;
+
+	if ((ret = maildir_keywords_lookup(mk, name, chridx_r)) != 0)
+		return ret;
+
+	/* see if we are full */
+	keywords = array_get(&mk->list, &count);
+	for (i = 0; i < count; i++) {
+		if (keywords[i] == NULL)
+			break;
+	}
+
+	if (i == count && count >= MAILDIR_MAX_KEYWORDS)
+		return -1;
+
+	if (!maildir_uidlist_is_locked(mk->mbox->uidlist))
+		return -1;
+
+	maildir_keywords_create(mk, name, i);
+	*chridx_r = i;
+	return 1;
+}
+
+static const char *
+maildir_keywords_idx(struct maildir_keywords *mk, unsigned int idx)
+{
+	const char *const *keywords;
+	unsigned int count;
+
+	keywords = array_get(&mk->list, &count);
+	if (idx >= count) {
+		if (mk->synced)
+			return NULL;
+
+		if (maildir_keywords_sync(mk) < 0)
+			return NULL;
+		i_assert(mk->synced);
+
+		keywords = array_get(&mk->list, &count);
+	}
+	return idx >= count ? NULL : keywords[idx];
+}
+
+static int maildir_keywords_write_fd(struct maildir_keywords *mk,
+				     const char *path, int fd)
+{
+	struct maildir_mailbox *mbox = mk->mbox;
+	struct mailbox *box = &mbox->box;
+	const struct mailbox_permissions *perm = mailbox_get_permissions(box);
+	const char *const *keywords;
+	unsigned int i, count;
+	string_t *str;
+	struct stat st;
+
+	str = t_str_new(256);
+	keywords = array_get(&mk->list, &count);
+	for (i = 0; i < count; i++) {
+		if (keywords[i] != NULL)
+			str_printfa(str, "%u %s\n", i, keywords[i]);
+	}
+	if (write_full(fd, str_data(str), str_len(str)) < 0) {
+		mailbox_set_critical(&mk->mbox->box,
+				     "write_full(%s) failed: %m", path);
+		return -1;
+	}
+
+	if (fstat(fd, &st) < 0) {
+		mailbox_set_critical(&mk->mbox->box,
+				     "fstat(%s) failed: %m", path);
+		return -1;
+	}
+
+	if (st.st_gid != perm->file_create_gid &&
+	    perm->file_create_gid != (gid_t)-1) {
+		if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) {
+			if (errno == EPERM) {
+				mailbox_set_critical(&mk->mbox->box, "%s",
+					eperm_error_get_chgrp("fchown", path,
+						perm->file_create_gid,
+						perm->file_create_gid_origin));
+			} else {
+				mailbox_set_critical(&mk->mbox->box,
+					"fchown(%s) failed: %m", path);
+			}
+		}
+	}
+
+	/* mtime must grow every time */
+	if (st.st_mtime <= mk->synced_mtime) {
+		struct utimbuf ut;
+
+		mk->synced_mtime = ioloop_time <= mk->synced_mtime ?
+			mk->synced_mtime + 1 : ioloop_time;
+		ut.actime = ioloop_time;
+		ut.modtime = mk->synced_mtime;
+		if (utime(path, &ut) < 0) {
+			mailbox_set_critical(&mk->mbox->box,
+				"utime(%s) failed: %m", path);
+			return -1;
+		}
+	}
+
+	if (fsync(fd) < 0) {
+		mailbox_set_critical(&mk->mbox->box,
+			"fsync(%s) failed: %m", path);
+		return -1;
+	}
+	return 0;
+}
+
+static int maildir_keywords_commit(struct maildir_keywords *mk)
+{
+	const struct mailbox_permissions *perm;
+	struct dotlock *dotlock;
+	const char *lock_path;
+	mode_t old_mask;
+	int i, fd;
+
+	mk->synced = FALSE;
+
+	if (!mk->changed || mk->mbox == NULL)
+		return 0;
+
+	lock_path = t_strconcat(mk->path, ".lock", NULL);
+	i_unlink_if_exists(lock_path);
+
+	perm = mailbox_get_permissions(&mk->mbox->box);
+	for (i = 0;; i++) {
+		/* we could just create the temp file directly, but doing it
+		   this ways avoids potential problems with overwriting
+		   contents in malicious symlinks */
+		old_mask = umask(0777 & ~perm->file_create_mode);
+		fd = file_dotlock_open(&mk->dotlock_settings, mk->path,
+				       DOTLOCK_CREATE_FLAG_NONBLOCK, &dotlock);
+		umask(old_mask);
+		if (fd != -1)
+			break;
+
+		if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) {
+			mailbox_set_critical(&mk->mbox->box,
+				"file_dotlock_open(%s) failed: %m", mk->path);
+			return -1;
+		}
+		/* the control dir doesn't exist. create it unless the whole
+		   mailbox was just deleted. */
+		if (!maildir_set_deleted(&mk->mbox->box))
+			return -1;
+	}
+
+	if (maildir_keywords_write_fd(mk, lock_path, fd) < 0) {
+		file_dotlock_delete(&dotlock);
+		return -1;
+	}
+
+	if (file_dotlock_replace(&dotlock, 0) < 0) {
+		mailbox_set_critical(&mk->mbox->box,
+			"file_dotlock_replace(%s) failed: %m", mk->path);
+		return -1;
+	}
+
+	mk->changed = FALSE;
+	return 0;
+}
+
+struct maildir_keywords_sync_ctx *
+maildir_keywords_sync_init(struct maildir_keywords *mk,
+			   struct mail_index *index)
+{
+	struct maildir_keywords_sync_ctx *ctx;
+
+	ctx = i_new(struct maildir_keywords_sync_ctx, 1);
+	ctx->mk = mk;
+	ctx->index = index;
+	ctx->keywords = mail_index_get_keywords(index);
+	i_array_init(&ctx->idx_to_chr, MAILDIR_MAX_KEYWORDS);
+	return ctx;
+}
+
+struct maildir_keywords_sync_ctx *
+maildir_keywords_sync_init_readonly(struct maildir_keywords *mk,
+				    struct mail_index *index)
+{
+	struct maildir_keywords_sync_ctx *ctx;
+
+	ctx = maildir_keywords_sync_init(mk, index);
+	ctx->readonly = TRUE;
+	return ctx;
+}
+
+void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **_ctx)
+{
+	struct maildir_keywords_sync_ctx *ctx = *_ctx;
+
+	*_ctx = NULL;
+
+	T_BEGIN {
+		(void)maildir_keywords_commit(ctx->mk);
+	} T_END;
+
+	array_free(&ctx->idx_to_chr);
+	i_free(ctx);
+}
+
+unsigned int maildir_keywords_char_idx(struct maildir_keywords_sync_ctx *ctx,
+				       char keyword)
+{
+	const char *name;
+	unsigned int chridx, idx;
+
+	i_assert(keyword >= MAILDIR_KEYWORD_FIRST &&
+		 keyword <= MAILDIR_KEYWORD_LAST);
+	chridx = keyword - MAILDIR_KEYWORD_FIRST;
+
+	if (ctx->chridx_to_idx[chridx] != 0)
+		return ctx->chridx_to_idx[chridx];
+
+	/* lookup / create */
+	name = maildir_keywords_idx(ctx->mk, chridx);
+	if (name == NULL) {
+		/* name is lost. just generate one ourself. */
+		name = t_strdup_printf("unknown-%u", chridx);
+		while (maildir_keywords_lookup(ctx->mk, name, &idx) > 0) {
+			/* don't create a duplicate name.
+			   keep changing the name until it doesn't exist */
+			name = t_strconcat(name, "?", NULL);
+		}
+                maildir_keywords_create(ctx->mk, name, chridx);
+	}
+
+	mail_index_keyword_lookup_or_create(ctx->index, name, &idx);
+        ctx->chridx_to_idx[chridx] = idx;
+	return idx;
+}
+
+char maildir_keywords_idx_char(struct maildir_keywords_sync_ctx *ctx,
+			       unsigned int idx)
+{
+	const char *name;
+	char *chr_p;
+	unsigned int chridx;
+	int ret;
+
+	chr_p = array_idx_get_space(&ctx->idx_to_chr, idx);
+	if (*chr_p != '\0')
+		return *chr_p;
+
+	name = array_idx_elem(ctx->keywords, idx);
+	ret = !ctx->readonly ?
+		maildir_keywords_lookup_or_create(ctx->mk, name, &chridx) :
+		maildir_keywords_lookup(ctx->mk, name, &chridx);
+	if (ret <= 0)
+		return '\0';
+
+	*chr_p = chridx + MAILDIR_KEYWORD_FIRST;
+	return *chr_p;
+}
diff --git a/src/lib-storage/index/maildir/maildir-keywords.h b/src/lib-storage/index/maildir/maildir-keywords.h
new file mode 100644
index 0000000..43e47d7
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-keywords.h
@@ -0,0 +1,36 @@
+#ifndef MAILDIR_KEYWORDS_H
+#define MAILDIR_KEYWORDS_H
+
+#define MAILDIR_KEYWORDS_NAME "dovecot-keywords"
+
+struct maildir_mailbox;
+struct maildir_keywords;
+struct maildir_keywords_sync_ctx;
+
+struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox);
+void maildir_keywords_deinit(struct maildir_keywords **mk);
+
+/* Initialize a read-only maildir_keywords instance. Mailbox needs to contain
+   the dovecot-keywords file, but otherwise it doesn't have to be in maildir
+   format. */
+struct maildir_keywords *
+maildir_keywords_init_readonly(struct mailbox *box);
+
+struct maildir_keywords_sync_ctx *
+maildir_keywords_sync_init(struct maildir_keywords *mk,
+			   struct mail_index *index);
+/* Don't try to add any nonexistent keywords */
+struct maildir_keywords_sync_ctx *
+maildir_keywords_sync_init_readonly(struct maildir_keywords *mk,
+				    struct mail_index *index);
+void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **ctx);
+
+/* Returns keyword index. */
+unsigned int maildir_keywords_char_idx(struct maildir_keywords_sync_ctx *ctx,
+				       char keyword);
+/* Returns keyword character for given index, or \0 if keyword couldn't be
+   added. */
+char maildir_keywords_idx_char(struct maildir_keywords_sync_ctx *ctx,
+			       unsigned int idx);
+
+#endif
diff --git a/src/lib-storage/index/maildir/maildir-mail.c b/src/lib-storage/index/maildir/maildir-mail.c
new file mode 100644
index 0000000..c3abbd3
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-mail.c
@@ -0,0 +1,809 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "nfs-workarounds.h"
+#include "index-mail.h"
+#include "maildir-storage.h"
+#include "maildir-filename.h"
+#include "maildir-uidlist.h"
+#include "maildir-sync.h"
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+struct maildir_open_context {
+	int fd;
+	char *path;
+};
+
+static int
+do_open(struct maildir_mailbox *mbox, const char *path,
+	struct maildir_open_context *ctx)
+{
+	ctx->fd = nfs_safe_open(path, O_RDONLY);
+	if (ctx->fd != -1) {
+		ctx->path = i_strdup(path);
+		return 1;
+	}
+	if (errno == ENOENT)
+		return 0;
+
+	if (errno == EACCES) {
+		mailbox_set_critical(&mbox->box, "%s",
+			mail_error_eacces_msg("open", path));
+	} else {
+		mailbox_set_critical(&mbox->box,
+				     "open(%s) failed: %m", path);
+	}
+	return -1;
+}
+
+static int
+do_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st)
+{
+	if (stat(path, st) == 0)
+		return 1;
+	if (errno == ENOENT)
+		return 0;
+
+	if (errno == EACCES) {
+		mailbox_set_critical(&mbox->box, "%s",
+			mail_error_eacces_msg("stat", path));
+	} else {
+		mailbox_set_critical(&mbox->box, "stat(%s) failed: %m", path);
+	}
+	return -1;
+}
+
+static struct istream *
+maildir_open_mail(struct maildir_mailbox *mbox, struct mail *mail,
+		  bool *deleted_r)
+{
+	struct istream *input;
+	const char *path;
+	struct maildir_open_context ctx;
+
+	*deleted_r = FALSE;
+
+	if (!mail_stream_access_start(mail))
+		return NULL;
+
+	ctx.fd = -1;
+	ctx.path = NULL;
+
+	mail->transaction->stats.open_lookup_count++;
+	if (!mail->saving) {
+		if (maildir_file_do(mbox, mail->uid, do_open, &ctx) < 0)
+			return NULL;
+	} else {
+		path = maildir_save_file_get_path(mail->transaction, mail->seq);
+		if (do_open(mbox, path, &ctx) <= 0)
+			return NULL;
+	}
+
+	if (ctx.fd == -1) {
+		*deleted_r = TRUE;
+		return NULL;
+	}
+
+	input = i_stream_create_fd_autoclose(&ctx.fd, 0);
+	if (input->stream_errno == EISDIR) {
+		i_stream_destroy(&input);
+		if (maildir_lose_unexpected_dir(&mbox->storage->storage,
+						ctx.path) >= 0)
+			*deleted_r = TRUE;
+	} else {
+		i_stream_set_name(input, ctx.path);
+		index_mail_set_read_buffer_size(mail, input);
+	}
+	i_free(ctx.path);
+	return input;
+}
+
+static int maildir_mail_stat(struct mail *mail, struct stat *st_r)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box);
+	struct index_mail *imail = INDEX_MAIL(mail);
+	const char *path;
+	int fd, ret;
+
+	if (!mail_metadata_access_start(mail))
+		return -1;
+
+	if (imail->data.access_part != 0 &&
+	    imail->data.stream == NULL) {
+		/* we're going to open the mail anyway */
+		struct istream *input;
+
+		(void)mail_get_stream(mail, NULL, NULL, &input);
+	}
+
+	if (imail->data.stream != NULL &&
+	    (fd = i_stream_get_fd(imail->data.stream)) != -1) {
+		mail->transaction->stats.fstat_lookup_count++;
+		if (fstat(fd, st_r) < 0) {
+			mail_set_critical(mail, "fstat(%s) failed: %m",
+				i_stream_get_name(imail->data.stream));
+			return -1;
+		}
+	} else if (!mail->saving) {
+		mail->transaction->stats.stat_lookup_count++;
+		ret = maildir_file_do(mbox, mail->uid, do_stat, st_r);
+		if (ret <= 0) {
+			if (ret == 0)
+				mail_set_expunged(mail);
+			return -1;
+		}
+	} else {
+		mail->transaction->stats.stat_lookup_count++;
+		path = maildir_save_file_get_path(mail->transaction, mail->seq);
+		if (stat(path, st_r) < 0) {
+			mail_set_critical(mail, "stat(%s) failed: %m", path);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static int maildir_mail_get_received_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct stat st;
+
+	if (index_mail_get_received_date(_mail, date_r) == 0)
+		return 0;
+
+	if (maildir_mail_stat(_mail, &st) < 0)
+		return -1;
+
+	*date_r = data->received_date = st.st_mtime;
+	return 0;
+}
+
+static int maildir_mail_get_save_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct stat st;
+
+	if (index_mail_get_save_date(_mail, date_r) > 0)
+		return 1;
+
+	if (maildir_mail_stat(_mail, &st) < 0)
+		return -1;
+
+	*date_r = data->save_date = st.st_ctime;
+	return 1;
+}
+
+static int
+maildir_mail_get_fname(struct maildir_mailbox *mbox, struct mail *mail,
+		       const char **fname_r)
+{
+	enum maildir_uidlist_rec_flag flags;
+	struct mail_index_view *view;
+	uint32_t seq;
+	bool exists;
+	int ret;
+
+	ret = maildir_sync_lookup(mbox, mail->uid, &flags, fname_r);
+	if (ret != 0)
+		return ret;
+
+	/* file exists in index file, but not in dovecot-uidlist anymore. */
+	mail_set_expunged(mail);
+
+	/* one reason this could happen is if we delayed opening
+	   dovecot-uidlist and we're trying to open a mail that got recently
+	   expunged. Let's test this theory first: */
+	mail_index_refresh(mbox->box.index);
+	view = mail_index_view_open(mbox->box.index);
+	exists = mail_index_lookup_seq(view, mail->uid, &seq);
+	mail_index_view_close(&view);
+
+	if (exists) {
+		/* the message still exists in index. this means there's some
+		   kind of a desync, which doesn't get fixed if cur/ mtime is
+		   the same as in index. fix this by forcing a resync. */
+		(void)maildir_storage_sync_force(mbox, mail->uid);
+	}
+	return 0;
+}
+
+static int maildir_get_pop3_state(struct index_mail *mail)
+{
+	struct mailbox *box = mail->mail.mail.box;
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	const struct mail_cache_field *fields;
+	unsigned int i, count, psize_idx, vsize_idx;
+	enum mail_cache_decision_type dec, vsize_dec;
+	enum mail_fetch_field allowed_pop3_fields;
+	bool not_pop3_only = FALSE;
+
+	if (mail->pop3_state_set)
+		return mail->pop3_state;
+
+	/* if this mail itself has non-pop3 fields we know we're not
+	   pop3-only */
+	allowed_pop3_fields = MAIL_FETCH_FLAGS | MAIL_FETCH_STREAM_HEADER |
+		MAIL_FETCH_STREAM_BODY | MAIL_FETCH_STORAGE_ID |
+		MAIL_FETCH_VIRTUAL_SIZE;
+
+	if (mail->data.wanted_headers != NULL ||
+	    (mail->data.wanted_fields & ENUM_NEGATE(allowed_pop3_fields)) != 0)
+		not_pop3_only = TRUE;
+
+	/* get vsize decisions */
+	psize_idx = ibox->cache_fields[MAIL_CACHE_PHYSICAL_FULL_SIZE].idx;
+	vsize_idx = ibox->cache_fields[MAIL_CACHE_VIRTUAL_FULL_SIZE].idx;
+	if (not_pop3_only) {
+		vsize_dec = mail_cache_field_get_decision(box->cache,
+							  vsize_idx);
+		vsize_dec &= ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED);
+	} else {
+		/* also check if there are any non-[pv]size cached fields */
+		vsize_dec = MAIL_CACHE_DECISION_NO;
+		fields = mail_cache_register_get_list(box->cache,
+						      pool_datastack_create(),
+						      &count);
+		for (i = 0; i < count; i++) {
+			dec = fields[i].decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED);
+			if (fields[i].idx == vsize_idx)
+				vsize_dec = dec;
+			else if (dec != MAIL_CACHE_DECISION_NO &&
+				 fields[i].idx != psize_idx)
+				not_pop3_only = TRUE;
+		}
+	}
+
+	if (index_mail_get_vsize_extension(&mail->mail.mail) != NULL) {
+		/* having a vsize extension in index is the same as having
+		   vsize's caching decision YES */
+		vsize_dec = MAIL_CACHE_DECISION_YES;
+	}
+
+	if (!not_pop3_only) {
+		/* either nothing is cached, or only vsize is cached. */
+		mail->pop3_state = 1;
+	} else if (vsize_dec != MAIL_CACHE_DECISION_YES &&
+		   (box->flags & MAILBOX_FLAG_POP3_SESSION) == 0) {
+		/* if virtual size isn't cached permanently,
+		   POP3 isn't being used */
+		mail->pop3_state = -1;
+	} else {
+		/* possibly a mixed pop3/imap */
+		mail->pop3_state = 0;
+	}
+	mail->pop3_state_set = TRUE;
+	return mail->pop3_state;
+}
+
+static int maildir_quick_size_lookup(struct index_mail *mail, bool vsize,
+				     uoff_t *size_r)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box);
+	enum maildir_uidlist_rec_ext_key key;
+	const char *path, *fname, *value;
+
+	if (!_mail->saving) {
+		if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0)
+			return -1;
+	} else {
+		if (maildir_save_file_get_size(_mail->transaction, _mail->seq,
+					       vsize, size_r) == 0)
+			return 1;
+
+		path = maildir_save_file_get_path(_mail->transaction,
+						  _mail->seq);
+		fname = strrchr(path, '/');
+		fname = fname != NULL ? fname + 1 : path;
+	}
+
+	/* size can be included in filename */
+	if (vsize || !mbox->storage->set->maildir_broken_filename_sizes) {
+		if (maildir_filename_get_size(fname,
+				vsize ? MAILDIR_EXTRA_VIRTUAL_SIZE :
+					MAILDIR_EXTRA_FILE_SIZE, size_r))
+			return 1;
+	}
+
+	/* size can be included in uidlist entry */
+	if (!_mail->saving) {
+		key = vsize ? MAILDIR_UIDLIST_REC_EXT_VSIZE :
+			MAILDIR_UIDLIST_REC_EXT_PSIZE;
+		value = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid,
+						   key);
+		if (value != NULL && str_to_uoff(value, size_r) == 0)
+			return 1;
+	}
+	return 0;
+}
+
+static void
+maildir_handle_size_caching(struct index_mail *mail, bool quick_check,
+			    bool vsize)
+{
+	struct mailbox *box = mail->mail.mail.box;
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+	enum mail_fetch_field field;
+	uoff_t size;
+	int pop3_state;
+
+	field = vsize ? MAIL_FETCH_VIRTUAL_SIZE : MAIL_FETCH_PHYSICAL_SIZE;
+	if ((mail->data.dont_cache_fetch_fields & field) != 0)
+		return;
+
+	if (quick_check && maildir_quick_size_lookup(mail, vsize, &size) > 0) {
+		/* already in filename / uidlist. don't add it anywhere,
+		   including to the uidlist if it's already in filename.
+		   do some extra checks here to catch potential cache bugs. */
+		if (vsize && mail->data.virtual_size != size) {
+			mail_set_mail_cache_corrupted(&mail->mail.mail,
+				"Corrupted virtual size: "
+				"%"PRIuUOFF_T" != %"PRIuUOFF_T,
+				mail->data.virtual_size, size);
+			mail->data.virtual_size = size;
+		} else if (!vsize && mail->data.physical_size != size) {
+			mail_set_mail_cache_corrupted(&mail->mail.mail,
+				"Corrupted physical size: "
+				"%"PRIuUOFF_T" != %"PRIuUOFF_T,
+				mail->data.physical_size, size);
+			mail->data.physical_size = size;
+		}
+		mail->data.dont_cache_fetch_fields |= field;
+		return;
+	}
+
+	/* 1 = pop3-only, 0 = mixed, -1 = no pop3 */
+	pop3_state = maildir_get_pop3_state(mail);
+	if (pop3_state >= 0 && mail->mail.mail.uid != 0) {
+		/* if size is wanted permanently, store it to uidlist
+		   so that in case cache file gets lost we can get it quickly */
+		mail->data.dont_cache_fetch_fields |= field;
+		size = vsize ? mail->data.virtual_size :
+			mail->data.physical_size;
+		maildir_uidlist_set_ext(mbox->uidlist, mail->mail.mail.uid,
+					vsize ? MAILDIR_UIDLIST_REC_EXT_VSIZE :
+					MAILDIR_UIDLIST_REC_EXT_PSIZE,
+					dec2str(size));
+	}
+}
+
+static int maildir_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box);
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct message_size hdr_size, body_size;
+	struct istream *input;
+	uoff_t old_offset;
+
+	if (maildir_uidlist_is_read(mbox->uidlist) ||
+	    (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) {
+		/* try to get the size from uidlist. this is especially useful
+		   with pop3 to avoid unnecessarily opening the cache file. */
+		if (maildir_quick_size_lookup(mail, TRUE,
+					      &data->virtual_size) < 0)
+			return -1;
+	}
+
+	if (data->virtual_size == UOFF_T_MAX) {
+		if (index_mail_get_cached_virtual_size(mail, size_r)) {
+			i_assert(mail->data.virtual_size != UOFF_T_MAX);
+			maildir_handle_size_caching(mail, TRUE, TRUE);
+			return 0;
+		}
+		if (maildir_quick_size_lookup(mail, TRUE,
+					      &data->virtual_size) < 0)
+			return -1;
+	}
+	if (data->virtual_size != UOFF_T_MAX) {
+		data->dont_cache_fetch_fields |= MAIL_FETCH_VIRTUAL_SIZE;
+		*size_r = data->virtual_size;
+		return 0;
+	}
+
+	/* fallback to reading the file */
+	old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
+	if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0)
+		return -1;
+	i_stream_seek(data->stream, old_offset);
+
+	maildir_handle_size_caching(mail, FALSE, TRUE);
+	*size_r = data->virtual_size;
+	return 0;
+}
+
+static int maildir_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box);
+	struct index_mail_data *data = &mail->data;
+	struct stat st;
+	struct message_size hdr_size, body_size;
+	struct istream *input;
+	const char *path;
+	int ret;
+
+	if (maildir_uidlist_is_read(mbox->uidlist) ||
+	    (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) {
+		/* try to get the size from uidlist (see virtual size above) */
+		if (maildir_quick_size_lookup(mail, FALSE,
+					      &data->physical_size) < 0)
+			return -1;
+	}
+
+	if (data->physical_size == UOFF_T_MAX) {
+		if (index_mail_get_physical_size(_mail, size_r) == 0) {
+			i_assert(mail->data.physical_size != UOFF_T_MAX);
+			maildir_handle_size_caching(mail, TRUE, FALSE);
+			return 0;
+		}
+		if (maildir_quick_size_lookup(mail, FALSE,
+					      &data->physical_size) < 0)
+			return -1;
+	}
+	if (data->physical_size != UOFF_T_MAX) {
+		data->dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
+		*size_r = data->physical_size;
+		return 0;
+	}
+
+	if (mail->mail.v.istream_opened != NULL) {
+		/* we can't use stat(), because this may be a mail that some
+		   plugin has changed (e.g. zlib). need to do it the slow
+		   way. */
+		if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0)
+			return -1;
+		st.st_size = hdr_size.physical_size + body_size.physical_size;
+	} else if (!_mail->saving) {
+		ret = maildir_file_do(mbox, _mail->uid, do_stat, &st);
+		if (ret <= 0) {
+			if (ret == 0)
+				mail_set_expunged(_mail);
+			return -1;
+		}
+	} else {
+		/* saved mail which hasn't been committed yet */
+		path = maildir_save_file_get_path(_mail->transaction,
+						  _mail->seq);
+		if (stat(path, &st) < 0) {
+			mail_set_critical(_mail, "stat(%s) failed: %m", path);
+			return -1;
+		}
+	}
+
+	data->physical_size = st.st_size;
+	maildir_handle_size_caching(mail, FALSE, FALSE);
+	*size_r = st.st_size;
+	return 0;
+}
+
+static int
+maildir_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+			 const char **value_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box);
+	const char *path, *fname = NULL, *end, *guid, *uidl, *order;
+	struct stat st;
+
+	switch (field) {
+	case MAIL_FETCH_GUID:
+		/* use GUID from uidlist if it exists */
+		i_assert(!_mail->saving);
+
+		if (mail->data.guid != NULL) {
+			*value_r = mail->data.guid;
+			return 0;
+		}
+
+		/* first make sure that we have a refreshed uidlist */
+		if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0)
+			return -1;
+
+		guid = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid,
+						  MAILDIR_UIDLIST_REC_EXT_GUID);
+		if (guid != NULL) {
+			if (*guid != '\0') {
+				*value_r = mail->data.guid =
+					p_strdup(mail->mail.data_pool, guid);
+				return 0;
+			}
+
+			mail_set_critical(_mail,
+				"Maildir: Corrupted dovecot-uidlist: "
+				"UID had empty GUID, clearing it");
+			maildir_uidlist_unset_ext(mbox->uidlist, _mail->uid,
+				MAILDIR_UIDLIST_REC_EXT_GUID);
+		}
+
+		/* default to base filename: */
+		if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID,
+					     value_r) < 0)
+			return -1;
+		mail->data.guid = mail->data.filename;
+		return 0;
+	case MAIL_FETCH_STORAGE_ID:
+		if (mail->data.filename != NULL) {
+			*value_r = mail->data.filename;
+			return 0;
+		}
+		if (fname != NULL) {
+			/* we came here from MAIL_FETCH_GUID,
+			   avoid a second lookup */
+		} else if (!_mail->saving) {
+			if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0)
+				return -1;
+		} else {
+			path = maildir_save_file_get_path(_mail->transaction,
+							  _mail->seq);
+			fname = strrchr(path, '/');
+			fname = fname != NULL ? fname + 1 : path;
+		}
+		end = strchr(fname, MAILDIR_INFO_SEP);
+		mail->data.filename = end == NULL ?
+			p_strdup(mail->mail.data_pool, fname) :
+			p_strdup_until(mail->mail.data_pool, fname, end);
+		*value_r = mail->data.filename;
+		return 0;
+	case MAIL_FETCH_UIDL_BACKEND:
+		uidl = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid,
+					MAILDIR_UIDLIST_REC_EXT_POP3_UIDL);
+		if (uidl == NULL) {
+			/* use the default */
+			*value_r = "";
+		} else if (*uidl == '\0') {
+			/* special optimization case: use the base file name */
+			return maildir_mail_get_special(_mail,
+					MAIL_FETCH_STORAGE_ID, value_r);
+		} else {
+			*value_r = p_strdup(mail->mail.data_pool, uidl);
+		}
+		return 0;
+	case MAIL_FETCH_POP3_ORDER:
+		order = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid,
+					MAILDIR_UIDLIST_REC_EXT_POP3_ORDER);
+		if (order == NULL) {
+			*value_r = "";
+		} else {
+			*value_r = p_strdup(mail->mail.data_pool, order);
+		}
+		return 0;
+	case MAIL_FETCH_REFCOUNT:
+		if (maildir_mail_stat(_mail, &st) < 0)
+			return -1;
+		*value_r = p_strdup_printf(mail->mail.data_pool, "%lu",
+					   (unsigned long)st.st_nlink);
+		return 0;
+	case MAIL_FETCH_REFCOUNT_ID:
+		if (maildir_mail_stat(_mail, &st) < 0)
+			return -1;
+		*value_r = p_strdup_printf(mail->mail.data_pool, "%llu",
+					   (unsigned long long)st.st_ino);
+		return 0;
+	default:
+		return index_mail_get_special(_mail, field, value_r);
+	}
+}
+							
+static int
+maildir_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
+			struct message_size *hdr_size,
+			struct message_size *body_size,
+			struct istream **stream_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box);
+	struct index_mail_data *data = &mail->data;
+	bool deleted;
+
+	if (data->stream == NULL) {
+		data->stream = maildir_open_mail(mbox, _mail, &deleted);
+		if (data->stream == NULL) {
+			if (deleted)
+				mail_set_expunged(_mail);
+			return -1;
+		}
+		if (mail->mail.v.istream_opened != NULL) {
+			if (mail->mail.v.istream_opened(_mail,
+							&data->stream) < 0) {
+				i_stream_unref(&data->stream);
+				return -1;
+			}
+		}
+	}
+
+	return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
+}
+
+static void maildir_update_pop3_uidl(struct mail *_mail, const char *uidl)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(_mail->box);
+	const char *fname;
+
+	if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID,
+				     &fname) == 0 &&
+	    strcmp(uidl, fname) == 0) {
+		/* special case optimization: empty UIDL means the same
+		   as base filename */
+		uidl = "";
+	}
+
+	maildir_uidlist_set_ext(mbox->uidlist, _mail->uid,
+				MAILDIR_UIDLIST_REC_EXT_POP3_UIDL, uidl);
+}
+
+static void maildir_mail_remove_sizes_from_uidlist(struct mail *mail)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box);
+
+	if (maildir_uidlist_lookup_ext(mbox->uidlist, mail->uid,
+				       MAILDIR_UIDLIST_REC_EXT_VSIZE) != NULL) {
+		maildir_uidlist_unset_ext(mbox->uidlist, mail->uid,
+					  MAILDIR_UIDLIST_REC_EXT_VSIZE);
+	}
+	if (maildir_uidlist_lookup_ext(mbox->uidlist, mail->uid,
+				       MAILDIR_UIDLIST_REC_EXT_PSIZE) != NULL) {
+		maildir_uidlist_unset_ext(mbox->uidlist, mail->uid,
+					  MAILDIR_UIDLIST_REC_EXT_PSIZE);
+	}
+}
+
+struct maildir_size_fix_ctx {
+	uoff_t physical_size;
+	char wrong_key;
+};
+
+static int
+do_fix_size(struct maildir_mailbox *mbox, const char *path,
+	    struct maildir_size_fix_ctx *ctx)
+{
+	const char *fname, *newpath, *extra, *info, *dir;
+	struct stat st;
+
+	fname = strrchr(path, '/');
+	i_assert(fname != NULL);
+	dir = t_strdup_until(path, fname++);
+
+	extra = strchr(fname, MAILDIR_EXTRA_SEP);
+	i_assert(extra != NULL);
+	info = strchr(fname, MAILDIR_INFO_SEP);
+	if (info == NULL) info = "";
+
+	if (ctx->physical_size == UOFF_T_MAX) {
+		if (stat(path, &st) < 0) {
+			if (errno == ENOENT)
+				return 0;
+			mailbox_set_critical(&mbox->box,
+					     "stat(%s) failed: %m", path);
+			return -1;
+		}
+		ctx->physical_size = st.st_size;
+	}
+
+	newpath = t_strdup_printf("%s/%s,S=%"PRIuUOFF_T"%s", dir,
+				  t_strdup_until(fname, extra),
+				  ctx->physical_size, info);
+
+	if (rename(path, newpath) == 0) {
+		mailbox_set_critical(&mbox->box,
+			"Maildir filename has wrong %c value, "
+			"renamed the file from %s to %s",
+			ctx->wrong_key, path, newpath);
+		return 1;
+	}
+	if (errno == ENOENT)
+		return 0;
+
+	mailbox_set_critical(&mbox->box, "rename(%s, %s) failed: %m",
+			     path, newpath);
+	return -1;
+}
+
+static void
+maildir_mail_remove_sizes_from_filename(struct mail *mail,
+					enum mail_fetch_field field)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(mail->box);
+	struct mail_private *pmail = (struct mail_private *)mail;
+	enum maildir_uidlist_rec_flag flags;
+	const char *fname;
+	uoff_t size;
+	struct maildir_size_fix_ctx ctx;
+
+	if (mbox->storage->set->maildir_broken_filename_sizes) {
+		/* never try to fix sizes in maildir filenames */
+		return;
+	}
+
+	if (maildir_sync_lookup(mbox, mail->uid, &flags, &fname) <= 0)
+		return;
+	if (strchr(fname, MAILDIR_EXTRA_SEP) == NULL)
+		return;
+
+	i_zero(&ctx);
+	ctx.physical_size = UOFF_T_MAX;
+	if (field == MAIL_FETCH_VIRTUAL_SIZE &&
+	    maildir_filename_get_size(fname, MAILDIR_EXTRA_VIRTUAL_SIZE,
+				      &size)) {
+		ctx.wrong_key = 'W';
+	} else if (field == MAIL_FETCH_PHYSICAL_SIZE &&
+		   maildir_filename_get_size(fname, MAILDIR_EXTRA_FILE_SIZE,
+					     &size)) {
+		ctx.wrong_key = 'S';
+	} else {
+		/* the broken size isn't in filename */
+		return;
+	}
+
+	if (pmail->v.istream_opened != NULL) {
+		/* the mail could be e.g. compressed. get the physical size
+		   the slow way by actually reading the mail. */
+		struct istream *input;
+		const struct stat *stp;
+
+		if (mail_get_stream(mail, NULL, NULL, &input) < 0)
+			return;
+		if (i_stream_stat(input, TRUE, &stp) < 0)
+			return;
+		ctx.physical_size = stp->st_size;
+	}
+
+	(void)maildir_file_do(mbox, mail->uid, do_fix_size, &ctx);
+}
+
+static void maildir_mail_set_cache_corrupted(struct mail *_mail,
+					     enum mail_fetch_field field,
+					     const char *reason)
+{
+	if (field == MAIL_FETCH_PHYSICAL_SIZE ||
+	    field == MAIL_FETCH_VIRTUAL_SIZE) {
+		maildir_mail_remove_sizes_from_uidlist(_mail);
+		maildir_mail_remove_sizes_from_filename(_mail, field);
+	}
+	index_mail_set_cache_corrupted(_mail, field, reason);
+}
+
+struct mail_vfuncs maildir_mail_vfuncs = {
+	index_mail_close,
+	index_mail_free,
+	index_mail_set_seq,
+	index_mail_set_uid,
+	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
+	index_mail_precache,
+	index_mail_add_temp_wanted_fields,
+
+	index_mail_get_flags,
+	index_mail_get_keywords,
+	index_mail_get_keyword_indexes,
+	index_mail_get_modseq,
+	index_mail_get_pvt_modseq,
+	index_mail_get_parts,
+	index_mail_get_date,
+	maildir_mail_get_received_date,
+	maildir_mail_get_save_date,
+	maildir_mail_get_virtual_size,
+	maildir_mail_get_physical_size,
+	index_mail_get_first_header,
+	index_mail_get_headers,
+	index_mail_get_header_stream,
+	maildir_mail_get_stream,
+	index_mail_get_binary_stream,
+	maildir_mail_get_special,
+	index_mail_get_backend_mail,
+	index_mail_update_flags,
+	index_mail_update_keywords,
+	index_mail_update_modseq,
+	index_mail_update_pvt_modseq,
+	maildir_update_pop3_uidl,
+	index_mail_expunge,
+	maildir_mail_set_cache_corrupted,
+	index_mail_opened,
+};
diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c
new file mode 100644
index 0000000..5cf7d6a
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-save.c
@@ -0,0 +1,1084 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "buffer.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "ostream.h"
+#include "fdatasync-path.h"
+#include "eacces-error.h"
+#include "str.h"
+#include "index-mail.h"
+#include "maildir-storage.h"
+#include "maildir-uidlist.h"
+#include "maildir-keywords.h"
+#include "maildir-filename.h"
+#include "maildir-filename-flags.h"
+#include "maildir-sync.h"
+#include "mailbox-recent-flags.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <utime.h>
+#include <sys/stat.h>
+
+#define MAILDIR_FILENAME_FLAG_MOVED 0x10000000
+
+struct maildir_filename {
+	struct maildir_filename *next;
+	const char *tmp_name, *dest_basename;
+	const char *pop3_uidl, *guid;
+
+	uoff_t size, vsize;
+	enum mail_flags flags;
+	unsigned int pop3_order;
+	bool preserve_filename:1;
+	ARRAY_TYPE(keyword_indexes) keywords;
+};
+
+struct maildir_save_context {
+	struct mail_save_context ctx;
+	pool_t pool;
+
+	struct maildir_mailbox *mbox;
+	struct mail_index_transaction *trans;
+	struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
+	struct maildir_keywords_sync_ctx *keywords_sync_ctx;
+	struct maildir_index_sync_context *sync_ctx;
+	struct mail *cur_dest_mail;
+
+	const char *tmpdir, *newdir, *curdir;
+	struct maildir_filename *files, **files_tail, *file_last;
+	unsigned int files_count;
+
+	struct istream *input;
+	int fd;
+	uint32_t first_seq, seq, last_nonrecent_uid;
+
+	bool have_keywords:1;
+	bool have_preserved_filenames:1;
+	bool locked:1;
+	bool failed:1;
+	bool last_save_finished:1;
+	bool locked_uidlist_refresh:1;
+};
+
+#define MAILDIR_SAVECTX(s)	container_of(s, struct maildir_save_context, ctx)
+
+static int maildir_file_move(struct maildir_save_context *ctx,
+			     struct maildir_filename *mf, const char *destname,
+			     bool newdir)
+{
+	struct mail_storage *storage = &ctx->mbox->storage->storage;
+	const char *tmp_path, *new_path;
+
+	i_assert(*destname != '\0');
+	i_assert(*mf->tmp_name != '\0');
+
+	/* if we have flags, we'll move it to cur/ directly, because files in
+	   new/ directory can't have flags. alternative would be to write it
+	   in new/ and set the flags dirty in index file, but in that case
+	   external MUAs would see wrong flags. */
+	tmp_path = t_strconcat(ctx->tmpdir, "/", mf->tmp_name, NULL);
+	new_path = newdir ?
+		t_strconcat(ctx->newdir, "/", destname, NULL) :
+		t_strconcat(ctx->curdir, "/", destname, NULL);
+
+	/* maildir spec says we should use link() + unlink() here. however
+	   since our filename is guaranteed to be unique, rename() works just
+	   as well, except faster. even if the filename wasn't unique, the
+	   problem could still happen if the file was already moved from
+	   new/ to cur/, so link() doesn't really provide any safety anyway.
+
+	   Besides the small temporary performance benefits, this rename() is
+	   almost required with OSX's HFS+ filesystem, since it implements
+	   hard links in a pretty ugly way, which makes the performance crawl
+	   when a lot of hard links are used. */
+	if (rename(tmp_path, new_path) == 0) {
+		mf->flags |= MAILDIR_FILENAME_FLAG_MOVED;
+		return 0;
+	} else if (ENOQUOTA(errno)) {
+		mail_storage_set_error(storage, MAIL_ERROR_NOQUOTA,
+				       MAIL_ERRSTR_NO_QUOTA);
+		return -1;
+	} else {
+		mailbox_set_critical(&ctx->mbox->box,
+				     "rename(%s, %s) failed: %m",
+				     tmp_path, new_path);
+		return -1;
+	}
+}
+
+static struct mail_save_context *
+maildir_save_transaction_init(struct mailbox_transaction_context *t)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(t->box);
+	struct maildir_save_context *ctx;
+	const char *path;
+	pool_t pool;
+
+	pool = pool_alloconly_create("maildir_save_context", 4096);
+	ctx = p_new(pool, struct maildir_save_context, 1);
+	ctx->ctx.transaction = t;
+	ctx->pool = pool;
+	ctx->mbox = mbox;
+	ctx->trans = t->itrans;
+	ctx->files_tail = &ctx->files;
+	ctx->fd = -1;
+
+	path = mailbox_get_path(&mbox->box);
+	ctx->tmpdir = p_strconcat(pool, path, "/tmp", NULL);
+	ctx->newdir = p_strconcat(pool, path, "/new", NULL);
+	ctx->curdir = p_strconcat(pool, path, "/cur", NULL);
+
+	ctx->last_save_finished = TRUE;
+	return &ctx->ctx;
+}
+
+struct maildir_filename *
+maildir_save_add(struct mail_save_context *_ctx, const char *tmp_fname,
+		 struct mail *src_mail)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+	struct mail_save_data *mdata = &_ctx->data;
+	struct maildir_filename *mf;
+	struct istream *input;
+
+	i_assert(*tmp_fname != '\0');
+
+	/* allow caller to specify recent flag only when uid is specified
+	   (we're replicating, converting, etc.). */
+	if (mdata->uid == 0)
+		mdata->flags |= MAIL_RECENT;
+	else if ((mdata->flags & MAIL_RECENT) == 0 &&
+		 ctx->last_nonrecent_uid < mdata->uid)
+		ctx->last_nonrecent_uid = mdata->uid;
+
+	/* now, we want to be able to rollback the whole append session,
+	   so we'll just store the name of this temp file and move it later
+	   into new/ or cur/. */
+	mf = p_new(ctx->pool, struct maildir_filename, 1);
+	mf->tmp_name = mf->dest_basename = p_strdup(ctx->pool, tmp_fname);
+	mf->flags = mdata->flags;
+	mf->size = UOFF_T_MAX;
+	mf->vsize = UOFF_T_MAX;
+
+	ctx->file_last = mf;
+	i_assert(*ctx->files_tail == NULL);
+	*ctx->files_tail = mf;
+	ctx->files_tail = &mf->next;
+	ctx->files_count++;
+
+	if (mdata->pop3_uidl != NULL)
+		mf->pop3_uidl = p_strdup(ctx->pool, mdata->pop3_uidl);
+	mf->pop3_order = mdata->pop3_order;
+
+	/* insert into index */
+	mail_index_append(ctx->trans, mdata->uid, &ctx->seq);
+	mail_index_update_flags(ctx->trans, ctx->seq,
+				MODIFY_REPLACE,
+				mdata->flags & ENUM_NEGATE(MAIL_RECENT));
+	if (mdata->keywords != NULL) {
+		mail_index_update_keywords(ctx->trans, ctx->seq,
+					   MODIFY_REPLACE, mdata->keywords);
+	}
+	if (mdata->min_modseq != 0) {
+		mail_index_update_modseq(ctx->trans, ctx->seq,
+					 mdata->min_modseq);
+	}
+
+	if (ctx->first_seq == 0) {
+		ctx->first_seq = ctx->seq;
+		i_assert(ctx->files->next == NULL);
+	}
+
+	mail_set_seq_saving(_ctx->dest_mail, ctx->seq);
+
+	if (ctx->input == NULL) {
+		/* copying with hardlinking. */
+		i_assert(src_mail != NULL);
+		index_copy_cache_fields(_ctx, src_mail, ctx->seq);
+		ctx->cur_dest_mail = NULL;
+	} else {
+		input = index_mail_cache_parse_init(_ctx->dest_mail,
+						    ctx->input);
+		i_stream_unref(&ctx->input);
+		ctx->input = input;
+		ctx->cur_dest_mail = _ctx->dest_mail;
+	}
+	return mf;
+}
+
+void maildir_save_set_dest_basename(struct mail_save_context *_ctx,
+				    struct maildir_filename *mf,
+				    const char *basename)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+
+	mf->preserve_filename = TRUE;
+	mf->dest_basename = p_strdup(ctx->pool, basename);
+	ctx->have_preserved_filenames = TRUE;
+}
+
+void maildir_save_set_sizes(struct maildir_filename *mf,
+			    uoff_t size, uoff_t vsize)
+{
+	mf->size = size;
+	mf->vsize = vsize;
+}
+
+static bool
+maildir_get_dest_filename(struct maildir_save_context *ctx,
+			  struct maildir_filename *mf,
+			  const char **fname_r)
+{
+	const char *basename = mf->dest_basename;
+
+	if (mf->size != UOFF_T_MAX && !mf->preserve_filename) {
+		basename = t_strdup_printf("%s,%c=%"PRIuUOFF_T, basename,
+					   MAILDIR_EXTRA_FILE_SIZE, mf->size);
+	}
+
+	if (mf->vsize != UOFF_T_MAX && !mf->preserve_filename) {
+		basename = t_strdup_printf("%s,%c=%"PRIuUOFF_T, basename,
+					   MAILDIR_EXTRA_VIRTUAL_SIZE,
+					   mf->vsize);
+	}
+
+	if (!array_is_created(&mf->keywords) || array_count(&mf->keywords) == 0) {
+		if ((mf->flags & MAIL_FLAGS_MASK) == MAIL_RECENT) {
+			*fname_r = basename;
+			return TRUE;
+		}
+
+		*fname_r = maildir_filename_flags_set(basename,
+					mf->flags & MAIL_FLAGS_MASK);
+		return FALSE;
+	}
+
+	i_assert(ctx->keywords_sync_ctx != NULL ||
+		 !array_is_created(&mf->keywords) || array_count(&mf->keywords) == 0);
+	*fname_r = maildir_filename_flags_kw_set(ctx->keywords_sync_ctx,
+						 basename,
+						 mf->flags & MAIL_FLAGS_MASK,
+						 &mf->keywords);
+	return FALSE;
+}
+
+static const char *maildir_mf_get_path(struct maildir_save_context *ctx,
+				       struct maildir_filename *mf)
+{
+	const char *fname, *dir;
+
+	if ((mf->flags & MAILDIR_FILENAME_FLAG_MOVED) == 0) {
+		/* file is still in tmp/ */
+		return t_strdup_printf("%s/%s", ctx->tmpdir, mf->tmp_name);
+	}
+
+	/* already moved to new/ or cur/ */
+	dir = maildir_get_dest_filename(ctx, mf, &fname) ?
+		ctx->newdir : ctx->curdir;
+	return t_strdup_printf("%s/%s", dir, fname);
+}
+
+
+static struct maildir_filename *
+maildir_save_get_mf(struct mailbox_transaction_context *t, uint32_t seq)
+{
+	struct maildir_save_context *save_ctx = MAILDIR_SAVECTX(t->save_ctx);
+	struct maildir_filename *mf;
+
+	i_assert(seq >= save_ctx->first_seq);
+
+	seq -= save_ctx->first_seq;
+	mf = save_ctx->files;
+	while (seq > 0) {
+		mf = mf->next;
+		i_assert(mf != NULL);
+		seq--;
+	}
+	return mf;
+}
+
+int maildir_save_file_get_size(struct mailbox_transaction_context *t,
+			       uint32_t seq, bool vsize, uoff_t *size_r)
+{
+	struct maildir_filename *mf = maildir_save_get_mf(t, seq);
+
+	*size_r = vsize ? mf->vsize : mf->size;
+	return *size_r == UOFF_T_MAX ? -1 : 0;
+}
+
+const char *maildir_save_file_get_path(struct mailbox_transaction_context *t,
+				       uint32_t seq)
+{
+	struct maildir_save_context *save_ctx = MAILDIR_SAVECTX(t->save_ctx);
+	struct maildir_filename *mf = maildir_save_get_mf(t, seq);
+
+	return maildir_mf_get_path(save_ctx, mf);
+}
+
+static int maildir_create_tmp(struct maildir_mailbox *mbox, const char *dir,
+			      const char **fname_r)
+{
+	struct mailbox *box = &mbox->box;
+	const struct mailbox_permissions *perm = mailbox_get_permissions(box);
+	size_t prefix_len;
+	const char *tmp_fname;
+	string_t *path;
+	mode_t old_mask;
+	int fd;
+
+	path = t_str_new(256);
+	str_append(path, dir);
+	str_append_c(path, '/');
+	prefix_len = str_len(path);
+
+	do {
+		tmp_fname = maildir_filename_generate();
+		str_truncate(path, prefix_len);
+		str_append(path, tmp_fname);
+
+		/* the generated filename is unique. the only reason why it
+		   might return an existing filename is if the time moved
+		   backwards. so we'll use O_EXCL anyway, although it's mostly
+		   useless. */
+		old_mask = umask(0777 & ~perm->file_create_mode);
+		fd = open(str_c(path),
+			  O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0777);
+		umask(old_mask);
+	} while (fd == -1 && errno == EEXIST);
+
+	*fname_r = tmp_fname;
+	if (fd == -1) {
+		if (ENOQUOTA(errno)) {
+			mail_storage_set_error(box->storage,
+				MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA);
+		} else {
+			mailbox_set_critical(box,
+				"open(%s) failed: %m", str_c(path));
+		}
+	} else if (perm->file_create_gid != (gid_t)-1) {
+		if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) {
+			if (errno == EPERM) {
+				mailbox_set_critical(box, "%s",
+					eperm_error_get_chgrp("fchown",
+						str_c(path),
+						perm->file_create_gid,
+						perm->file_create_gid_origin));
+			} else {
+				mailbox_set_critical(box,
+					"fchown(%s) failed: %m", str_c(path));
+			}
+		}
+	}
+
+	return fd;
+}
+
+struct mail_save_context *
+maildir_save_alloc(struct mailbox_transaction_context *t)
+{
+	i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	if (t->save_ctx == NULL)
+		t->save_ctx = maildir_save_transaction_init(t);
+	return t->save_ctx;
+}
+
+int maildir_save_begin(struct mail_save_context *_ctx, struct istream *input)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+	struct maildir_filename *mf;
+
+	/* new mail, new failure state */
+	ctx->failed = FALSE;
+
+	T_BEGIN {
+		/* create a new file in tmp/ directory */
+		const char *fname;
+
+		ctx->fd = maildir_create_tmp(ctx->mbox, ctx->tmpdir, &fname);
+		if (ctx->fd == -1)
+			ctx->failed = TRUE;
+		else {
+			if (ctx->mbox->storage->storage.set->mail_save_crlf)
+				ctx->input = i_stream_create_crlf(input);
+			else
+				ctx->input = i_stream_create_lf(input);
+			mf = maildir_save_add(_ctx, fname, NULL);
+			if (_ctx->data.guid != NULL) {
+				maildir_save_set_dest_basename(_ctx, mf,
+							       _ctx->data.guid);
+			}
+		}
+	} T_END;
+
+	if (!ctx->failed) {
+		_ctx->data.output = o_stream_create_fd_file(ctx->fd, 0, FALSE);
+		o_stream_set_name(_ctx->data.output, t_strdup_printf(
+			"%s/%s", ctx->tmpdir, ctx->file_last->tmp_name));
+		o_stream_cork(_ctx->data.output);
+		ctx->last_save_finished = FALSE;
+	}
+	return ctx->failed ? -1 : 0;
+}
+
+int maildir_save_continue(struct mail_save_context *_ctx)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+
+	if (ctx->failed)
+		return -1;
+
+	if (index_storage_save_continue(_ctx, ctx->input,
+					ctx->cur_dest_mail) < 0) {
+		ctx->failed = TRUE;
+		return -1;
+	}
+	return 0;
+}
+
+static int maildir_save_finish_received_date(struct maildir_save_context *ctx,
+					     const char *path)
+{
+	struct utimbuf buf;
+	struct stat st;
+
+	if (ctx->ctx.data.received_date != (time_t)-1) {
+		/* set the received_date by modifying mtime */
+		buf.actime = ioloop_time;
+		buf.modtime = ctx->ctx.data.received_date;
+
+		if (utime(path, &buf) < 0) {
+			mail_set_critical(ctx->ctx.dest_mail,
+					  "utime(%s) failed: %m", path);
+			return -1;
+		}
+	} else if (ctx->fd != -1) {
+		if (fstat(ctx->fd, &st) == 0)
+			ctx->ctx.data.received_date = st.st_mtime;
+		else {
+			mail_set_critical(ctx->ctx.dest_mail,
+					  "fstat(%s) failed: %m", path);
+			return -1;
+		}
+	} else {
+		/* hardlinked */
+		if (stat(path, &st) == 0)
+			ctx->ctx.data.received_date = st.st_mtime;
+		else {
+			mail_set_critical(ctx->ctx.dest_mail,
+					  "stat(%s) failed: %m", path);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static void maildir_save_remove_last_filename(struct maildir_save_context *ctx)
+{
+	struct maildir_filename **fm;
+
+	index_storage_save_abort_last(&ctx->ctx, ctx->seq);
+	ctx->seq--;
+
+	for (fm = &ctx->files; (*fm)->next != NULL; fm = &(*fm)->next) ;
+	i_assert(*fm == ctx->file_last);
+	*fm = NULL;
+
+	ctx->files_tail = fm;
+	ctx->file_last = NULL;
+	ctx->files_count--;
+}
+
+void maildir_save_finish_keywords(struct mail_save_context *_ctx)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+
+	ARRAY_TYPE(keyword_indexes) keyword_idx;
+	t_array_init(&keyword_idx, 8);
+	mail_index_lookup_keywords(ctx->ctx.transaction->view, ctx->seq,
+				   &keyword_idx);
+
+	if (array_count(&keyword_idx) > 0) {
+		/* copy keywords */
+		p_array_init(&ctx->file_last->keywords, ctx->pool,
+			     array_count(&keyword_idx));
+		array_copy(&ctx->file_last->keywords.arr, 0, &keyword_idx.arr, 0,
+			   array_count(&keyword_idx));
+		ctx->have_keywords = TRUE;
+	}
+}
+
+static int maildir_save_finish_real(struct mail_save_context *_ctx)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+	struct mail_storage *storage = &ctx->mbox->storage->storage;
+	const char *path, *output_errstr;
+	off_t real_size;
+	uoff_t size;
+	int output_errno;
+
+	ctx->last_save_finished = TRUE;
+	if (ctx->failed && ctx->fd == -1) {
+		/* tmp file creation failed */
+		return -1;
+	}
+
+	path = t_strconcat(ctx->tmpdir, "/", ctx->file_last->tmp_name, NULL);
+	if (o_stream_finish(_ctx->data.output) < 0) {
+		if (!mail_storage_set_error_from_errno(storage)) {
+			mail_set_critical(_ctx->dest_mail,
+				"write(%s) failed: %s", path,
+				o_stream_get_error(_ctx->data.output));
+		}
+		ctx->failed = TRUE;
+	}
+
+	if (_ctx->data.save_date != (time_t)-1) {
+		/* we can't change ctime, but we can add the date to cache */
+		struct index_mail *mail = INDEX_MAIL(_ctx->dest_mail);
+		uint32_t t = _ctx->data.save_date;
+
+		index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t));
+	}
+
+ 	if (maildir_save_finish_received_date(ctx, path) < 0)
+		ctx->failed = TRUE;
+
+	if (ctx->cur_dest_mail != NULL) {
+		index_mail_cache_parse_deinit(ctx->cur_dest_mail,
+					      ctx->ctx.data.received_date,
+					      !ctx->failed);
+	}
+	i_stream_unref(&ctx->input);
+
+	/* remember the size in case we want to add it to filename */
+	ctx->file_last->size = _ctx->data.output->offset;
+	if (ctx->cur_dest_mail == NULL ||
+	    mail_get_virtual_size(ctx->cur_dest_mail,
+				  &ctx->file_last->vsize) < 0)
+		ctx->file_last->vsize = UOFF_T_MAX;
+
+	output_errno = _ctx->data.output->stream_errno;
+	output_errstr = t_strdup(o_stream_get_error(_ctx->data.output));
+	o_stream_destroy(&_ctx->data.output);
+
+	maildir_save_finish_keywords(_ctx);
+
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER &&
+	    !ctx->failed) {
+		if (fsync(ctx->fd) < 0) {
+			if (!mail_storage_set_error_from_errno(storage)) {
+				mail_set_critical(_ctx->dest_mail,
+						  "fsync(%s) failed: %m", path);
+			}
+			ctx->failed = TRUE;
+		}
+	}
+	real_size = lseek(ctx->fd, 0, SEEK_END);
+	if (real_size == (off_t)-1) {
+		mail_set_critical(_ctx->dest_mail, "lseek(%s) failed: %m", path);
+	} else if (real_size != (off_t)ctx->file_last->size &&
+		   (!maildir_filename_get_size(ctx->file_last->dest_basename,
+					       MAILDIR_EXTRA_FILE_SIZE, &size) ||
+		    size != ctx->file_last->size)) {
+		/* e.g. zlib plugin was used. the "physical size" must be in
+		   the maildir filename, since stat() will return wrong size */
+		ctx->file_last->preserve_filename = FALSE;
+		/* preserve the GUID if needed */
+		if (ctx->file_last->guid == NULL)
+			ctx->file_last->guid = ctx->file_last->dest_basename;
+		/* reset the base name as well, just in case there's a
+		   ,W=vsize */
+		ctx->file_last->dest_basename = ctx->file_last->tmp_name;
+	}
+	if (close(ctx->fd) < 0) {
+		if (!mail_storage_set_error_from_errno(storage)) {
+			mail_set_critical(_ctx->dest_mail,
+					  "close(%s) failed: %m", path);
+		}
+		ctx->failed = TRUE;
+	}
+	ctx->fd = -1;
+
+	if (ctx->failed) {
+		/* delete the tmp file */
+		i_unlink_if_exists(path);
+
+		if (ENOQUOTA(output_errno)) {
+			mail_storage_set_error(storage,
+				MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA);
+		} else if (output_errno != 0) {
+			mail_set_critical(_ctx->dest_mail,
+				"write(%s) failed: %s", path, output_errstr);
+		}
+
+		maildir_save_remove_last_filename(ctx);
+		return -1;
+	}
+
+	ctx->file_last = NULL;
+	return 0;
+}
+
+int maildir_save_finish(struct mail_save_context *ctx)
+{
+	int ret;
+
+	T_BEGIN {
+		ret = maildir_save_finish_real(ctx);
+	} T_END;
+	index_save_context_free(ctx);
+	return ret;
+}
+
+void maildir_save_cancel(struct mail_save_context *_ctx)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+
+	ctx->failed = TRUE;
+	(void)maildir_save_finish(_ctx);
+}
+
+static void
+maildir_save_unlink_files(struct maildir_save_context *ctx)
+{
+	struct maildir_filename *mf;
+
+	for (mf = ctx->files; mf != NULL; mf = mf->next) T_BEGIN {
+		i_unlink(maildir_mf_get_path(ctx, mf));
+	} T_END;
+	ctx->files = NULL;
+}
+
+static int maildir_transaction_fsync_dirs(struct maildir_save_context *ctx,
+					  bool new_changed, bool cur_changed)
+{
+	struct mail_storage *storage = &ctx->mbox->storage->storage;
+
+	if (storage->set->parsed_fsync_mode == FSYNC_MODE_NEVER)
+		return 0;
+
+	if (new_changed) {
+		if (fdatasync_path(ctx->newdir) < 0) {
+			mailbox_set_critical(&ctx->mbox->box,
+				"fdatasync_path(%s) failed: %m", ctx->newdir);
+			return -1;
+		}
+	}
+	if (cur_changed) {
+		if (fdatasync_path(ctx->curdir) < 0) {
+			mailbox_set_critical(&ctx->mbox->box,
+				"fdatasync_path(%s) failed: %m", ctx->curdir);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static int seq_range_cmp(const struct seq_range *r1, const struct seq_range *r2)
+{
+	if (r1->seq1 < r2->seq2)
+		return -1;
+	else if (r1->seq1 > r2->seq2)
+		return 1;
+	else
+		return 0;
+}
+
+static uint32_t
+maildir_save_set_recent_flags(struct maildir_save_context *ctx)
+{
+	struct maildir_mailbox *mbox = ctx->mbox;
+	ARRAY_TYPE(seq_range) saved_sorted_uids;
+	const struct seq_range *uids;
+	unsigned int i, count;
+	uint32_t uid;
+
+	count = array_count(&ctx->ctx.transaction->changes->saved_uids);
+	if (count == 0)
+		return 0;
+
+	t_array_init(&saved_sorted_uids, count);
+	array_append_array(&saved_sorted_uids,
+			   &ctx->ctx.transaction->changes->saved_uids);
+	array_sort(&saved_sorted_uids, seq_range_cmp);
+
+	uids = array_get(&saved_sorted_uids, &count);
+	for (i = 0; i < count; i++) {
+		for (uid = uids[i].seq1; uid <= uids[i].seq2; uid++)
+			mailbox_recent_flags_set_uid(&mbox->box, uid);
+	}
+	return uids[count-1].seq2 + 1;
+}
+
+static int
+maildir_save_sync_index(struct maildir_save_context *ctx)
+{
+	struct mailbox_transaction_context *_t = ctx->ctx.transaction;
+	struct maildir_mailbox *mbox = ctx->mbox;
+	uint32_t first_uid, next_uid, first_recent_uid;
+	int ret;
+
+	/* we'll need to keep the lock past the sync deinit */
+	ret = maildir_uidlist_lock(mbox->uidlist);
+	i_assert(ret > 0);
+
+	if (maildir_sync_header_refresh(mbox) < 0)
+		return -1;
+	if ((ret = maildir_uidlist_refresh_fast_init(mbox->uidlist)) < 0)
+		return -1;
+
+	if (ret == 0) {
+		/* uidlist doesn't exist. make sure all existing message
+		   are added to uidlist first. */
+		(void)maildir_storage_sync_force(mbox, 0);
+	}
+
+	if (maildir_sync_index_begin(mbox, NULL, &ctx->sync_ctx) < 0)
+		return -1;
+	ctx->keywords_sync_ctx =
+		maildir_sync_get_keywords_sync_ctx(ctx->sync_ctx);
+
+	/* now that uidlist is locked, make sure all the existing mails
+	   have been added to index. we don't really look into the
+	   maildir, just add all the new mails listed in
+	   dovecot-uidlist to index. */
+	if (maildir_sync_index(ctx->sync_ctx, TRUE) < 0)
+		return -1;
+
+	/* if messages were added to index, assign them UIDs */
+	first_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
+	i_assert(first_uid != 0);
+	mail_index_append_finish_uids(ctx->trans, first_uid,
+				      &_t->changes->saved_uids);
+	i_assert(ctx->files_count == seq_range_count(&_t->changes->saved_uids));
+
+	/* these mails are all recent in our session */
+	T_BEGIN {
+		next_uid = maildir_save_set_recent_flags(ctx);
+	} T_END;
+
+	if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0)
+		first_recent_uid = next_uid;
+	else if (ctx->last_nonrecent_uid != 0)
+		first_recent_uid = ctx->last_nonrecent_uid + 1;
+	else
+		first_recent_uid = 0;
+
+	if (first_recent_uid != 0) {
+		/* maildir_sync_index() dropped recent flags from
+		   existing messages. we'll still need to drop recent
+		   flags from these newly added messages. */
+		mail_index_update_header(ctx->trans,
+					 offsetof(struct mail_index_header,
+						  first_recent_uid),
+					 &first_recent_uid,
+					 sizeof(first_recent_uid), FALSE);
+	}
+	return 0;
+}
+
+static void
+maildir_save_rollback_index_changes(struct maildir_save_context *ctx)
+{
+	uint32_t seq;
+
+	if (ctx->seq == 0)
+		return;
+
+	for (seq = ctx->seq; seq >= ctx->first_seq; seq--)
+		mail_index_expunge(ctx->trans, seq);
+
+	mail_cache_transaction_reset(ctx->ctx.transaction->cache_trans);
+}
+
+static bool maildir_filename_has_conflict(struct maildir_filename *mf,
+					  struct maildir_filename *prev_mf)
+{
+	if (strcmp(mf->dest_basename, prev_mf->dest_basename) == 0) {
+		/* already used this */
+		return TRUE;
+	}
+	if (prev_mf->guid != NULL &&
+	    strcmp(mf->dest_basename, prev_mf->guid) == 0) {
+		/* previous filename also had a conflict */
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static void
+maildir_filename_check_conflicts(struct maildir_save_context *ctx,
+				 struct maildir_filename *mf,
+				 struct maildir_filename *prev_mf)
+{
+	uoff_t size, vsize;
+
+	if (!ctx->locked_uidlist_refresh && ctx->locked) {
+		(void)maildir_uidlist_refresh(ctx->mbox->uidlist);
+		ctx->locked_uidlist_refresh = TRUE;
+	}
+
+	if (!maildir_filename_get_size(mf->dest_basename,
+				       MAILDIR_EXTRA_FILE_SIZE, &size))
+		size = UOFF_T_MAX;
+	if (!maildir_filename_get_size(mf->dest_basename,
+				       MAILDIR_EXTRA_VIRTUAL_SIZE, &vsize))
+		vsize = UOFF_T_MAX;
+
+	if (size != mf->size || vsize != mf->vsize ||
+	    !ctx->locked_uidlist_refresh ||
+	    (prev_mf != NULL && maildir_filename_has_conflict(mf, prev_mf)) ||
+	    maildir_uidlist_get_full_filename(ctx->mbox->uidlist,
+					      mf->dest_basename) != NULL) {
+		/* a) dest_basename didn't contain the (correct) size/vsize.
+		   they're required for good performance.
+
+		   b) file already exists. give it another name.
+		   but preserve the size/vsize in the filename if possible */
+		if (mf->size == UOFF_T_MAX)
+			mf->size = size;
+		if (mf->vsize == UOFF_T_MAX)
+			mf->vsize = size;
+
+		mf->guid = mf->dest_basename;
+		mf->dest_basename = p_strdup(ctx->pool,
+					     maildir_filename_generate());
+		mf->preserve_filename = FALSE;
+	}
+}
+
+static int
+maildir_filename_dest_basename_cmp(struct maildir_filename *const *f1,
+				   struct maildir_filename *const *f2)
+{
+	return strcmp((*f1)->dest_basename, (*f2)->dest_basename);
+}
+
+static int
+maildir_save_move_files_to_newcur(struct maildir_save_context *ctx)
+{
+	ARRAY(struct maildir_filename *) files;
+	struct maildir_filename *mf, *prev_mf;
+	bool newdir, new_changed, cur_changed;
+	int ret;
+
+	/* put files into an array sorted by the destination filename.
+	   this way we can easily check if there are duplicate destination
+	   filenames within this transaction. */
+	t_array_init(&files, ctx->files_count);
+	for (mf = ctx->files; mf != NULL; mf = mf->next)
+		array_push_back(&files, &mf);
+	array_sort(&files, maildir_filename_dest_basename_cmp);
+
+	new_changed = cur_changed = FALSE;
+	prev_mf = NULL;
+	array_foreach_elem(&files, mf) {
+		T_BEGIN {
+			const char *dest;
+
+			if (mf->preserve_filename) {
+				maildir_filename_check_conflicts(ctx, mf,
+								 prev_mf);
+			}
+
+			newdir = maildir_get_dest_filename(ctx, mf, &dest);
+			if (newdir)
+				new_changed = TRUE;
+			else
+				cur_changed = TRUE;
+			ret = maildir_file_move(ctx, mf, dest, newdir);
+		} T_END;
+		if (ret < 0)
+			return -1;
+		prev_mf = mf;
+	}
+
+	if (ctx->locked) {
+		i_assert(ctx->sync_ctx != NULL);
+		maildir_sync_set_new_msgs_count(ctx->sync_ctx,
+						array_count(&files));
+	}
+	return maildir_transaction_fsync_dirs(ctx, new_changed, cur_changed);
+}
+
+static void maildir_save_sync_uidlist(struct maildir_save_context *ctx)
+{
+	struct mailbox_transaction_context *t = ctx->ctx.transaction;
+	struct maildir_filename *mf;
+	struct seq_range_iter iter;
+	enum maildir_uidlist_rec_flag flags;
+	struct maildir_uidlist_rec *rec;
+	unsigned int n = 0;
+	uint32_t uid;
+	bool newdir, bret;
+	int ret;
+
+	seq_range_array_iter_init(&iter, &t->changes->saved_uids);
+	for (mf = ctx->files; mf != NULL; mf = mf->next) T_BEGIN {
+		const char *dest;
+
+		bret = seq_range_array_iter_nth(&iter, n++, &uid);
+		i_assert(bret);
+
+		newdir = maildir_get_dest_filename(ctx, mf, &dest);
+		flags = MAILDIR_UIDLIST_REC_FLAG_RECENT;
+		if (newdir)
+			flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+		ret = maildir_uidlist_sync_next_uid(ctx->uidlist_sync_ctx,
+						    dest, uid, flags, &rec);
+		i_assert(ret > 0);
+		i_assert(rec != NULL);
+		if (mf->guid != NULL) {
+			maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec,
+				MAILDIR_UIDLIST_REC_EXT_GUID, mf->guid);
+		}
+		if (mf->pop3_uidl != NULL) {
+			maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec,
+				MAILDIR_UIDLIST_REC_EXT_POP3_UIDL,
+				mf->pop3_uidl);
+		}
+		if (mf->pop3_order > 0) {
+			maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec,
+				MAILDIR_UIDLIST_REC_EXT_POP3_ORDER,
+				t_strdup_printf("%u", mf->pop3_order));
+		}
+	} T_END;
+	i_assert(!seq_range_array_iter_nth(&iter, n, &uid));
+}
+
+int maildir_transaction_save_commit_pre(struct mail_save_context *_ctx)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+	struct mailbox_transaction_context *_t = _ctx->transaction;
+	enum maildir_uidlist_sync_flags sync_flags;
+	int ret;
+
+	i_assert(_ctx->data.output == NULL);
+	i_assert(ctx->last_save_finished);
+
+	if (ctx->files_count == 0)
+		return 0;
+
+	sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL |
+		MAILDIR_UIDLIST_SYNC_NOREFRESH;
+
+	if ((_t->flags & MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS) != 0) {
+		/* we want to assign UIDs, we must lock uidlist */
+	} else if (ctx->have_keywords) {
+		/* keywords file updating relies on uidlist lock. */
+	} else if (ctx->have_preserved_filenames) {
+		/* we're trying to use some potentially existing filenames.
+		   we must lock to avoid race conditions where two sessions
+		   try to save the same filename. */
+	} else {
+		/* no requirement to lock uidlist. if we happen to get a lock,
+		   assign uids. */
+		sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK;
+	}
+	ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags,
+					&ctx->uidlist_sync_ctx);
+	if (ret > 0) {
+		ctx->locked = TRUE;
+		if (maildir_save_sync_index(ctx) < 0) {
+			maildir_transaction_save_rollback(_ctx);
+			return -1;
+		}
+	} else if (ret == 0 &&
+		   (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0) {
+		ctx->locked = FALSE;
+		i_assert(ctx->uidlist_sync_ctx == NULL);
+		/* since we couldn't lock uidlist, we'll have to drop the
+		   appends to index. */
+		maildir_save_rollback_index_changes(ctx);
+	} else {
+		maildir_transaction_save_rollback(_ctx);
+		return -1;
+	}
+
+	T_BEGIN {
+		ret = maildir_save_move_files_to_newcur(ctx);
+	} T_END;
+	if (ctx->locked) {
+		if (ret == 0) {
+			/* update dovecot-uidlist file. */
+			maildir_save_sync_uidlist(ctx);
+		}
+
+		if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx,
+						ret == 0) < 0)
+			ret = -1;
+	}
+
+	_t->changes->uid_validity =
+		maildir_uidlist_get_uid_validity(ctx->mbox->uidlist);
+
+	if (ctx->locked) {
+		/* It doesn't matter if index syncing fails */
+		ctx->keywords_sync_ctx = NULL;
+		if (ret < 0)
+			maildir_sync_index_rollback(&ctx->sync_ctx);
+		else
+			(void)maildir_sync_index_commit(&ctx->sync_ctx);
+	}
+
+	if (ret < 0) {
+		ctx->keywords_sync_ctx = !ctx->have_keywords ? NULL :
+			maildir_keywords_sync_init(ctx->mbox->keywords,
+						   ctx->mbox->box.index);
+
+		/* unlink the files we just moved in an attempt to rollback
+		   the transaction. uidlist is still locked, so at least other
+		   Dovecot instances haven't yet seen the files. we need
+		   to have the keywords sync context to be able to generate
+		   the destination filenames if keywords were used. */
+		maildir_save_unlink_files(ctx);
+
+		if (ctx->keywords_sync_ctx != NULL)
+			maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx);
+		/* returning failure finishes the save_context */
+		maildir_transaction_save_rollback(_ctx);
+		return -1;
+	}
+	return 0;
+}
+
+void maildir_transaction_save_commit_post(struct mail_save_context *_ctx,
+					  struct mail_index_transaction_commit_result *result ATTR_UNUSED)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+
+	_ctx->transaction = NULL; /* transaction is already freed */
+
+	if (ctx->locked)
+		maildir_uidlist_unlock(ctx->mbox->uidlist);
+	pool_unref(&ctx->pool);
+}
+
+void maildir_transaction_save_rollback(struct mail_save_context *_ctx)
+{
+	struct maildir_save_context *ctx = MAILDIR_SAVECTX(_ctx);
+
+	i_assert(_ctx->data.output == NULL);
+
+	if (!ctx->last_save_finished)
+		maildir_save_cancel(&ctx->ctx);
+
+	/* delete files in tmp/ */
+	maildir_save_unlink_files(ctx);
+
+	if (ctx->uidlist_sync_ctx != NULL)
+		(void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE);
+	if (ctx->sync_ctx != NULL)
+		maildir_sync_index_rollback(&ctx->sync_ctx);
+	if (ctx->locked)
+		maildir_uidlist_unlock(ctx->mbox->uidlist);
+
+	pool_unref(&ctx->pool);
+}
diff --git a/src/lib-storage/index/maildir/maildir-settings.c b/src/lib-storage/index/maildir/maildir-settings.c
new file mode 100644
index 0000000..d986d18
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-settings.c
@@ -0,0 +1,46 @@
+/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "settings-parser.h"
+#include "mail-storage-settings.h"
+#include "maildir-settings.h"
+
+#include <stddef.h>
+
+#undef DEF
+#define DEF(type, name) \
+	SETTING_DEFINE_STRUCT_##type(#name, name, struct maildir_settings)
+
+static const struct setting_define maildir_setting_defines[] = {
+	DEF(BOOL, maildir_copy_with_hardlinks),
+	DEF(BOOL, maildir_very_dirty_syncs),
+	DEF(BOOL, maildir_broken_filename_sizes),
+	DEF(BOOL, maildir_empty_new),
+
+	SETTING_DEFINE_LIST_END
+};
+
+static const struct maildir_settings maildir_default_settings = {
+	.maildir_copy_with_hardlinks = TRUE,
+	.maildir_very_dirty_syncs = FALSE,
+	.maildir_broken_filename_sizes = FALSE,
+	.maildir_empty_new = FALSE
+};
+
+static const struct setting_parser_info maildir_setting_parser_info = {
+	.module_name = "maildir",
+	.defines = maildir_setting_defines,
+	.defaults = &maildir_default_settings,
+
+	.type_offset = SIZE_MAX,
+	.struct_size = sizeof(struct maildir_settings),
+
+	.parent_offset = SIZE_MAX,
+	.parent = &mail_user_setting_parser_info
+};
+
+const struct setting_parser_info *maildir_get_setting_parser_info(void)
+{
+	return &maildir_setting_parser_info;
+}
+
diff --git a/src/lib-storage/index/maildir/maildir-settings.h b/src/lib-storage/index/maildir/maildir-settings.h
new file mode 100644
index 0000000..cfcb732
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-settings.h
@@ -0,0 +1,13 @@
+#ifndef MAILDIR_SETTINGS_H
+#define MAILDIR_SETTINGS_H
+
+struct maildir_settings {
+	bool maildir_copy_with_hardlinks;
+	bool maildir_very_dirty_syncs;
+	bool maildir_broken_filename_sizes;
+	bool maildir_empty_new;
+};
+
+const struct setting_parser_info *maildir_get_setting_parser_info(void);
+
+#endif
diff --git a/src/lib-storage/index/maildir/maildir-storage.c b/src/lib-storage/index/maildir/maildir-storage.c
new file mode 100644
index 0000000..e65579f
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-storage.c
@@ -0,0 +1,749 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "mkdir-parents.h"
+#include "eacces-error.h"
+#include "unlink-old-files.h"
+#include "mailbox-uidvalidity.h"
+#include "mailbox-list-private.h"
+#include "maildir-storage.h"
+#include "maildir-uidlist.h"
+#include "maildir-keywords.h"
+#include "maildir-sync.h"
+#include "index-mail.h"
+
+#include <sys/stat.h>
+
+#define MAILDIR_LIST_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, maildir_mailbox_list_module)
+#define MAILDIR_SUBFOLDER_FILENAME "maildirfolder"
+
+struct maildir_mailbox_list_context {
+	union mailbox_list_module_context module_ctx;
+	const struct maildir_settings *set;
+};
+
+extern struct mail_storage maildir_storage;
+extern struct mailbox maildir_mailbox;
+
+static struct event_category event_category_maildir = {
+	.name = "maildir",
+	.parent = &event_category_storage,
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(maildir_mailbox_list_module,
+				  &mailbox_list_module_register);
+static const char *maildir_subdirs[] = { "cur", "new", "tmp" };
+
+static void maildir_mailbox_close(struct mailbox *box);
+
+static struct mail_storage *maildir_storage_alloc(void)
+{
+	struct maildir_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("maildir storage", 512+256);
+	storage = p_new(pool, struct maildir_storage, 1);
+	storage->storage = maildir_storage;
+	storage->storage.pool = pool;
+	return &storage->storage;
+}
+
+static int
+maildir_storage_create(struct mail_storage *_storage, struct mail_namespace *ns,
+		       const char **error_r ATTR_UNUSED)
+{
+	struct maildir_storage *storage = MAILDIR_STORAGE(_storage);
+	struct mailbox_list *list = ns->list;
+	const char *dir;
+
+	storage->set = mail_namespace_get_driver_settings(ns, _storage);
+
+	storage->temp_prefix = p_strdup(_storage->pool,
+					mailbox_list_get_temp_prefix(list));
+
+	if (list->set.control_dir == NULL && list->set.inbox_path == NULL &&
+	    (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) {
+		/* put the temp files into tmp/ directory preferably */
+		storage->temp_prefix = p_strconcat(_storage->pool, "tmp/",
+						   storage->temp_prefix, NULL);
+		dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_DIR);
+	} else {
+		/* control dir should also be writable */
+		dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL);
+	}
+	_storage->temp_path_prefix = p_strconcat(_storage->pool, dir, "/",
+						 storage->temp_prefix, NULL);
+	return 0;
+}
+
+static void maildir_storage_get_list_settings(const struct mail_namespace *ns,
+					      struct mailbox_list_settings *set)
+{
+	if (set->layout == NULL)
+		set->layout = MAILBOX_LIST_NAME_MAILDIRPLUSPLUS;
+	if (set->subscription_fname == NULL)
+		set->subscription_fname = MAILDIR_SUBSCRIPTION_FILE_NAME;
+
+	if (set->inbox_path == NULL && *set->maildir_name == '\0' &&
+	    (strcmp(set->layout, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) == 0 ||
+	     strcmp(set->layout, MAILBOX_LIST_NAME_FS) == 0) &&
+	    (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0) {
+		/* Maildir++ INBOX is the Maildir base itself */
+		set->inbox_path = set->root_dir;
+	}
+}
+
+static const char *
+maildir_storage_find_root_dir(const struct mail_namespace *ns)
+{
+	bool debug = ns->mail_set->mail_debug;
+	const char *home, *path;
+
+	/* we'll need to figure out the maildir location ourself.
+	   It's ~/Maildir unless we are chrooted. */
+	if (ns->owner != NULL &&
+	    mail_user_get_home(ns->owner, &home) > 0) {
+		path = t_strconcat(home, "/Maildir", NULL);
+		if (access(path, R_OK|W_OK|X_OK) == 0) {
+			if (debug)
+				i_debug("maildir: root exists (%s)", path);
+			return path;
+		} 
+		if (debug)
+			i_debug("maildir: access(%s, rwx): failed: %m", path);
+	} else {
+		if (debug)
+			i_debug("maildir: Home directory not set");
+		if (access("/cur", R_OK|W_OK|X_OK) == 0) {
+			if (debug)
+				i_debug("maildir: /cur exists, assuming chroot");
+			return "/";
+		}
+	}
+	return NULL;
+}
+
+static bool maildir_storage_autodetect(const struct mail_namespace *ns,
+				       struct mailbox_list_settings *set)
+{
+	bool debug = ns->mail_set->mail_debug;
+	struct stat st;
+	const char *path, *root_dir;
+
+	if (set->root_dir != NULL)
+		root_dir = set->root_dir;
+	else {
+		root_dir = maildir_storage_find_root_dir(ns);
+		if (root_dir == NULL) {
+			if (debug)
+				i_debug("maildir: couldn't find root dir");
+			return FALSE;
+		}
+	}
+
+	path = t_strconcat(root_dir, "/cur", NULL);
+	if (stat(path, &st) < 0) {
+		if (debug)
+			i_debug("maildir autodetect: stat(%s) failed: %m", path);
+		return FALSE;
+	}
+
+	if (!S_ISDIR(st.st_mode)) {
+		if (debug)
+			i_debug("maildir autodetect: %s not a directory", path);
+		return FALSE;
+	}
+
+	set->root_dir = root_dir;
+	maildir_storage_get_list_settings(ns, set);
+	return TRUE;
+}
+
+static int
+mkdir_verify(struct mailbox *box, const char *dir, bool verify)
+{
+	const struct mailbox_permissions *perm;
+	struct stat st;
+
+	if (verify) {
+		if (stat(dir, &st) == 0)
+			return 0;
+
+		if (errno != ENOENT) {
+			mailbox_set_critical(box, "stat(%s) failed: %m", dir);
+			return -1;
+		}
+	}
+
+	perm = mailbox_get_permissions(box);
+	if (mkdir_parents_chgrp(dir, perm->dir_create_mode,
+				perm->file_create_gid,
+				perm->file_create_gid_origin) == 0)
+		return 0;
+
+	if (errno == EEXIST) {
+		if (verify)
+			return 0;
+		mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS,
+				       "Mailbox already exists");
+	} else if (errno == ENOENT) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+			"Mailbox was deleted while it was being created");
+	} else if (errno == EACCES) {
+		if (box->list->ns->type == MAIL_NAMESPACE_TYPE_SHARED) {
+			/* shared namespace, don't log permission errors */
+			mail_storage_set_error(box->storage, MAIL_ERROR_PERM,
+					       MAIL_ERRSTR_NO_PERMISSION);
+			return -1;
+		}
+		mailbox_set_critical(box, "%s",
+			mail_error_create_eacces_msg("mkdir", dir));
+	} else {
+		mailbox_set_critical(box, "mkdir(%s) failed: %m", dir);
+	}
+	return -1;
+}
+
+static int maildir_check_tmp(struct mail_storage *storage, const char *dir)
+{
+	unsigned int interval = storage->set->mail_temp_scan_interval;
+	const char *path;
+	struct stat st;
+
+	/* if tmp/ directory exists, we need to clean it up once in a while */
+	path = t_strconcat(dir, "/tmp", NULL);
+	if (stat(path, &st) < 0) {
+		if (errno == ENOENT || errno == ENAMETOOLONG)
+			return 0;
+		if (errno == EACCES) {
+			mail_storage_set_critical(storage, "%s",
+				mail_error_eacces_msg("stat", path));
+			return -1;
+		}
+		mail_storage_set_critical(storage, "stat(%s) failed: %m", path);
+		return -1;
+	}
+
+	if (interval == 0) {
+		/* disabled */
+	} else if (st.st_atime > st.st_ctime + MAILDIR_TMP_DELETE_SECS) {
+		/* the directory should be empty. we won't do anything
+		   until ctime changes. */
+	} else if (st.st_atime < ioloop_time - (time_t)interval) {
+		/* time to scan */
+		(void)unlink_old_files(path, "",
+				       ioloop_time - MAILDIR_TMP_DELETE_SECS);
+	}
+	return 1;
+}
+
+/* create or fix maildir, ignore if it already exists */
+static int create_maildir_subdirs(struct mailbox *box, bool verify)
+{
+	const char *path, *box_path;
+	unsigned int i;
+	enum mail_error error;
+	int ret = 0;
+
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX,
+				&box_path) < 0)
+		return -1;
+
+	for (i = 0; i < N_ELEMENTS(maildir_subdirs); i++) {
+		path = t_strconcat(box_path, "/", maildir_subdirs[i], NULL);
+		if (mkdir_verify(box, path, verify) < 0) {
+			error = mailbox_get_last_mail_error(box);
+			if (error != MAIL_ERROR_EXISTS)
+				return -1;
+			/* try to create all of the directories in case one
+			   of them doesn't exist */
+			ret = -1;
+		}
+	}
+	return ret;
+}
+
+static void maildir_lock_touch_timeout(struct maildir_mailbox *mbox)
+{
+	(void)maildir_uidlist_lock_touch(mbox->uidlist);
+}
+
+static struct mailbox *
+maildir_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
+		      const char *vname, enum mailbox_flags flags)
+{
+	struct maildir_mailbox *mbox;
+	pool_t pool;
+
+	pool = pool_alloconly_create("maildir mailbox", 1024*3);
+	mbox = p_new(pool, struct maildir_mailbox, 1);
+	mbox->box = maildir_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.mail_vfuncs = &maildir_mail_vfuncs;
+	mbox->maildir_list_index_ext_id = (uint32_t)-1;
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
+
+	mbox->storage = MAILDIR_STORAGE(storage);
+	return &mbox->box;
+}
+
+static int maildir_mailbox_open_existing(struct mailbox *box)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+
+	mbox->uidlist = maildir_uidlist_init(mbox);
+	mbox->keywords = maildir_keywords_init(mbox);
+
+	if ((box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0) {
+		if (maildir_uidlist_lock(mbox->uidlist) <= 0) {
+			maildir_mailbox_close(box);
+			return -1;
+		}
+		mbox->keep_lock_to = timeout_add(MAILDIR_LOCK_TOUCH_SECS * 1000,
+						 maildir_lock_touch_timeout,
+						 mbox);
+	}
+
+	if (index_storage_mailbox_open(box, FALSE) < 0) {
+		maildir_mailbox_close(box);
+		return -1;
+	}
+
+	mbox->maildir_ext_id =
+		mail_index_ext_register(mbox->box.index, "maildir",
+					sizeof(mbox->maildir_hdr), 0, 0);
+	return 0;
+}
+
+static bool maildir_storage_is_readonly(struct mailbox *box)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+
+	if (index_storage_is_readonly(box))
+		return TRUE;
+
+	if (maildir_is_backend_readonly(mbox)) {
+		/* return read-only only if there are no private flags
+		   (that are stored in index files) */
+		if (mailbox_get_private_flags_mask(box) == 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static int
+maildir_mailbox_exists(struct mailbox *box, bool auto_boxes,
+		       enum mailbox_existence *existence_r)
+{
+	if (auto_boxes && mailbox_is_autocreated(box)) {
+		*existence_r = MAILBOX_EXISTENCE_SELECT;
+		return 0;
+	}
+
+	return index_storage_mailbox_exists_full(box, "cur", existence_r);
+}
+
+static int maildir_mailbox_open(struct mailbox *box)
+{
+	const char *box_path = mailbox_get_path(box);
+	const char *root_dir;
+	struct stat st;
+	int ret;
+
+	/* begin by checking if tmp/ directory exists and if it should be
+	   cleaned up. */
+	ret = maildir_check_tmp(box->storage, box_path);
+	if (ret > 0) {
+		/* exists */
+		return maildir_mailbox_open_existing(box);
+	}
+	if (ret < 0)
+		return -1;
+
+	/* tmp/ directory doesn't exist. does the maildir? autocreate missing
+	   dirs only with Maildir++ and imapdir layouts. */
+	if (strcmp(box->list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) != 0 &&
+	    strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPDIR) != 0) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+			T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname));
+		return -1;
+	}
+	root_dir = mailbox_list_get_root_forced(box->list,
+						MAILBOX_LIST_PATH_TYPE_MAILBOX);
+	if (strcmp(box_path, root_dir) == 0 && !box->inbox_any) {
+		/* root directory for some namespace. */
+		errno = ENOENT;
+	} else if (stat(box_path, &st) == 0) {
+		/* yes, we'll need to create the missing dirs */
+		if (create_maildir_subdirs(box, TRUE) < 0)
+			return -1;
+
+		return maildir_mailbox_open_existing(box);
+	}
+
+	if (errno == ENOENT || errno == ENAMETOOLONG) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+			T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname));
+		return -1;
+	} else {
+		mailbox_set_critical(box, "stat(%s) failed: %m", box_path);
+		return -1;
+	}
+}
+
+static int maildir_create_shared(struct mailbox *box)
+{
+	const struct mailbox_permissions *perm = mailbox_get_permissions(box);
+	const char *path;
+	mode_t old_mask;
+	int fd, ret;
+
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX,
+				  &path);
+	if (ret < 0)
+		return -1;
+	i_assert(ret > 0);
+
+	old_mask = umask(0);
+	path = t_strconcat(path, "/dovecot-shared", NULL);
+	fd = open(path, O_WRONLY | O_CREAT, perm->file_create_mode);
+	umask(old_mask);
+
+	if (fd == -1) {
+		mailbox_set_critical(box, "open(%s) failed: %m", path);
+		return -1;
+	}
+
+	if (fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) {
+		if (errno == EPERM) {
+			mailbox_set_critical(box, "%s",
+				eperm_error_get_chgrp("fchown", path,
+					perm->file_create_gid,
+					perm->file_create_gid_origin));
+		} else {
+			mailbox_set_critical(box,
+				"fchown(%s) failed: %m", path);
+		}
+	}
+	i_close_fd(&fd);
+	return 0;
+}
+
+static int
+maildir_mailbox_update(struct mailbox *box, const struct mailbox_update *update)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+	struct maildir_uidlist *uidlist;
+	bool locked = FALSE;
+	int ret = 0;
+
+	if (!box->opened) {
+		if (mailbox_open(box) < 0)
+			return -1;
+	}
+	uidlist = mbox->uidlist;
+
+	if (update->uid_validity != 0 || update->min_next_uid != 0 ||
+	    !guid_128_is_empty(update->mailbox_guid)) {
+		if (maildir_uidlist_lock(uidlist) <= 0)
+			return -1;
+
+		locked = TRUE;
+		if (!guid_128_is_empty(update->mailbox_guid))
+			maildir_uidlist_set_mailbox_guid(uidlist, update->mailbox_guid);
+		if (update->uid_validity != 0)
+			maildir_uidlist_set_uid_validity(uidlist, update->uid_validity);
+		if (update->min_next_uid != 0) {
+			maildir_uidlist_set_next_uid(uidlist, update->min_next_uid,
+						     FALSE);
+		}
+		ret = maildir_uidlist_update(uidlist);
+	}
+	if (ret == 0)
+		ret = index_storage_mailbox_update(box, update);
+	if (locked)
+		maildir_uidlist_unlock(uidlist);
+	return ret;
+}
+
+static int maildir_create_maildirfolder_file(struct mailbox *box)
+{
+	const struct mailbox_permissions *perm;
+	const char *path;
+	mode_t old_mask;
+	int fd;
+
+	/* Maildir++ spec wants that maildirfolder named file is created for
+	   all subfolders. Do this only with Maildir++ layout. */
+	if (strcmp(box->list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) != 0)
+		return 0;
+	perm = mailbox_get_permissions(box);
+
+	path = t_strconcat(mailbox_get_path(box),
+			   "/"MAILDIR_SUBFOLDER_FILENAME, NULL);
+	old_mask = umask(0);
+	fd = open(path, O_CREAT | O_WRONLY, perm->file_create_mode);
+	umask(old_mask);
+	if (fd != -1) {
+		/* ok */
+	} else if (errno == ENOENT) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+			"Mailbox was deleted while it was being created");
+		return -1;
+	} else {
+		mailbox_set_critical(box, "open(%s, O_CREAT) failed: %m", path);
+		return -1;
+	}
+
+	if (perm->file_create_gid != (gid_t)-1) {
+		if (fchown(fd, (uid_t)-1, perm->file_create_gid) == 0) {
+			/* ok */
+		} else if (errno == EPERM) {
+			mailbox_set_critical(box, "%s",
+				eperm_error_get_chgrp("fchown", path,
+						      perm->file_create_gid,
+						      perm->file_create_gid_origin));
+		} else {
+			mailbox_set_critical(box, "fchown(%s) failed: %m", path);
+		}
+	}
+	i_close_fd(&fd);
+	return 0;
+}
+
+static int
+maildir_mailbox_create(struct mailbox *box, const struct mailbox_update *update,
+		       bool directory)
+{
+	const char *root_dir, *shared_path;
+	/* allow physical location to exist when we rebuild list index, this
+	   happens with LAYOUT=INDEX only. */
+	bool verify = box->storage->rebuilding_list_index;
+	struct stat st;
+	int ret;
+
+	if ((ret = index_storage_mailbox_create(box, directory)) <= 0)
+		return ret;
+	ret = 0;
+	/* the maildir is created now. finish the creation as best as we can */
+	if (create_maildir_subdirs(box, verify) < 0)
+		ret = -1;
+	if (maildir_create_maildirfolder_file(box) < 0)
+		ret = -1;
+	/* if dovecot-shared exists in the root dir, copy it to newly
+	   created mailboxes */
+	root_dir = mailbox_list_get_root_forced(box->list,
+						MAILBOX_LIST_PATH_TYPE_MAILBOX);
+	shared_path = t_strconcat(root_dir, "/dovecot-shared", NULL);
+	if (stat(shared_path, &st) == 0) {
+		if (maildir_create_shared(box) < 0)
+			ret = -1;
+	}
+	if (update != NULL) {
+		if (maildir_mailbox_update(box, update) < 0)
+			ret = -1;
+	}
+	return ret;
+}
+
+static int
+maildir_mailbox_get_metadata(struct mailbox *box,
+			     enum mailbox_metadata_items items,
+			     struct mailbox_metadata *metadata_r)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+
+	if (index_mailbox_get_metadata(box, items, metadata_r) < 0)
+		return -1;
+
+	if ((items & MAILBOX_METADATA_GUID) != 0) {
+		if (maildir_uidlist_get_mailbox_guid(mbox->uidlist,
+						     metadata_r->guid) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static void maildir_mailbox_close(struct mailbox *box)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+
+	if (mbox->keep_lock_to != NULL) {
+		maildir_uidlist_unlock(mbox->uidlist);
+		timeout_remove(&mbox->keep_lock_to);
+	}
+
+	if (mbox->flags_view != NULL)
+		mail_index_view_close(&mbox->flags_view);
+	if (mbox->keywords != NULL)
+		maildir_keywords_deinit(&mbox->keywords);
+	if (mbox->uidlist != NULL)
+		maildir_uidlist_deinit(&mbox->uidlist);
+	index_storage_mailbox_close(box);
+}
+
+static void maildir_notify_changes(struct mailbox *box)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+	const char *box_path = mailbox_get_path(box);
+
+	if (box->notify_callback == NULL)
+		mailbox_watch_remove_all(&mbox->box);
+	else {
+		mailbox_watch_add(&mbox->box,
+			t_strconcat(box_path, "/new", NULL));
+		mailbox_watch_add(&mbox->box,
+			t_strconcat(box_path, "/cur", NULL));
+	}
+}
+
+static bool
+maildir_is_internal_name(struct mailbox_list *list ATTR_UNUSED,
+			 const char *name)
+{
+	return strcmp(name, "cur") == 0 ||
+		strcmp(name, "new") == 0 ||
+		strcmp(name, "tmp") == 0;
+}
+
+static void maildir_storage_add_list(struct mail_storage *storage,
+				     struct mailbox_list *list)
+{
+	struct maildir_mailbox_list_context *mlist;
+
+	mlist = p_new(list->pool, struct maildir_mailbox_list_context, 1);
+	mlist->module_ctx.super = list->v;
+	mlist->set = mail_namespace_get_driver_settings(list->ns, storage);
+
+	list->v.is_internal_name = maildir_is_internal_name;
+	MODULE_CONTEXT_SET(list, maildir_mailbox_list_module, mlist);
+}
+
+uint32_t maildir_get_uidvalidity_next(struct mailbox_list *list)
+{
+	const char *path;
+
+	path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL);
+	path = t_strconcat(path, "/"MAILDIR_UIDVALIDITY_FNAME, NULL);
+	return mailbox_uidvalidity_next(list, path);
+}
+
+static enum mail_flags maildir_get_private_flags_mask(struct mailbox *box)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+	const char *path, *path2;
+	struct stat st;
+
+	if (mbox->private_flags_mask_set)
+		return mbox->_private_flags_mask;
+	mbox->private_flags_mask_set = TRUE;
+
+	path = mailbox_list_get_root_forced(box->list, MAILBOX_LIST_PATH_TYPE_MAILBOX);
+	if (box->list->set.index_pvt_dir != NULL) {
+		/* private index directory is set. we'll definitely have
+		   private flags. */
+		mbox->_private_flags_mask = MAIL_SEEN;
+	} else if (!mailbox_list_get_root_path(box->list,
+					       MAILBOX_LIST_PATH_TYPE_INDEX,
+					       &path2) ||
+		   strcmp(path, path2) == 0) {
+		/* no separate index directory. we can't have private flags,
+		   so don't even bother checking if dovecot-shared exists */
+	} else {
+		path = t_strconcat(mailbox_get_path(box),
+				   "/dovecot-shared", NULL);
+		if (stat(path, &st) == 0)
+			mbox->_private_flags_mask = MAIL_SEEN;
+	}
+	return mbox->_private_flags_mask;
+}
+
+bool maildir_is_backend_readonly(struct maildir_mailbox *mbox)
+{
+	if (!mbox->backend_readonly_set) {
+		const char *box_path = mailbox_get_path(&mbox->box);
+
+		mbox->backend_readonly_set = TRUE;
+		if (access(t_strconcat(box_path, "/cur", NULL), W_OK) < 0 &&
+		    errno == EACCES)
+			mbox->backend_readonly = TRUE;
+	}
+	return mbox->backend_readonly;
+}
+
+struct mail_storage maildir_storage = {
+	.name = MAILDIR_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS |
+		MAIL_STORAGE_CLASS_FLAG_BINARY_DATA,
+	.event_category = &event_category_maildir,
+
+	.v = {
+                maildir_get_setting_parser_info,
+		maildir_storage_alloc,
+		maildir_storage_create,
+		index_storage_destroy,
+		maildir_storage_add_list,
+		maildir_storage_get_list_settings,
+		maildir_storage_autodetect,
+		maildir_mailbox_alloc,
+		NULL,
+		mail_storage_list_index_rebuild,
+	}
+};
+
+struct mailbox maildir_mailbox = {
+	.v = {
+		maildir_storage_is_readonly,
+		index_storage_mailbox_enable,
+		maildir_mailbox_exists,
+		maildir_mailbox_open,
+		maildir_mailbox_close,
+		index_storage_mailbox_free,
+		maildir_mailbox_create,
+		maildir_mailbox_update,
+		index_storage_mailbox_delete,
+		index_storage_mailbox_rename,
+		index_storage_get_status,
+		maildir_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		maildir_list_index_has_changed,
+		maildir_list_index_update_sync,
+		maildir_storage_sync_init,
+		index_mailbox_sync_next,
+		index_mailbox_sync_deinit,
+		NULL,
+		maildir_notify_changes,
+		index_transaction_begin,
+		index_transaction_commit,
+		index_transaction_rollback,
+		maildir_get_private_flags_mask,
+		index_mail_alloc,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		maildir_save_alloc,
+		maildir_save_begin,
+		maildir_save_continue,
+		maildir_save_finish,
+		maildir_save_cancel,
+		maildir_copy,
+		maildir_transaction_save_commit_pre,
+		maildir_transaction_save_commit_post,
+		maildir_transaction_save_rollback,
+		index_storage_is_inconsistent
+	}
+};
diff --git a/src/lib-storage/index/maildir/maildir-storage.h b/src/lib-storage/index/maildir/maildir-storage.h
new file mode 100644
index 0000000..da92866
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-storage.h
@@ -0,0 +1,148 @@
+#ifndef MAILDIR_STORAGE_H
+#define MAILDIR_STORAGE_H
+
+#include "maildir-settings.h"
+
+#define MAILDIR_STORAGE_NAME "maildir"
+#define MAILDIR_SUBSCRIPTION_FILE_NAME "subscriptions"
+#define MAILDIR_UIDVALIDITY_FNAME "dovecot-uidvalidity"
+
+/* "base,S=123:2," means:
+   <base> [<extra sep> <extra data> [..]] <info sep> 2 <flags sep> */
+#define MAILDIR_INFO_SEP ':'
+#define MAILDIR_EXTRA_SEP ','
+#define MAILDIR_FLAGS_SEP ','
+
+#define MAILDIR_INFO_SEP_S ":"
+#define MAILDIR_EXTRA_SEP_S ","
+#define MAILDIR_FLAGS_SEP_S ","
+
+/* ":2," is the standard flags separator */
+#define MAILDIR_FLAGS_FULL_SEP MAILDIR_INFO_SEP_S "2" MAILDIR_FLAGS_SEP_S
+
+#define MAILDIR_KEYWORD_FIRST 'a'
+#define MAILDIR_KEYWORD_LAST 'z'
+#define MAILDIR_MAX_KEYWORDS (MAILDIR_KEYWORD_LAST - MAILDIR_KEYWORD_FIRST + 1)
+
+/* Maildir++ extension: include file size in the filename to avoid stat() */
+#define MAILDIR_EXTRA_FILE_SIZE 'S'
+/* Something (can't remember what anymore) could use 'W' in filename to avoid
+   calculating file's virtual size (added missing CRs). */
+#define MAILDIR_EXTRA_VIRTUAL_SIZE 'W'
+
+/* Delete files having ctime older than this from tmp/. 36h is standard. */
+#define MAILDIR_TMP_DELETE_SECS (36*60*60)
+
+/* How often to touch the uidlist lock file when it's locked.
+   This is done both when using KEEP_LOCKED flag and when syncing a large
+   maildir. */
+#define MAILDIR_LOCK_TOUCH_SECS 10
+
+/* If an operation fails with ENOENT, we'll check if the mailbox is deleted
+   or if some directory is just missing. If it's missing, we'll create the
+   directories and try again this many times before failing. */
+#define MAILDIR_DELETE_RETRY_COUNT 3
+
+#include "index-storage.h"
+
+struct timeval;
+struct maildir_save_context;
+struct maildir_copy_context;
+
+struct maildir_index_header {
+	uint32_t new_check_time, new_mtime, new_mtime_nsecs;
+	uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs;
+	uint32_t uidlist_mtime, uidlist_mtime_nsecs, uidlist_size;
+};
+
+struct maildir_list_index_record {
+	uint32_t new_mtime, cur_mtime;
+};
+
+struct maildir_storage {
+	struct mail_storage storage;
+
+	const struct maildir_settings *set;
+	const char *temp_prefix;
+};
+
+struct maildir_mailbox {
+	struct mailbox box;
+	struct maildir_storage *storage;
+	struct mail_index_view *flags_view;
+
+	struct timeout *keep_lock_to;
+
+	/* Filled lazily by mailbox_get_private_flags_mask() */
+	enum mail_flags _private_flags_mask;
+
+	/* maildir sync: */
+	struct maildir_uidlist *uidlist;
+	struct maildir_keywords *keywords;
+
+	struct maildir_index_header maildir_hdr;
+	uint32_t maildir_ext_id;
+	uint32_t maildir_list_index_ext_id;
+
+	bool synced:1;
+	bool syncing_commit:1;
+	bool private_flags_mask_set:1;
+	bool backend_readonly:1;
+	bool backend_readonly_set:1;
+	bool sync_uidlist_refreshed:1;
+};
+
+#define MAILDIR_STORAGE(s)	container_of(s, struct maildir_storage, storage)
+#define MAILDIR_MAILBOX(s)	container_of(s, struct maildir_mailbox, box)
+
+extern struct mail_vfuncs maildir_mail_vfuncs;
+
+/* Return -1 = error, 0 = file not found, 1 = ok */
+typedef int maildir_file_do_func(struct maildir_mailbox *mbox,
+				 const char *path, void *context);
+
+int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid,
+		    maildir_file_do_func *callback, void *context);
+#define maildir_file_do(mbox, seq, callback, context) \
+	maildir_file_do(mbox, seq - \
+		CALLBACK_TYPECHECK(callback, int (*)( \
+			struct maildir_mailbox *, const char *, typeof(context))), \
+		(maildir_file_do_func *)callback, context)
+
+bool maildir_set_deleted(struct mailbox *box);
+uint32_t maildir_get_uidvalidity_next(struct mailbox_list *list);
+int maildir_lose_unexpected_dir(struct mail_storage *storage, const char *path);
+bool maildir_is_backend_readonly(struct maildir_mailbox *mbox);
+
+struct mail_save_context *
+maildir_save_alloc(struct mailbox_transaction_context *_t);
+int maildir_save_begin(struct mail_save_context *ctx, struct istream *input);
+int maildir_save_continue(struct mail_save_context *ctx);
+void maildir_save_finish_keywords(struct mail_save_context *ctx);
+int maildir_save_finish(struct mail_save_context *ctx);
+void maildir_save_cancel(struct mail_save_context *ctx);
+
+struct maildir_filename *
+maildir_save_add(struct mail_save_context *_ctx, const char *tmp_fname,
+		 struct mail *src_mail) ATTR_NULL(3);
+void maildir_save_set_dest_basename(struct mail_save_context *ctx,
+				    struct maildir_filename *mf,
+				    const char *basename);
+void maildir_save_set_sizes(struct maildir_filename *mf,
+			    uoff_t size, uoff_t vsize);
+
+int maildir_save_file_get_size(struct mailbox_transaction_context *t,
+			       uint32_t seq, bool vsize, uoff_t *size_r);
+const char *maildir_save_file_get_path(struct mailbox_transaction_context *t,
+				       uint32_t seq);
+
+int maildir_transaction_save_commit_pre(struct mail_save_context *ctx);
+void maildir_transaction_save_commit_post(struct mail_save_context *ctx,
+					  struct mail_index_transaction_commit_result *result);
+void maildir_transaction_save_rollback(struct mail_save_context *ctx);
+
+int maildir_copy(struct mail_save_context *ctx, struct mail *mail);
+int maildir_transaction_copy_commit(struct maildir_copy_context *ctx);
+void maildir_transaction_copy_rollback(struct maildir_copy_context *ctx);
+
+#endif
diff --git a/src/lib-storage/index/maildir/maildir-sync-index.c b/src/lib-storage/index/maildir/maildir-sync-index.c
new file mode 100644
index 0000000..da0e40c
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-sync-index.c
@@ -0,0 +1,810 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "maildir-storage.h"
+#include "index-sync-changes.h"
+#include "maildir-uidlist.h"
+#include "maildir-keywords.h"
+#include "maildir-filename-flags.h"
+#include "maildir-sync.h"
+#include "mailbox-recent-flags.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+struct maildir_index_sync_context {
+        struct maildir_mailbox *mbox;
+	struct maildir_sync_context *maildir_sync_ctx;
+
+	struct mail_index_view *view;
+	struct mail_index_sync_ctx *sync_ctx;
+        struct maildir_keywords_sync_ctx *keywords_sync_ctx;
+	struct mail_index_transaction *trans;
+
+	struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
+	struct index_sync_changes_context *sync_changes;
+	enum mail_flags flags;
+	ARRAY_TYPE(keyword_indexes) keywords, idx_keywords;
+
+	uint32_t uid;
+	bool update_maildir_hdr_cur;
+
+	time_t start_time;
+	unsigned int flag_change_count, expunge_count, new_msgs_count;
+};
+
+struct maildir_keywords_sync_ctx *
+maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx)
+{
+	return ctx->keywords_sync_ctx;
+}
+
+void maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx,
+				     unsigned int count)
+{
+	ctx->new_msgs_count = count;
+}
+
+static bool
+maildir_expunge_is_valid_guid(struct maildir_index_sync_context *ctx,
+			      uint32_t uid, const char *filename,
+			      guid_128_t expunged_guid_128)
+{
+	guid_128_t guid_128;
+	const char *guid;
+
+	if (guid_128_is_empty(expunged_guid_128)) {
+		/* no GUID associated with expunge */
+		return TRUE;
+	}
+
+	T_BEGIN {
+		guid = maildir_uidlist_lookup_ext(ctx->mbox->uidlist, uid,
+						  MAILDIR_UIDLIST_REC_EXT_GUID);
+		if (guid == NULL)
+			guid = t_strcut(filename, *MAILDIR_INFO_SEP_S);
+		mail_generate_guid_128_hash(guid, guid_128);
+	} T_END;
+
+	if (memcmp(guid_128, expunged_guid_128, sizeof(guid_128)) == 0)
+		return TRUE;
+
+	mailbox_set_critical(&ctx->mbox->box,
+		"Expunged GUID mismatch for UID %u: %s vs %s",
+		ctx->uid, guid_128_to_string(guid_128),
+		guid_128_to_string(expunged_guid_128));
+	return FALSE;
+}
+
+static int maildir_expunge(struct maildir_mailbox *mbox, const char *path,
+			   struct maildir_index_sync_context *ctx)
+{
+	struct mailbox *box = &mbox->box;
+
+	ctx->expunge_count++;
+
+	if (unlink(path) == 0) {
+		mailbox_sync_notify(box, ctx->uid, MAILBOX_SYNC_TYPE_EXPUNGE);
+		return 1;
+	}
+	if (errno == ENOENT)
+		return 0;
+	if (UNLINK_EISDIR(errno))
+		return maildir_lose_unexpected_dir(box->storage, path);
+
+	mailbox_set_critical(&mbox->box, "unlink(%s) failed: %m", path);
+	return -1;
+}
+
+static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path,
+			      struct maildir_index_sync_context *ctx)
+{
+	struct mailbox *box = &mbox->box;
+	struct stat st;
+	const char *dir, *fname, *newfname, *newpath;
+	enum mail_index_sync_type sync_type;
+	uint8_t flags8;
+
+	ctx->flag_change_count++;
+
+	fname = strrchr(path, '/');
+	i_assert(fname != NULL);
+	fname++;
+	dir = t_strdup_until(path, fname);
+
+	i_assert(*fname != '\0');
+
+	/* get the current flags and keywords */
+	maildir_filename_flags_get(ctx->keywords_sync_ctx,
+				   fname, &ctx->flags, &ctx->keywords);
+
+	/* apply changes */
+	flags8 = ctx->flags;
+	index_sync_changes_apply(ctx->sync_changes, NULL,
+				 &flags8, &ctx->keywords, &sync_type);
+	ctx->flags = flags8;
+
+	/* and try renaming with the new name */
+	newfname = maildir_filename_flags_kw_set(ctx->keywords_sync_ctx, fname,
+						 ctx->flags, &ctx->keywords);
+	newpath = t_strconcat(dir, newfname, NULL);
+	if (strcmp(path, newpath) == 0) {
+		/* just make sure that the file still exists. avoid rename()
+		   here because it's slow on HFS. */
+		if (stat(path, &st) < 0) {
+			if (errno == ENOENT)
+				return 0;
+			mailbox_set_critical(box, "stat(%s) failed: %m", path);
+			return -1;
+		}
+	} else {
+		if (rename(path, newpath) < 0) {
+			if (errno == ENOENT)
+				return 0;
+			if (!ENOSPACE(errno) && errno != EACCES) {
+				mailbox_set_critical(box,
+					"rename(%s, %s) failed: %m",
+					path, newpath);
+			}
+			return -1;
+		}
+	}
+	mailbox_sync_notify(box, ctx->uid, index_sync_type_convert(sync_type));
+	return 1;
+}
+
+static int maildir_handle_uid_insertion(struct maildir_index_sync_context *ctx,
+					enum maildir_uidlist_rec_flag uflags,
+					const char *filename, uint32_t uid)
+{
+	int ret;
+
+	if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
+		/* partial syncing */
+		return 0;
+	}
+
+	/* most likely a race condition: we read the maildir, then someone else
+	   expunged messages and committed changes to index. so, this message
+	   shouldn't actually exist. */
+	if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) == 0) {
+		/* mark it racy and check in next sync */
+		ctx->mbox->maildir_hdr.cur_check_time = 0;
+		maildir_sync_set_racing(ctx->maildir_sync_ctx);
+		maildir_uidlist_add_flags(ctx->mbox->uidlist, filename,
+					  MAILDIR_UIDLIST_REC_FLAG_RACING);
+		return 0;
+	}
+
+	if (ctx->uidlist_sync_ctx == NULL) {
+		ret = maildir_uidlist_sync_init(ctx->mbox->uidlist,
+						MAILDIR_UIDLIST_SYNC_PARTIAL |
+						MAILDIR_UIDLIST_SYNC_KEEP_STATE,
+						&ctx->uidlist_sync_ctx);
+		if (ret <= 0)
+			return -1;
+	}
+
+	uflags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+	maildir_uidlist_sync_remove(ctx->uidlist_sync_ctx, filename);
+	ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
+					filename, uflags);
+	i_assert(ret > 0);
+
+	/* give the new UID to it immediately */
+	maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
+
+	i_warning("Maildir %s: Expunged message reappeared, giving a new UID "
+		  "(old uid=%u, file=%s)%s", mailbox_get_path(&ctx->mbox->box),
+		  uid, filename, !str_begins(filename, "msg.") ? "" :
+		  " (Your MDA is saving MH files into Maildir?)");
+	return 0;
+}
+
+int maildir_sync_index_begin(struct maildir_mailbox *mbox,
+			     struct maildir_sync_context *maildir_sync_ctx,
+			     struct maildir_index_sync_context **ctx_r)
+{
+	struct mailbox *_box = &mbox->box;
+	struct maildir_index_sync_context *ctx;
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	enum mail_index_sync_flags sync_flags;
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box);
+	/* don't drop recent messages if we're saving messages */
+	if (maildir_sync_ctx == NULL)
+		sync_flags &= ENUM_NEGATE(MAIL_INDEX_SYNC_FLAG_DROP_RECENT);
+
+	if (index_storage_expunged_sync_begin(_box, &sync_ctx, &view,
+					      &trans, sync_flags) < 0)
+		return -1;
+
+	ctx = i_new(struct maildir_index_sync_context, 1);
+	ctx->mbox = mbox;
+	ctx->maildir_sync_ctx = maildir_sync_ctx;
+	ctx->sync_ctx = sync_ctx;
+	ctx->view = view;
+	ctx->trans = trans;
+	ctx->keywords_sync_ctx =
+		maildir_keywords_sync_init(mbox->keywords, _box->index);
+	ctx->sync_changes =
+		index_sync_changes_init(ctx->sync_ctx, ctx->view, ctx->trans,
+					maildir_is_backend_readonly(mbox));
+	ctx->start_time = time(NULL);
+
+	*ctx_r = ctx;
+	return 0;
+}
+
+static bool
+maildir_index_header_has_changed(const struct maildir_index_header *old_hdr,
+				 const struct maildir_index_header *new_hdr)
+{
+#define DIR_DELAYED_REFRESH(hdr, name) \
+	((hdr)->name ## _check_time <= \
+		(hdr)->name ## _mtime + MAILDIR_SYNC_SECS)
+
+	if (old_hdr->new_mtime != new_hdr->new_mtime ||
+	    old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs ||
+	    old_hdr->cur_mtime != new_hdr->cur_mtime ||
+	    old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs ||
+	    old_hdr->uidlist_mtime != new_hdr->uidlist_mtime ||
+	    old_hdr->uidlist_mtime_nsecs != new_hdr->uidlist_mtime_nsecs ||
+	    old_hdr->uidlist_size != new_hdr->uidlist_size)
+		return TRUE;
+
+	return DIR_DELAYED_REFRESH(old_hdr, new) !=
+		DIR_DELAYED_REFRESH(new_hdr, new) ||
+		DIR_DELAYED_REFRESH(old_hdr, cur) !=
+		DIR_DELAYED_REFRESH(new_hdr, cur);
+}
+
+static void
+maildir_sync_index_update_ext_header(struct maildir_index_sync_context *ctx)
+{
+	struct maildir_mailbox *mbox = ctx->mbox;
+	const char *cur_path;
+	const void *data;
+	size_t data_size;
+	struct stat st;
+
+	cur_path = t_strconcat(mailbox_get_path(&mbox->box), "/cur", NULL);
+	if (ctx->update_maildir_hdr_cur && stat(cur_path, &st) == 0) {
+		if ((time_t)mbox->maildir_hdr.cur_check_time < st.st_mtime)
+			mbox->maildir_hdr.cur_check_time = st.st_mtime;
+		mbox->maildir_hdr.cur_mtime = st.st_mtime;
+		mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st);
+	}
+
+	mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id,
+				  &data, &data_size);
+	if (data_size != sizeof(mbox->maildir_hdr) ||
+	    maildir_index_header_has_changed(data, &mbox->maildir_hdr)) {
+		mail_index_update_header_ext(ctx->trans, mbox->maildir_ext_id,
+					     0, &mbox->maildir_hdr,
+					     sizeof(mbox->maildir_hdr));
+	}
+}
+
+static int maildir_sync_index_finish(struct maildir_index_sync_context *ctx,
+				     bool success)
+{
+	struct maildir_mailbox *mbox = ctx->mbox;
+	unsigned int time_diff;
+	int ret = success ? 0 : -1;
+
+	time_diff = time(NULL) - ctx->start_time;
+	if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) {
+		i_warning("Maildir %s: Synchronization took %u seconds "
+			  "(%u new msgs, %u flag change attempts, "
+			  "%u expunge attempts)",
+			  mailbox_get_path(&ctx->mbox->box), time_diff,
+			  ctx->new_msgs_count, ctx->flag_change_count,
+			  ctx->expunge_count);
+		mail_index_sync_no_warning(ctx->sync_ctx);
+	}
+
+	if (ret < 0)
+		mail_index_sync_rollback(&ctx->sync_ctx);
+	else {
+		maildir_sync_index_update_ext_header(ctx);
+
+		/* Set syncing_commit=TRUE so that if any sync callbacks try
+		   to access mails which got lost (eg. expunge callback trying
+		   to open the file which was just unlinked) we don't try to
+		   start a second index sync and crash. */
+		mbox->syncing_commit = TRUE;
+		if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
+			mailbox_set_index_error(&mbox->box);
+			ret = -1;
+		}
+		mbox->syncing_commit = FALSE;
+	}
+
+	index_storage_expunging_deinit(&mbox->box);
+	maildir_keywords_sync_deinit(&ctx->keywords_sync_ctx);
+	index_sync_changes_deinit(&ctx->sync_changes);
+	i_free(ctx);
+	return ret;
+}
+
+int maildir_sync_index_commit(struct maildir_index_sync_context **_ctx)
+{
+	struct maildir_index_sync_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+	return maildir_sync_index_finish(ctx, TRUE);
+}
+
+void maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx)
+{
+	struct maildir_index_sync_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+	(void)maildir_sync_index_finish(ctx, FALSE);
+}
+
+static int uint_cmp(const unsigned int *i1, const unsigned int *i2)
+{
+	if (*i1 < *i2)
+		return -1;
+	else if (*i1 > *i2)
+		return 1;
+	else
+		return 0;
+}
+
+static void
+maildir_sync_mail_keywords(struct maildir_index_sync_context *ctx, uint32_t seq)
+{
+	struct mailbox *box = &ctx->mbox->box;
+	struct mail_keywords *kw;
+	unsigned int i, j, old_count, new_count;
+	const unsigned int *old_indexes, *new_indexes;
+	bool have_indexonly_keywords;
+	int diff;
+
+	mail_index_lookup_keywords(ctx->view, seq, &ctx->idx_keywords);
+	if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords)) {
+		/* no changes - we should get here usually */
+		return;
+	}
+
+	/* sort the keywords */
+	array_sort(&ctx->idx_keywords, uint_cmp);
+	array_sort(&ctx->keywords, uint_cmp);
+
+	/* drop keywords that are in index-only. we don't want to touch them. */
+	old_indexes = array_get(&ctx->idx_keywords, &old_count);
+	have_indexonly_keywords = FALSE;
+	for (i = old_count; i > 0; i--) {
+		if (maildir_keywords_idx_char(ctx->keywords_sync_ctx,
+					      old_indexes[i-1]) == '\0') {
+			have_indexonly_keywords = TRUE;
+			array_delete(&ctx->idx_keywords, i-1, 1);
+		}
+	}
+
+	if (!have_indexonly_keywords) {
+		/* no index-only keywords found, so something changed.
+		   just replace them all. */
+		kw = mail_index_keywords_create_from_indexes(box->index,
+							     &ctx->keywords);
+		mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, kw);
+		mail_index_keywords_unref(&kw);
+		return;
+	}
+
+	/* check again if non-index-only keywords changed */
+	if (index_keyword_array_cmp(&ctx->keywords, &ctx->idx_keywords))
+		return;
+
+	/* we can't reset all the keywords or we'd drop indexonly keywords too.
+	   so first remove the unwanted keywords and then add back the wanted
+	   ones. we can get these lists easily by removing common elements
+	   from old and new keywords. */
+	new_indexes = array_get(&ctx->keywords, &new_count);
+	for (i = j = 0; i < old_count && j < new_count; ) {
+		diff = (int)old_indexes[i] - (int)new_indexes[j];
+		if (diff == 0) {
+			array_delete(&ctx->keywords, j, 1);
+			array_delete(&ctx->idx_keywords, i, 1);
+			old_indexes = array_get(&ctx->idx_keywords, &old_count);
+			new_indexes = array_get(&ctx->keywords, &new_count);
+		} else if (diff < 0) {
+			i++;
+		} else {
+			j++;
+		}
+	}
+
+	if (array_count(&ctx->idx_keywords) > 0) {
+		kw = mail_index_keywords_create_from_indexes(box->index,
+							     &ctx->idx_keywords);
+		mail_index_update_keywords(ctx->trans, seq, MODIFY_REMOVE, kw);
+		mail_index_keywords_unref(&kw);
+	}
+
+	if (array_count(&ctx->keywords) > 0) {
+		kw = mail_index_keywords_create_from_indexes(box->index,
+							     &ctx->keywords);
+		mail_index_update_keywords(ctx->trans, seq, MODIFY_ADD, kw);
+		mail_index_keywords_unref(&kw);
+	}
+}
+
+int maildir_sync_index(struct maildir_index_sync_context *ctx,
+		       bool partial)
+{
+	struct maildir_mailbox *mbox = ctx->mbox;
+	struct mail_index_view *view = ctx->view;
+	struct mail_index_view *view2;
+	struct maildir_uidlist_iter_ctx *iter;
+	struct mail_index_transaction *trans = ctx->trans;
+	const struct mail_index_header *hdr;
+	struct mail_index_header empty_hdr;
+	const struct mail_index_record *rec;
+	uint32_t seq, seq2, uid, prev_uid;
+        enum maildir_uidlist_rec_flag uflags;
+	const char *filename;
+	uint32_t uid_validity, next_uid, hdr_next_uid, first_recent_uid;
+	uint32_t first_uid;
+	unsigned int changes = 0;
+	int ret = 0;
+	time_t time_before_sync;
+	guid_128_t expunged_guid_128;
+	enum mail_flags private_flags_mask;
+	bool expunged, full_rescan = FALSE;
+
+	i_assert(!mbox->syncing_commit);
+
+	first_uid = 1;
+	hdr = mail_index_get_header(view);
+	uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
+	if (uid_validity != hdr->uid_validity &&
+	    uid_validity != 0 && hdr->uid_validity != 0) {
+		/* uidvalidity changed and index isn't being synced for the
+		   first time, reset the index so we can add all messages as
+		   new */
+		i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)",
+			  mailbox_get_path(&ctx->mbox->box),
+			  hdr->uid_validity, uid_validity);
+		mail_index_reset(trans);
+		mailbox_recent_flags_reset(&mbox->box);
+
+		first_uid = hdr->messages_count + 1;
+		i_zero(&empty_hdr);
+		empty_hdr.next_uid = 1;
+		hdr = &empty_hdr;
+	}
+	hdr_next_uid = hdr->next_uid;
+
+	ctx->mbox->box.tmp_sync_view = view;
+	private_flags_mask = mailbox_get_private_flags_mask(&mbox->box);
+	time_before_sync = time(NULL);
+	mbox->syncing_commit = TRUE;
+	seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1);
+	i_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS);
+	i_array_init(&ctx->idx_keywords, MAILDIR_MAX_KEYWORDS);
+	iter = maildir_uidlist_iter_init(mbox->uidlist);
+	while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) {
+		maildir_filename_flags_get(ctx->keywords_sync_ctx, filename,
+					   &ctx->flags, &ctx->keywords);
+
+		i_assert(uid > prev_uid);
+		prev_uid = uid;
+
+		/* the private flags are kept only in indexes. don't use them
+		   at all even for newly seen mails */
+		ctx->flags &= ENUM_NEGATE(private_flags_mask);
+
+	again:
+		seq++;
+		ctx->uid = uid;
+
+		if (seq > hdr->messages_count) {
+			if (uid < hdr_next_uid) {
+				if (maildir_handle_uid_insertion(ctx, uflags,
+								 filename,
+								 uid) < 0)
+					ret = -1;
+				seq--;
+				continue;
+			}
+
+			/* Trust uidlist recent flags only for newly added
+			   messages. When saving/copying messages with flags
+			   they're stored to cur/ and uidlist treats them
+			   as non-recent. */
+			if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) == 0) {
+				if (uid >= first_recent_uid)
+					first_recent_uid = uid + 1;
+			}
+
+			hdr_next_uid = uid + 1;
+			mail_index_append(trans, uid, &seq);
+			mail_index_update_flags(trans, seq, MODIFY_REPLACE,
+						ctx->flags);
+			if (array_count(&ctx->keywords) > 0) {
+				struct mail_keywords *kw;
+
+				kw = mail_index_keywords_create_from_indexes(
+					mbox->box.index, &ctx->keywords);
+				mail_index_update_keywords(trans, seq,
+							   MODIFY_REPLACE, kw);
+				mail_index_keywords_unref(&kw);
+			}
+			continue;
+		}
+
+		rec = mail_index_lookup(view, seq);
+		if (uid > rec->uid) {
+			/* already expunged (no point in showing guid in the
+			   expunge record anymore) */
+			mail_index_expunge(ctx->trans, seq);
+			goto again;
+		}
+
+		if (uid < rec->uid) {
+			if (maildir_handle_uid_insertion(ctx, uflags,
+							 filename, uid) < 0)
+				ret = -1;
+			seq--;
+			continue;
+		}
+
+		index_sync_changes_read(ctx->sync_changes, ctx->uid, &expunged,
+					expunged_guid_128);
+		if (expunged) {
+			if (!maildir_expunge_is_valid_guid(ctx, ctx->uid,
+							   filename,
+							   expunged_guid_128))
+				continue;
+			if (maildir_file_do(mbox, ctx->uid,
+					    maildir_expunge, ctx) >= 0) {
+				/* successful expunge */
+				mail_index_expunge(ctx->trans, seq);
+			}
+			if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
+				maildir_sync_notify(ctx->maildir_sync_ctx);
+			continue;
+		}
+
+		/* the private flags are stored only in indexes, keep them */
+		ctx->flags |= rec->flags & private_flags_mask;
+
+		if (index_sync_changes_have(ctx->sync_changes)) {
+			/* apply flag changes to maildir */
+			if (maildir_file_do(mbox, ctx->uid,
+					    maildir_sync_flags, ctx) < 0)
+				ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
+			if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0)
+				maildir_sync_notify(ctx->maildir_sync_ctx);
+		}
+
+		if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
+			/* partial syncing */
+			if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) {
+				/* we last saw this mail in new/, but it's
+				   not there anymore. possibly expunged,
+				   make sure. */
+				full_rescan = TRUE;
+			}
+			continue;
+		}
+
+		if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+			/* we haven't been able to update maildir with this
+			   record's flag changes. don't sync them. */
+			continue;
+		}
+
+		if (ctx->flags != (rec->flags & MAIL_FLAGS_NONRECENT)) {
+			mail_index_update_flags(trans, seq, MODIFY_REPLACE,
+						ctx->flags);
+		}
+
+		maildir_sync_mail_keywords(ctx, seq);
+	}
+	maildir_uidlist_iter_deinit(&iter);
+
+	if (!partial) {
+		/* expunge the rest */
+		for (seq++; seq <= hdr->messages_count; seq++)
+			mail_index_expunge(ctx->trans, seq);
+	}
+
+	/* add \Recent flags. use updated view so it contains newly
+	   appended messages. */
+	view2 = mail_index_transaction_open_updated_view(trans);
+	if (mail_index_lookup_seq_range(view2, first_recent_uid, (uint32_t)-1,
+					&seq, &seq2) && seq2 >= first_uid) {
+		if (seq < first_uid) {
+			/* UIDVALIDITY changed, skip over the old messages */
+			seq = first_uid;
+		}
+		mailbox_recent_flags_set_seqs(&mbox->box, view2, seq, seq2);
+	}
+	mail_index_view_close(&view2);
+
+	if (ctx->uidlist_sync_ctx != NULL) {
+		if (maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx,
+						TRUE) < 0)
+			ret = -1;
+	}
+
+	mailbox_sync_notify(&mbox->box, 0, 0);
+	ctx->mbox->box.tmp_sync_view = NULL;
+
+	/* check cur/ mtime later. if we came here from saving messages they
+	   could still be moved to cur/ directory. */
+	ctx->update_maildir_hdr_cur = TRUE;
+	mbox->maildir_hdr.cur_check_time = time_before_sync;
+
+	if (uid_validity == 0) {
+		uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity :
+			maildir_get_uidvalidity_next(mbox->box.list);
+		maildir_uidlist_set_uid_validity(mbox->uidlist, uid_validity);
+	}
+	maildir_uidlist_set_next_uid(mbox->uidlist, hdr_next_uid, FALSE);
+
+	if (uid_validity != hdr->uid_validity) {
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+
+	next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
+	if (hdr_next_uid < next_uid) {
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, next_uid),
+			&next_uid, sizeof(next_uid), FALSE);
+	}
+
+	i_assert(hdr->first_recent_uid <= first_recent_uid);
+	if (hdr->first_recent_uid < first_recent_uid) {
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, first_recent_uid),
+			&first_recent_uid, sizeof(first_recent_uid), FALSE);
+	}
+	array_free(&ctx->keywords);
+	array_free(&ctx->idx_keywords);
+	mbox->syncing_commit = FALSE;
+	return ret < 0 ? -1 : (full_rescan ? 0 : 1);
+}
+
+static unsigned int
+maildir_list_get_ext_id(struct maildir_mailbox *mbox,
+			struct mail_index_view *view)
+{
+	if (mbox->maildir_list_index_ext_id == (uint32_t)-1) {
+		mbox->maildir_list_index_ext_id =
+			mail_index_ext_register(mail_index_view_get_index(view),
+				"maildir", 0,
+				sizeof(struct maildir_list_index_record),
+				sizeof(uint32_t));
+	}
+	return mbox->maildir_list_index_ext_id;
+}
+
+int maildir_list_index_has_changed(struct mailbox *box,
+				   struct mail_index_view *list_view,
+				   uint32_t seq, bool quick,
+				   const char **reason_r)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+	const struct maildir_list_index_record *rec;
+	const void *data;
+	const char *root_dir, *new_dir, *cur_dir;
+	struct stat st;
+	uint32_t ext_id;
+	bool expunged;
+	int ret;
+
+	ret = index_storage_list_index_has_changed(box, list_view, seq,
+						   quick, reason_r);
+	if (ret != 0 || box->storage->set->mailbox_list_index_very_dirty_syncs)
+		return ret;
+	if (mbox->storage->set->maildir_very_dirty_syncs) {
+		/* we don't track cur/new directories with dirty syncs */
+		return 0;
+	}
+
+	ext_id = maildir_list_get_ext_id(mbox, list_view);
+	mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
+	rec = data;
+
+	if (rec == NULL) {
+		*reason_r = "Maildir record is missing";
+		return 1;
+	} else if (expunged) {
+		*reason_r = "Maildir record is expunged";
+		return 1;
+	} else if (rec->new_mtime == 0) {
+		/* not synced */
+		*reason_r = "Maildir record new_mtime=0";
+		return 1;
+	} else if (rec->cur_mtime == 0) {
+		/* dirty-synced */
+		*reason_r = "Maildir record cur_mtime=0";
+		return 1;
+	}
+
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX,
+				  &root_dir);
+	if (ret < 0)
+		return ret;
+	i_assert(ret > 0);
+
+	/* check if new/ changed */
+	new_dir = t_strconcat(root_dir, "/new", NULL);
+	if (stat(new_dir, &st) < 0) {
+		mailbox_set_critical(box, "stat(%s) failed: %m", new_dir);
+		return -1;
+	}
+	if ((time_t)rec->new_mtime != st.st_mtime) {
+		*reason_r = t_strdup_printf(
+			"Maildir new_mtime changed %u != %"PRIdTIME_T,
+			rec->new_mtime, st.st_mtime);
+		return 1;
+	}
+
+	/* check if cur/ changed */
+	cur_dir = t_strconcat(root_dir, "/cur", NULL);
+	if (stat(cur_dir, &st) < 0) {
+		mailbox_set_critical(box, "stat(%s) failed: %m", cur_dir);
+		return -1;
+	}
+	if ((time_t)rec->cur_mtime != st.st_mtime) {
+		*reason_r = t_strdup_printf(
+			"Maildir cur_mtime changed %u != %"PRIdTIME_T,
+			rec->cur_mtime, st.st_mtime);
+		return 1;
+	}
+	return 0;
+}
+
+void maildir_list_index_update_sync(struct mailbox *box,
+				    struct mail_index_transaction *trans,
+				    uint32_t seq)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+	struct mail_index_view *list_view;
+	const struct maildir_index_header *mhdr = &mbox->maildir_hdr;
+	const struct maildir_list_index_record *old_rec;
+	struct maildir_list_index_record new_rec;
+	const void *data;
+	uint32_t ext_id;
+	bool expunged;
+
+	index_storage_list_index_update_sync(box, trans, seq);
+	if (mbox->storage->set->maildir_very_dirty_syncs) {
+		/* we don't track cur/new directories with dirty syncs */
+		return;
+	}
+
+	/* get the current record */
+	list_view = mail_index_transaction_get_view(trans);
+	ext_id = maildir_list_get_ext_id(mbox, list_view);
+	mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
+	if (expunged)
+		return;
+	old_rec = data;
+
+	i_zero(&new_rec);
+	if (mhdr->new_check_time <= mhdr->new_mtime + MAILDIR_SYNC_SECS ||
+	    mhdr->cur_check_time <= mhdr->cur_mtime + MAILDIR_SYNC_SECS) {
+		/* dirty, we need a refresh next time */
+	} else {
+		new_rec.new_mtime = mhdr->new_mtime;
+		new_rec.cur_mtime = mhdr->cur_mtime;
+	}
+
+	if (old_rec == NULL ||
+	    memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0)
+		mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL);
+}
diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c
new file mode 100644
index 0000000..0814415
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-sync.c
@@ -0,0 +1,1132 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+/*
+   Here's a description of how we handle Maildir synchronization and
+   it's problems:
+
+   We want to be as efficient as we can. The most efficient way to
+   check if changes have occurred is to stat() the new/ and cur/
+   directories and uidlist file - if their mtimes haven't changed,
+   there's no changes and we don't need to do anything.
+
+   Problem 1: Multiple changes can happen within a single second -
+   nothing guarantees that once we synced it, someone else didn't just
+   then make a modification. Such modifications wouldn't get noticed
+   until a new modification occurred later.
+
+   Problem 2: Syncing cur/ directory is much more costly than syncing
+   new/. Moving mails from new/ to cur/ will always change mtime of
+   cur/ causing us to sync it as well.
+
+   Problem 3: We may not be able to move mail from new/ to cur/
+   because we're out of quota, or simply because we're accessing a
+   read-only mailbox.
+
+
+   MAILDIR_SYNC_SECS
+   -----------------
+
+   Several checks below use MAILDIR_SYNC_SECS, which should be maximum
+   clock drift between all computers accessing the maildir (eg. via
+   NFS), rounded up to next second. Our default is 1 second, since
+   everyone should be using NTP.
+
+   Note that setting it to 0 works only if there's only one computer
+   accessing the maildir. It's practically impossible to make two
+   clocks _exactly_ synchronized.
+
+   It might be possible to only use file server's clock by looking at
+   the atime field, but I don't know how well that would actually work.
+
+   cur directory
+   -------------
+
+   We have dirty_cur_time variable which is set to cur/ directory's
+   mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
+   synchronized the directory.
+
+   When dirty_cur_time is non-zero, we don't synchronize the cur/
+   directory until
+
+      a) cur/'s mtime changes
+      b) opening a mail fails with ENOENT
+      c) time() > dirty_cur_time + MAILDIR_SYNC_SECS
+
+   This allows us to modify the maildir multiple times without having
+   to sync it at every change. The sync will eventually be done to
+   make sure we didn't miss any external changes.
+
+   The dirty_cur_time is set when:
+
+      - we change message flags
+      - we expunge messages
+      - we move mail from new/ to cur/
+      - we sync cur/ directory and it's mtime is >= time() - MAILDIR_SYNC_SECS
+
+   It's unset when we do the final syncing, ie. when mtime is
+   older than time() - MAILDIR_SYNC_SECS.
+
+   new directory
+   -------------
+
+   If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
+   it. dirty_cur_time-like feature might save us a few syncs, but
+   that might break a client which saves a mail in one connection and
+   tries to fetch it in another one. new/ directory is almost always
+   empty, so syncing it should be very fast anyway. Actually this can
+   still happen if we sync only new/ dir while another client is also
+   moving mails from it to cur/ - it takes us a while to see them.
+   That's pretty unlikely to happen however, and only way to fix it
+   would be to always synchronize cur/ after new/.
+
+   Normally we move all mails from new/ to cur/ whenever we sync it. If
+   it's not possible for some reason, we mark the mail with "probably
+   exists in new/ directory" flag.
+
+   If rename() still fails because of ENOSPC or EDQUOT, we still save
+   the flag changes in index with dirty-flag on. When moving the mail
+   to cur/ directory, or when we notice it's already moved there, we
+   apply the flag changes to the filename, rename it and remove the
+   dirty flag. If there's dirty flags, this should be tried every time
+   after expunge or when closing the mailbox.
+
+   uidlist
+   -------
+
+   This file contains UID <-> filename mappings. It's updated only when
+   new mail arrives, so it may contain filenames that have already been
+   deleted. Updating is done by getting uidlist.lock file, writing the
+   whole uidlist into it and rename()ing it over the old uidlist. This
+   means there's no need to lock the file for reading.
+
+   Whenever uidlist is rewritten, it's mtime must be larger than the old
+   one's. Use utime() before rename() if needed. Note that inode checking
+   wouldn't have been sufficient as inode numbers can be reused.
+
+   This file is usually read the first time you need to know filename for
+   given UID. After that it's not re-read unless new mails come that we
+   don't know about.
+
+   broken clients
+   --------------
+
+   Originally the middle identifier in Maildir filename was specified
+   only as <process id>_<delivery counter>. That however created a
+   problem with randomized PIDs which made it possible that the same
+   PID was reused within one second.
+
+   So if within one second a mail was delivered, MUA moved it to cur/
+   and another mail was delivered by a new process using same PID as
+   the first one, we likely ended up overwriting the first mail when
+   the second mail was moved over it.
+
+   Nowadays everyone should be giving a bit more specific identifier,
+   for example include microseconds in it which Dovecot does.
+
+   There's a simple way to prevent this from happening in some cases:
+   Don't move the mail from new/ to cur/ if it's mtime is >= time() -
+   MAILDIR_SYNC_SECS. The second delivery's link() call then fails
+   because the file is already in new/, and it will then use a
+   different filename. There's a few problems with this however:
+
+      - it requires extra stat() call which is unneeded extra I/O
+      - another MUA might still move the mail to cur/
+      - if first file's flags are modified by either Dovecot or another
+        MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
+	but that'd be ugly)
+
+   Because this is useful only for very few people and it requires
+   extra I/O, I decided not to implement this. It should be however
+   quite easy to do since we need to be able to deal with files in new/
+   in any case.
+
+   It's also possible to never accidentally overwrite a mail by using
+   link() + unlink() rather than rename(). This however isn't very
+   good idea as it introduces potential race conditions when multiple
+   clients are accessing the mailbox:
+
+   Trying to move the same mail from new/ to cur/ at the same time:
+
+      a) Client 1 uses slightly different filename than client 2,
+         for example one sets read-flag on but the other doesn't.
+	 You have the same mail duplicated now.
+
+      b) Client 3 sees the mail between Client 1's and 2's link() calls
+         and changes it's flag. You have the same mail duplicated now.
+
+   And it gets worse when they're unlink()ing in cur/ directory:
+
+      c) Client 1 changes mails's flag and client 2 changes it back
+         between 1's link() and unlink(). The mail is now expunged.
+
+      d) If you try to deal with the duplicates by unlink()ing another
+         one of them, you might end up unlinking both of them.
+
+   So, what should we do then if we notice a duplicate? First of all,
+   it might not be a duplicate at all, readdir() might have just
+   returned it twice because it was just renamed. What we should do is
+   create a completely new base name for it and rename() it to that.
+   If the call fails with ENOENT, it only means that it wasn't a
+   duplicate after all.
+*/
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "buffer.h"
+#include "hash.h"
+#include "str.h"
+#include "eacces-error.h"
+#include "nfs-workarounds.h"
+#include "maildir-storage.h"
+#include "maildir-uidlist.h"
+#include "maildir-filename.h"
+#include "maildir-sync.h"
+
+#include <stdio.h>
+#include <stddef.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/stat.h>
+
+#define MAILDIR_FILENAME_FLAG_FOUND 128
+
+/* When rename()ing many files from new/ to cur/, it's possible that next
+   readdir() skips some files. we don't of course wish to lose them, so we
+   go and rescan the new/ directory again from beginning until no files are
+   left. This value is just an optimization to avoid checking the directory
+   twice needlessly. usually only NFS is the problem case. 1 is the safest
+   bet here, but I guess 5 will do just fine too. */
+#define MAILDIR_RENAME_RESCAN_COUNT 5
+
+/* This is mostly to avoid infinite looping when rename() destination already
+   exists as the hard link of the file itself. */
+#define MAILDIR_SCAN_DIR_MAX_COUNT 5
+
+#define DUPE_LINKS_DELETE_SECS 30
+
+enum maildir_scan_why {
+	WHY_FORCED	= 0x01,
+	WHY_FIRSTSYNC	= 0x02,
+	WHY_NEWCHANGED	= 0x04,
+	WHY_CURCHANGED	= 0x08,
+	WHY_DROPRECENT	= 0x10,
+	WHY_FINDRECENT	= 0x20,
+	WHY_DELAYEDNEW	= 0x40,
+	WHY_DELAYEDCUR	= 0x80
+};
+
+struct maildir_sync_context {
+        struct maildir_mailbox *mbox;
+	const char *new_dir, *cur_dir;
+
+	enum mailbox_sync_flags flags;
+	time_t last_touch, last_notify;
+
+	struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
+	struct maildir_index_sync_context *index_sync_ctx;
+
+	bool partial:1;
+	bool locked:1;
+	bool racing:1;
+};
+
+void maildir_sync_set_racing(struct maildir_sync_context *ctx)
+{
+	ctx->racing = TRUE;
+}
+
+void maildir_sync_notify(struct maildir_sync_context *ctx)
+{
+	time_t now;
+
+	if (ctx == NULL) {
+		/* we got here from maildir-save.c. it has no
+		   maildir_sync_context,  */
+		return;
+	}
+
+	now = time(NULL);
+	if (now - ctx->last_touch > MAILDIR_LOCK_TOUCH_SECS && ctx->locked) {
+		(void)maildir_uidlist_lock_touch(ctx->mbox->uidlist);
+		ctx->last_touch = now;
+	}
+	if (now - ctx->last_notify > MAIL_STORAGE_STAYALIVE_SECS) {
+		struct mailbox *box = &ctx->mbox->box;
+
+		if (box->storage->callbacks.notify_ok != NULL) {
+			box->storage->callbacks.
+				notify_ok(box, "Hang in there..",
+					  box->storage->callback_context);
+		}
+		ctx->last_notify = now;
+	}
+}
+
+static struct maildir_sync_context *
+maildir_sync_context_new(struct maildir_mailbox *mbox,
+			 enum mailbox_sync_flags flags)
+{
+        struct maildir_sync_context *ctx;
+
+	ctx = t_new(struct maildir_sync_context, 1);
+	ctx->mbox = mbox;
+	ctx->new_dir = t_strconcat(mailbox_get_path(&mbox->box), "/new", NULL);
+	ctx->cur_dir = t_strconcat(mailbox_get_path(&mbox->box), "/cur", NULL);
+	ctx->last_touch = ioloop_time;
+	ctx->last_notify = ioloop_time;
+	ctx->flags = flags;
+	return ctx;
+}
+
+static void maildir_sync_deinit(struct maildir_sync_context *ctx)
+{
+	if (ctx->uidlist_sync_ctx != NULL)
+		(void)maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, FALSE);
+	if (ctx->index_sync_ctx != NULL)
+		maildir_sync_index_rollback(&ctx->index_sync_ctx);
+	if (ctx->mbox->storage->storage.rebuild_list_index)
+		(void)mail_storage_list_index_rebuild_and_set_uncorrupted(&ctx->mbox->storage->storage);
+}
+
+static int maildir_fix_duplicate(struct maildir_sync_context *ctx,
+				 const char *dir, const char *fname2)
+{
+	const char *fname1, *path1, *path2;
+	const char *new_fname, *new_path;
+	struct stat st1, st2;
+	uoff_t size;
+
+	fname1 = maildir_uidlist_sync_get_full_filename(ctx->uidlist_sync_ctx,
+							fname2);
+	i_assert(fname1 != NULL);
+
+	path1 = t_strconcat(dir, "/", fname1, NULL);
+	path2 = t_strconcat(dir, "/", fname2, NULL);
+
+	if (stat(path1, &st1) < 0 || stat(path2, &st2) < 0) {
+		/* most likely the files just don't exist anymore.
+		   don't really care about other errors much. */
+		return 0;
+	}
+	if (st1.st_ino == st2.st_ino &&
+	    CMP_DEV_T(st1.st_dev, st2.st_dev)) {
+		/* Files are the same. this means either a race condition
+		   between stat() calls, or that the files were link()ed. */
+		if (st1.st_nlink > 1 && st2.st_nlink == st1.st_nlink &&
+		    st1.st_ctime == st2.st_ctime &&
+		    st1.st_ctime < ioloop_time - DUPE_LINKS_DELETE_SECS) {
+			/* The file has hard links and it hasn't had any
+			   changes (such as renames) for a while, so this
+			   isn't a race condition.
+
+			   rename()ing one file on top of the other would fix
+			   this safely, except POSIX decided that rename()
+			   doesn't work that way. So we'll have unlink() one
+			   and hope that another process didn't just decide to
+			   unlink() the other (uidlist lock prevents this from
+			   happening) */
+			if (i_unlink(path2) == 0)
+				i_warning("Unlinked a duplicate: %s", path2);
+		}
+		return 0;
+	}
+
+	new_fname = maildir_filename_generate();
+	/* preserve S= and W= sizes if they're available.
+	   (S=size is required for zlib plugin to work) */
+	if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_FILE_SIZE, &size)) {
+		new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T,
+			new_fname, MAILDIR_EXTRA_FILE_SIZE, size);
+	}
+	if (maildir_filename_get_size(fname2, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) {
+		new_fname = t_strdup_printf("%s,%c=%"PRIuUOFF_T,
+			new_fname, MAILDIR_EXTRA_VIRTUAL_SIZE, size);
+	}
+	new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box),
+			       "/new/", new_fname, NULL);
+
+	if (rename(path2, new_path) == 0)
+		i_warning("Fixed a duplicate: %s -> %s", path2, new_fname);
+	else if (errno != ENOENT) {
+		mailbox_set_critical(&ctx->mbox->box,
+			"Couldn't fix a duplicate: rename(%s, %s) failed: %m",
+			path2, new_path);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+maildir_rename_empty_basename(struct maildir_sync_context *ctx,
+			      const char *dir, const char *fname)
+{
+	const char *old_path, *new_fname, *new_path;
+
+	old_path = t_strconcat(dir, "/", fname, NULL);
+	new_fname = maildir_filename_generate();
+	new_path = t_strconcat(mailbox_get_path(&ctx->mbox->box),
+			       "/new/", new_fname, NULL);
+	if (rename(old_path, new_path) == 0)
+		i_warning("Fixed broken filename: %s -> %s", old_path, new_fname);
+	else if (errno != ENOENT) {
+		mailbox_set_critical(&ctx->mbox->box,
+			"Couldn't fix a broken filename: rename(%s, %s) failed: %m",
+			old_path, new_path);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+maildir_stat(struct maildir_mailbox *mbox, const char *path, struct stat *st_r)
+{
+	struct mailbox *box = &mbox->box;
+	int i;
+
+	for (i = 0;; i++) {
+		if (nfs_safe_stat(path, st_r) == 0)
+			return 0;
+		if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT)
+			break;
+
+		if (!maildir_set_deleted(box))
+			return -1;
+		/* try again */
+	}
+
+	mailbox_set_critical(box, "stat(%s) failed: %m", path);
+	return -1;
+}
+
+static int
+maildir_scan_dir(struct maildir_sync_context *ctx, bool new_dir, bool final,
+		 enum maildir_scan_why why)
+{
+	const char *path;
+	DIR *dirp;
+	string_t *src, *dest;
+	struct dirent *dp;
+	struct stat st;
+	enum maildir_uidlist_rec_flag flags;
+	unsigned int time_diff, i, readdir_count = 0, move_count = 0;
+	time_t start_time;
+	int ret = 1;
+	bool move_new, dir_changed = FALSE;
+
+	path = new_dir ? ctx->new_dir : ctx->cur_dir;
+	for (i = 0;; i++) {
+		dirp = opendir(path);
+		if (dirp != NULL)
+			break;
+
+		if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) {
+			if (errno == EACCES) {
+				mailbox_set_critical(&ctx->mbox->box, "%s",
+					eacces_error_get("opendir", path));
+			} else {
+				mailbox_set_critical(&ctx->mbox->box,
+					"opendir(%s) failed: %m", path);
+			}
+			return -1;
+		}
+
+		if (!maildir_set_deleted(&ctx->mbox->box))
+			return -1;
+		/* try again */
+	}
+
+#ifdef HAVE_DIRFD
+	if (fstat(dirfd(dirp), &st) < 0) {
+		mailbox_set_critical(&ctx->mbox->box,
+			"fstat(%s) failed: %m", path);
+		(void)closedir(dirp);
+		return -1;
+	}
+#else
+	if (maildir_stat(ctx->mbox, path, &st) < 0) {
+		(void)closedir(dirp);
+		return -1;
+	}
+#endif
+
+	start_time = time(NULL);
+	if (new_dir) {
+		ctx->mbox->maildir_hdr.new_check_time = start_time;
+		ctx->mbox->maildir_hdr.new_mtime = st.st_mtime;
+		ctx->mbox->maildir_hdr.new_mtime_nsecs = ST_MTIME_NSEC(st);
+	} else {
+		ctx->mbox->maildir_hdr.cur_check_time = start_time;
+		ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime;
+		ctx->mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st);
+	}
+
+	src = t_str_new(1024);
+	dest = t_str_new(1024);
+
+	move_new = new_dir && ctx->locked &&
+		((ctx->mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0 ||
+		 ctx->mbox->storage->set->maildir_empty_new);
+
+	errno = 0;
+	for (; (dp = readdir(dirp)) != NULL; errno = 0) {
+		if (dp->d_name[0] == '.')
+			continue;
+
+		if (dp->d_name[0] == MAILDIR_INFO_SEP) {
+			/* don't even try to use file with empty base name */
+			if (maildir_rename_empty_basename(ctx, path,
+							  dp->d_name) < 0)
+				break;
+			continue;
+		}
+
+		flags = 0;
+		if (move_new) {
+			i_assert(dp->d_name[0] != '\0');
+
+			str_truncate(src, 0);
+			str_truncate(dest, 0);
+			str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
+			str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
+			if (strchr(dp->d_name, MAILDIR_INFO_SEP) == NULL) {
+				str_append(dest, MAILDIR_FLAGS_FULL_SEP);
+			}
+			if (rename(str_c(src), str_c(dest)) == 0) {
+				/* we moved it - it's \Recent for us */
+				dir_changed = TRUE;
+				move_count++;
+				flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
+					MAILDIR_UIDLIST_REC_FLAG_RECENT;
+			} else if (ENOTFOUND(errno)) {
+				/* someone else moved it already */
+				dir_changed = TRUE;
+				move_count++;
+				flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
+					MAILDIR_UIDLIST_REC_FLAG_RECENT;
+			} else if (ENOSPACE(errno) || errno == EACCES) {
+				/* not enough disk space / read-only maildir,
+				   leave here */
+				flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+				move_new = FALSE;
+			} else {
+				flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+				mailbox_set_critical(&ctx->mbox->box,
+					"rename(%s, %s) failed: %m",
+					str_c(src), str_c(dest));
+			}
+			if ((move_count % MAILDIR_SLOW_MOVE_COUNT) == 0)
+				maildir_sync_notify(ctx);
+		} else if (new_dir) {
+			flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+				MAILDIR_UIDLIST_REC_FLAG_RECENT;
+		}
+
+		readdir_count++;
+		if ((readdir_count % MAILDIR_SLOW_CHECK_COUNT) == 0)
+			maildir_sync_notify(ctx);
+
+		ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
+						dp->d_name, flags);
+		if (ret <= 0) {
+			if (ret < 0)
+				break;
+
+			/* possibly duplicate - try fixing it */
+			T_BEGIN {
+				ret = maildir_fix_duplicate(ctx, path,
+							    dp->d_name);
+			} T_END;
+			if (ret < 0)
+				break;
+		}
+	}
+
+#ifdef __APPLE__
+	if (errno == EINVAL && move_count > 0 && !final) {
+		/* OS X HFS+: readdir() fails sometimes when rename()
+		   have been done. */
+		move_count = MAILDIR_RENAME_RESCAN_COUNT + 1;
+	} else
+#endif
+
+	if (errno != 0) {
+		mailbox_set_critical(&ctx->mbox->box,
+				     "readdir(%s) failed: %m", path);
+		ret = -1;
+	}
+
+	if (closedir(dirp) < 0) {
+		mailbox_set_critical(&ctx->mbox->box,
+				     "closedir(%s) failed: %m", path);
+		ret = -1;
+	}
+
+	if (dir_changed) {
+		/* save the exact new times. the new mtimes should be >=
+		   "start_time", but just in case something weird happens and
+		   mtime doesn't update, use "start_time". */
+		if (stat(ctx->new_dir, &st) == 0) {
+			ctx->mbox->maildir_hdr.new_check_time =
+				I_MAX(st.st_mtime, start_time);
+			ctx->mbox->maildir_hdr.new_mtime = st.st_mtime;
+			ctx->mbox->maildir_hdr.new_mtime_nsecs =
+				ST_MTIME_NSEC(st);
+		}
+		if (stat(ctx->cur_dir, &st) == 0) {
+			ctx->mbox->maildir_hdr.new_check_time =
+				I_MAX(st.st_mtime, start_time);
+			ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime;
+			ctx->mbox->maildir_hdr.cur_mtime_nsecs =
+				ST_MTIME_NSEC(st);
+		}
+	}
+	time_diff = time(NULL) - start_time;
+	if (time_diff >= MAILDIR_SYNC_TIME_WARN_SECS) {
+		i_warning("Maildir: Scanning %s took %u seconds "
+			  "(%u readdir()s, %u rename()s to cur/, why=0x%x)",
+			  path, time_diff, readdir_count, move_count, why);
+	}
+
+	return ret < 0 ? -1 :
+		(move_count <= MAILDIR_RENAME_RESCAN_COUNT || final ? 0 : 1);
+}
+
+static void maildir_sync_get_header(struct maildir_mailbox *mbox)
+{
+	const void *data;
+	size_t data_size;
+
+	mail_index_get_header_ext(mbox->box.view, mbox->maildir_ext_id,
+				  &data, &data_size);
+	if (data_size == 0) {
+		/* header doesn't exist */
+	} else {
+		memcpy(&mbox->maildir_hdr, data,
+		       I_MIN(sizeof(mbox->maildir_hdr), data_size));
+	}
+}
+
+int maildir_sync_header_refresh(struct maildir_mailbox *mbox)
+{
+	if (mail_index_refresh(mbox->box.index) < 0) {
+		mailbox_set_index_error(&mbox->box);
+		return -1;
+	}
+	maildir_sync_get_header(mbox);
+	return 0;
+}
+
+static int maildir_sync_quick_check(struct maildir_mailbox *mbox, bool undirty,
+				    const char *new_dir, const char *cur_dir,
+				    bool *new_changed_r, bool *cur_changed_r,
+				    enum maildir_scan_why *why_r)
+{
+#define DIR_DELAYED_REFRESH(hdr, name) \
+	((hdr)->name ## _check_time <= \
+		(hdr)->name ## _mtime + MAILDIR_SYNC_SECS && \
+	 (undirty || \
+	  (time_t)(hdr)->name ## _check_time < ioloop_time - MAILDIR_SYNC_SECS))
+
+#define DIR_MTIME_CHANGED(st, hdr, name) \
+	((st).st_mtime != (time_t)(hdr)->name ## _mtime || \
+	 !ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), (hdr)->name ## _mtime_nsecs))
+
+	struct maildir_index_header *hdr = &mbox->maildir_hdr;
+	struct stat new_st, cur_st;
+	bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE;
+
+	*why_r = 0;
+
+	if (mbox->maildir_hdr.new_mtime == 0) {
+		maildir_sync_get_header(mbox);
+		if (mbox->maildir_hdr.new_mtime == 0) {
+			/* first sync */
+			*why_r |= WHY_FIRSTSYNC;
+			*new_changed_r = *cur_changed_r = TRUE;
+			return 0;
+		}
+	}
+
+	*new_changed_r = *cur_changed_r = FALSE;
+
+	/* try to avoid stat()ing by first checking delayed changes */
+	if (DIR_DELAYED_REFRESH(hdr, new) ||
+	    (DIR_DELAYED_REFRESH(hdr, cur) &&
+	     !mbox->storage->set->maildir_very_dirty_syncs)) {
+		/* refresh index and try again */
+		if (maildir_sync_header_refresh(mbox) < 0)
+			return -1;
+		refreshed = TRUE;
+
+		if (DIR_DELAYED_REFRESH(hdr, new)) {
+			*why_r |= WHY_DELAYEDNEW;
+			*new_changed_r = TRUE;
+		}
+		if (DIR_DELAYED_REFRESH(hdr, cur) &&
+		    !mbox->storage->set->maildir_very_dirty_syncs) {
+			*why_r |= WHY_DELAYEDCUR;
+			*cur_changed_r = TRUE;
+		}
+		if (*new_changed_r && *cur_changed_r)
+			return 0;
+	}
+
+	if (!*new_changed_r) {
+		if (maildir_stat(mbox, new_dir, &new_st) < 0)
+			return -1;
+		check_new = TRUE;
+	}
+	if (!*cur_changed_r) {
+		if (maildir_stat(mbox, cur_dir, &cur_st) < 0)
+			return -1;
+		check_cur = TRUE;
+	}
+
+	for (;;) {
+		if (check_new) {
+			*new_changed_r = DIR_MTIME_CHANGED(new_st, hdr, new);
+			if (*new_changed_r)
+				*why_r |= WHY_NEWCHANGED;
+		}
+		if (check_cur) {
+			*cur_changed_r = DIR_MTIME_CHANGED(cur_st, hdr, cur);
+			if (*cur_changed_r)
+				*why_r |= WHY_CURCHANGED;
+		}
+
+		if ((!*new_changed_r && !*cur_changed_r) || refreshed)
+			break;
+
+		/* refresh index and try again */
+		if (maildir_sync_header_refresh(mbox) < 0)
+			return -1;
+		refreshed = TRUE;
+	}
+
+	return 0;
+}
+
+static void maildir_sync_update_next_uid(struct maildir_mailbox *mbox)
+{
+	const struct mail_index_header *hdr;
+	uint32_t uid_validity;
+
+	hdr = mail_index_get_header(mbox->box.view);
+	if (hdr->uid_validity == 0)
+		return;
+
+	uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
+	if (uid_validity == hdr->uid_validity || uid_validity == 0) {
+		/* make sure uidlist's next_uid is at least as large as
+		   index file's. typically this happens only if uidlist gets
+		   deleted. */
+		maildir_uidlist_set_uid_validity(mbox->uidlist,
+						 hdr->uid_validity);
+		maildir_uidlist_set_next_uid(mbox->uidlist,
+					     hdr->next_uid, FALSE);
+	}
+}
+
+static bool
+have_recent_messages(struct maildir_sync_context *ctx, bool seen_changes)
+{
+	const struct mail_index_header *hdr;
+	uint32_t next_uid;
+
+	hdr = mail_index_get_header(ctx->mbox->box.view);
+	if (!seen_changes) {
+		/* index is up to date. get the next-uid from it */
+		next_uid = hdr->next_uid;
+	} else {
+		(void)maildir_uidlist_refresh(ctx->mbox->uidlist);
+		next_uid = maildir_uidlist_get_next_uid(ctx->mbox->uidlist);
+	}
+	return hdr->first_recent_uid < next_uid;
+}
+
+static int maildir_sync_get_changes(struct maildir_sync_context *ctx,
+				    bool *new_changed_r, bool *cur_changed_r,
+				    enum maildir_scan_why *why_r)
+{
+	struct maildir_mailbox *mbox = ctx->mbox;
+	enum mail_index_sync_flags flags = 0;
+	bool undirty = (ctx->flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0;
+
+	*why_r = 0;
+
+	if (maildir_sync_quick_check(mbox, undirty, ctx->new_dir, ctx->cur_dir,
+				     new_changed_r, cur_changed_r, why_r) < 0)
+		return -1;
+
+	/* if there are files in new/, we'll need to move them. we'll check
+	   this by seeing if we have any recent messages */
+	if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) {
+		if (!*new_changed_r && have_recent_messages(ctx, FALSE)) {
+			*new_changed_r = TRUE;
+			*why_r |= WHY_DROPRECENT;
+		}
+	} else if (*new_changed_r) {
+		/* if recent messages have been externally deleted from new/,
+		   we need to get them out of index. this requires that
+		   we make sure they weren't just moved to cur/. */
+		if (!*cur_changed_r && have_recent_messages(ctx, TRUE)) {
+			*cur_changed_r = TRUE;
+			*why_r |= WHY_FINDRECENT;
+		}
+	}
+
+	if (*new_changed_r || *cur_changed_r)
+		return 1;
+
+	if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0)
+		flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
+
+	if (mbox->synced) {
+		/* refresh index only after the first sync, i.e. avoid wasting
+		   time on refreshing it immediately after it was just opened */
+		mail_index_refresh(mbox->box.index);
+	}
+	return mail_index_sync_have_any(mbox->box.index, flags) ? 1 : 0;
+}
+
+static int ATTR_NULL(3)
+maildir_sync_context(struct maildir_sync_context *ctx, bool forced,
+		     uint32_t *find_uid, bool *lost_files_r)
+{
+	enum maildir_uidlist_sync_flags sync_flags;
+	enum maildir_uidlist_rec_flag flags;
+	bool new_changed, cur_changed, lock_failure;
+	const char *fname;
+	enum maildir_scan_why why;
+	int ret;
+
+	*lost_files_r = FALSE;
+
+	if (forced) {
+		new_changed = cur_changed = TRUE;
+		why = WHY_FORCED;
+	} else {
+		ret = maildir_sync_get_changes(ctx, &new_changed, &cur_changed,
+					       &why);
+		if (ret <= 0)
+			return ret;
+	}
+
+	/*
+	   Locking, locking, locking.. Wasn't maildir supposed to be lockless?
+
+	   We can get here either as beginning a real maildir sync, or when
+	   committing changes to maildir but a file was lost (maybe renamed).
+
+	   So, we're going to need two locks. One for index and one for
+	   uidlist. To avoid deadlocking do the uidlist lock always first.
+
+	   uidlist is needed only for figuring out UIDs for newly seen files,
+	   so theoretically we wouldn't need to lock it unless there are new
+	   files. It has a few problems though, assuming the index lock didn't
+	   already protect it (eg. in-memory indexes):
+
+	   1. Just because you see a new file which doesn't exist in uidlist
+	   file, doesn't mean that the file really exists anymore, or that
+	   your readdir() lists all new files. Meaning that this is possible:
+
+	     A: opendir(), readdir() -> new file ...
+	     -- new files are written to the maildir --
+	     B: opendir(), readdir() -> new file, lock uidlist,
+		readdir() -> another new file, rewrite uidlist, unlock
+	     A: ... lock uidlist, readdir() -> nothing left, rewrite uidlist,
+		unlock
+
+	   The second time running A didn't see the two new files. To handle
+	   this correctly, it must not remove the new unseen files from
+	   uidlist. This is possible to do, but adds extra complexity.
+
+	   2. If another process is rename()ing files while we are
+	   readdir()ing, it's possible that readdir() never lists some files,
+	   causing Dovecot to assume they were expunged. In next sync they
+	   would show up again, but client could have already been notified of
+	   that and they would show up under new UIDs, so the damage is
+	   already done.
+
+	   Both of the problems can be avoided if we simply lock the uidlist
+	   before syncing and keep it until sync is finished. Typically this
+	   would happen in any case, as there is the index lock..
+
+	   The second case is still a problem with external changes though,
+	   because maildir doesn't require any kind of locking. Luckily this
+	   problem rarely happens except under high amount of modifications.
+	*/
+
+	if (!cur_changed) {
+		ctx->partial = TRUE;
+		sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL;
+	} else {
+		ctx->partial = FALSE;
+		sync_flags = 0;
+		if (forced)
+			sync_flags |= MAILDIR_UIDLIST_SYNC_FORCE;
+		if ((ctx->flags & MAILBOX_SYNC_FLAG_FAST) != 0)
+			sync_flags |= MAILDIR_UIDLIST_SYNC_TRYLOCK;
+	}
+	ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags,
+					&ctx->uidlist_sync_ctx);
+	lock_failure = ret <= 0;
+	if (ret <= 0) {
+		struct mail_storage *storage = ctx->mbox->box.storage;
+
+		if (ret == 0) {
+			/* timeout */
+			return 0;
+		}
+		/* locking failed. sync anyway without locking so that it's
+		   possible to expunge messages when out of quota. */
+		if (forced) {
+			/* we're already forcing a sync, we're trying to find
+			   a message that was probably already expunged, don't
+			   loop for a long time trying to find it. */
+			return -1;
+		}
+		ret = maildir_uidlist_sync_init(ctx->mbox->uidlist, sync_flags |
+						MAILDIR_UIDLIST_SYNC_NOLOCK,
+						&ctx->uidlist_sync_ctx);
+		if (ret <= 0) {
+			i_assert(ret != 0);
+			return -1;
+		}
+
+		if (storage->callbacks.notify_no != NULL) {
+			storage->callbacks.notify_no(&ctx->mbox->box,
+				"Internal mailbox synchronization failure, "
+				"showing only old mails.",
+				storage->callback_context);
+		}
+	}
+	ctx->locked = maildir_uidlist_is_locked(ctx->mbox->uidlist);
+	if (!ctx->locked)
+		ctx->partial = TRUE;
+
+	if (!ctx->mbox->syncing_commit && (ctx->locked || lock_failure)) {
+		if (maildir_sync_index_begin(ctx->mbox, ctx,
+					     &ctx->index_sync_ctx) < 0)
+			return -1;
+	}
+
+	if (new_changed || cur_changed) {
+		/* if we're going to check cur/ dir our current logic requires
+		   that new/ dir is checked as well. it's a good idea anyway. */
+		unsigned int count = 0;
+		bool final = FALSE;
+
+		while ((ret = maildir_scan_dir(ctx, TRUE, final, why)) > 0) {
+			/* rename()d at least some files, which might have
+			   caused some other files to be missed. check again
+			   (see MAILDIR_RENAME_RESCAN_COUNT). */
+			if (++count >= MAILDIR_SCAN_DIR_MAX_COUNT)
+				final = TRUE;
+		}
+		if (ret < 0)
+			return -1;
+
+		if (cur_changed) {
+			if (maildir_scan_dir(ctx, FALSE, TRUE, why) < 0)
+				return -1;
+		}
+
+		maildir_sync_update_next_uid(ctx->mbox);
+
+		/* finish uidlist syncing, but keep it still locked */
+		maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
+	}
+
+	if (!ctx->locked) {
+		/* make sure we sync the maildir later */
+		ctx->mbox->maildir_hdr.new_mtime = 0;
+		ctx->mbox->maildir_hdr.cur_mtime = 0;
+	}
+
+	if (ctx->index_sync_ctx != NULL) {
+		/* NOTE: index syncing here might cause a re-sync due to
+		   files getting lost, so this function might be called
+		   reentrantly. */
+		ret = maildir_sync_index(ctx->index_sync_ctx, ctx->partial);
+		if (ret < 0)
+			maildir_sync_index_rollback(&ctx->index_sync_ctx);
+		else if (maildir_sync_index_commit(&ctx->index_sync_ctx) < 0)
+			return -1;
+
+		if (ret < 0)
+			return -1;
+		if (ret == 0)
+			*lost_files_r = TRUE;
+
+		i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist) ||
+			 lock_failure);
+	}
+
+	if (find_uid != NULL && *find_uid != 0) {
+		ret = maildir_uidlist_lookup(ctx->mbox->uidlist,
+					     *find_uid, &flags, &fname);
+		if (ret < 0)
+			return -1;
+		if (ret == 0) {
+			/* UID is expunged */
+			*find_uid = 0;
+		} else if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) {
+			*find_uid = 0;
+		} else {
+			/* we didn't find it, possibly expunged? */
+		}
+	}
+
+	return maildir_uidlist_sync_deinit(&ctx->uidlist_sync_ctx, TRUE);
+}
+
+int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid,
+			enum maildir_uidlist_rec_flag *flags_r,
+			const char **fname_r)
+{
+	int ret;
+
+	ret = maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r);
+	if (ret != 0)
+		return ret;
+
+	if (maildir_uidlist_is_open(mbox->uidlist)) {
+		/* refresh uidlist and check again in case it was added
+		   after the last mailbox sync */
+		if (mbox->sync_uidlist_refreshed) {
+			/* we've already refreshed it, don't bother again */
+			return ret;
+		}
+		mbox->sync_uidlist_refreshed = TRUE;
+		if (maildir_uidlist_refresh(mbox->uidlist) < 0)
+			return -1;
+	} else {
+		/* the uidlist doesn't exist. */
+		if (maildir_storage_sync_force(mbox, uid) < 0)
+			return -1;
+	}
+
+	/* try again */
+	return maildir_uidlist_lookup(mbox->uidlist, uid, flags_r, fname_r);
+}
+
+static int maildir_sync_run(struct maildir_mailbox *mbox,
+			    enum mailbox_sync_flags flags, bool force_resync,
+			    uint32_t *uid, bool *lost_files_r)
+{
+	struct maildir_sync_context *ctx;
+	bool retry, lost_files;
+	int ret;
+
+	T_BEGIN {
+		ctx = maildir_sync_context_new(mbox, flags);
+		ret = maildir_sync_context(ctx, force_resync, uid, lost_files_r);
+		retry = ctx->racing;
+		maildir_sync_deinit(ctx);
+	} T_END;
+
+	if (retry) T_BEGIN {
+		/* we're racing some file. retry the sync again to see if the
+		   file is really gone or not. if it is, this is a bit of
+		   unnecessary work, but if it's not, this is necessary for
+		   e.g. doveadm force-resync to work. */
+		ctx = maildir_sync_context_new(mbox, 0);
+		ret = maildir_sync_context(ctx, TRUE, NULL, &lost_files);
+		maildir_sync_deinit(ctx);
+	} T_END;
+	return ret;
+}
+
+int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid)
+{
+	bool lost_files;
+	int ret;
+
+	ret = maildir_sync_run(mbox, MAILBOX_SYNC_FLAG_FAST,
+			       TRUE, &uid, &lost_files);
+	if (uid != 0) {
+		/* maybe it's expunged. check again. */
+		ret = maildir_sync_run(mbox, 0, TRUE, NULL, &lost_files);
+	}
+	return ret;
+}
+
+int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox)
+{
+	struct mail_index_view_sync_ctx *sync_ctx;
+	bool delayed_expunges;
+
+	mail_index_refresh(mbox->box.index);
+	if (mbox->flags_view == NULL)
+		mbox->flags_view = mail_index_view_open(mbox->box.index);
+
+	sync_ctx = mail_index_view_sync_begin(mbox->flags_view,
+			MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
+	if (mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) < 0) {
+		mailbox_set_index_error(&mbox->box);
+		return -1;
+	}
+	/* make sure the map stays in private memory */
+	if (mbox->flags_view->map->refcount > 1) {
+		struct mail_index_map *map;
+
+		map = mail_index_map_clone(mbox->flags_view->map);
+		mail_index_unmap(&mbox->flags_view->map);
+		mbox->flags_view->map = map;
+	}
+	mail_index_record_map_move_to_private(mbox->flags_view->map);
+	mail_index_map_move_to_memory(mbox->flags_view->map);
+	return 0;
+}
+
+struct mailbox_sync_context *
+maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct maildir_mailbox *mbox = MAILDIR_MAILBOX(box);
+	bool lost_files, force_resync;
+	int ret = 0;
+
+	force_resync = (flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0;
+	if (index_mailbox_want_full_sync(&mbox->box, flags)) {
+		ret = maildir_sync_run(mbox, flags, force_resync,
+				       NULL, &lost_files);
+		i_assert(!maildir_uidlist_is_locked(mbox->uidlist) ||
+			 (box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0);
+
+		if (lost_files) {
+			/* lost some files from new/, see if they're in cur/ */
+			ret = maildir_storage_sync_force(mbox, 0);
+		}
+	}
+
+	if (mbox->storage->set->maildir_very_dirty_syncs) {
+		if (maildir_sync_refresh_flags_view(mbox) < 0)
+			ret = -1;
+		maildir_uidlist_set_all_nonsynced(mbox->uidlist);
+	}
+	mbox->synced = TRUE;
+	mbox->sync_uidlist_refreshed = FALSE;
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
+
+int maildir_sync_is_synced(struct maildir_mailbox *mbox)
+{
+	bool new_changed, cur_changed;
+	enum maildir_scan_why why;
+	int ret;
+
+	T_BEGIN {
+		const char *box_path = mailbox_get_path(&mbox->box);
+		const char *new_dir, *cur_dir;
+
+		new_dir = t_strconcat(box_path, "/new", NULL);
+		cur_dir = t_strconcat(box_path, "/cur", NULL);
+
+		ret = maildir_sync_quick_check(mbox, FALSE, new_dir, cur_dir,
+					       &new_changed, &cur_changed,
+					       &why);
+	} T_END;
+	return ret < 0 ? -1 : (!new_changed && !cur_changed);
+}
diff --git a/src/lib-storage/index/maildir/maildir-sync.h b/src/lib-storage/index/maildir/maildir-sync.h
new file mode 100644
index 0000000..9bc6db4
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-sync.h
@@ -0,0 +1,59 @@
+#ifndef MAILDIR_SYNC_H
+#define MAILDIR_SYNC_H
+
+/* All systems accessing the filesystem must have their clock less than this
+   many seconds apart from each others. 0 works only for local filesystems. */
+#define MAILDIR_SYNC_SECS 1
+
+/* After moving this many mails from new/ to cur/, check if we need to touch
+   the uidlist lock. */
+#define MAILDIR_SLOW_MOVE_COUNT 100
+/* readdir() should be pretty fast to do, but check anyway every n files
+   to see if we need to touch the uidlist lock. */
+#define MAILDIR_SLOW_CHECK_COUNT 10000
+/* If syncing takes longer than this, log a warning. */
+#define MAILDIR_SYNC_TIME_WARN_SECS MAIL_TRANSACTION_LOG_LOCK_WARN_SECS
+
+struct maildir_mailbox;
+struct maildir_sync_context;
+struct maildir_keywords_sync_ctx;
+struct maildir_index_sync_context;
+
+int maildir_sync_is_synced(struct maildir_mailbox *mbox);
+
+struct mailbox_sync_context *
+maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid);
+
+int maildir_sync_header_refresh(struct maildir_mailbox *mbox);
+
+int maildir_sync_index_begin(struct maildir_mailbox *mbox,
+			     struct maildir_sync_context *maildir_sync_ctx,
+			     struct maildir_index_sync_context **ctx_r)
+	ATTR_NULL(2);
+int maildir_sync_index(struct maildir_index_sync_context *sync_ctx,
+		       bool partial);
+int maildir_sync_index_commit(struct maildir_index_sync_context **_ctx);
+void maildir_sync_index_rollback(struct maildir_index_sync_context **_ctx);
+
+struct maildir_keywords_sync_ctx *
+maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx);
+void maildir_sync_set_racing(struct maildir_sync_context *ctx);
+void maildir_sync_notify(struct maildir_sync_context *ctx);
+void maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx,
+				     unsigned int count);
+int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox);
+
+int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid,
+			enum maildir_uidlist_rec_flag *flags_r,
+			const char **fname_r);
+
+int maildir_list_index_has_changed(struct mailbox *box,
+				   struct mail_index_view *list_view,
+				   uint32_t seq, bool quick,
+				   const char **reason_r);
+void maildir_list_index_update_sync(struct mailbox *box,
+				    struct mail_index_transaction *trans,
+				    uint32_t seq);
+
+#endif
diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
new file mode 100644
index 0000000..bb17b9c
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -0,0 +1,2151 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+/*
+   Version 1 format has been used for most versions of Dovecot up to v1.0.x.
+   It's also compatible with Courier IMAP's courierimapuiddb file.
+   The format is:
+
+   header: 1 <uid validity> <next uid>
+   entry: <uid> <filename>
+
+   --
+
+   Version 2 format was written by a few development Dovecot versions, but
+   v1.0.x still parses the format. The format has <flags> field after <uid>.
+
+   --
+
+   Version 3 format is an extensible format used by Dovecot v1.1 and later.
+   It's also parsed by v1.0.2 (and later). The format is:
+
+   header: 3 [<key><value> ...]
+   entry: <uid> [<key><value> ...] :<filename>
+
+   See enum maildir_uidlist_*_ext_key for used keys.
+*/
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "istream.h"
+#include "ostream.h"
+#include "str.h"
+#include "file-dotlock.h"
+#include "nfs-workarounds.h"
+#include "eacces-error.h"
+#include "maildir-storage.h"
+#include "maildir-filename.h"
+#include "maildir-uidlist.h"
+
+#include <stdio.h>
+#include <sys/stat.h>
+
+/* NFS: How many times to retry reading dovecot-uidlist file if ESTALE
+   error occurs in the middle of reading it */
+#define UIDLIST_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT
+
+#define UIDLIST_VERSION 3
+#define UIDLIST_COMPRESS_PERCENTAGE 75
+
+#define UIDLIST_IS_LOCKED(uidlist) \
+	((uidlist)->lock_count > 0)
+
+struct maildir_uidlist_rec {
+	uint32_t uid;
+	uint32_t flags;
+	char *filename;
+	unsigned char *extensions; /* <data>\0[<data>\0 ...]\0 */
+};
+ARRAY_DEFINE_TYPE(maildir_uidlist_rec_p, struct maildir_uidlist_rec *);
+
+HASH_TABLE_DEFINE_TYPE(path_to_maildir_uidlist_rec,
+		       char *, struct maildir_uidlist_rec *);
+
+struct maildir_uidlist {
+	struct mailbox *box;
+	char *path;
+	struct maildir_index_header *mhdr;
+
+	int fd;
+	dev_t fd_dev;
+	ino_t fd_ino;
+	off_t fd_size;
+
+	unsigned int lock_count;
+
+	struct dotlock_settings dotlock_settings;
+	struct dotlock *dotlock;
+
+	pool_t record_pool;
+	ARRAY_TYPE(maildir_uidlist_rec_p) records;
+	HASH_TABLE_TYPE(path_to_maildir_uidlist_rec) files;
+	unsigned int change_counter;
+
+	unsigned int version;
+	unsigned int uid_validity, next_uid, prev_read_uid, last_seen_uid;
+	unsigned int hdr_next_uid;
+	unsigned int read_records_count, read_line_count;
+	uoff_t last_read_offset;
+	string_t *hdr_extensions;
+
+	guid_128_t mailbox_guid;
+
+	bool recreate:1;
+	bool recreate_on_change:1;
+	bool initial_read:1;
+	bool initial_hdr_read:1;
+	bool retry_rewind:1;
+	bool locked_refresh:1;
+	bool unsorted:1;
+	bool have_mailbox_guid:1;
+	bool opened_readonly:1;
+};
+
+struct maildir_uidlist_sync_ctx {
+	struct maildir_uidlist *uidlist;
+	enum maildir_uidlist_sync_flags sync_flags;
+
+	pool_t record_pool;
+	ARRAY_TYPE(maildir_uidlist_rec_p) records;
+	HASH_TABLE_TYPE(path_to_maildir_uidlist_rec) files;
+
+	unsigned int first_unwritten_pos, first_new_pos;
+	unsigned int new_files_count;
+	unsigned int finish_change_counter;
+
+	bool partial:1;
+	bool finished:1;
+	bool changed:1;
+	bool failed:1;
+	bool locked:1;
+};
+
+struct maildir_uidlist_iter_ctx {
+	struct maildir_uidlist *uidlist;
+	struct maildir_uidlist_rec *const *next, *const *end;
+
+	unsigned int change_counter;
+	uint32_t prev_uid;
+};
+
+static int maildir_uidlist_open_latest(struct maildir_uidlist *uidlist);
+static bool maildir_uidlist_iter_next_rec(struct maildir_uidlist_iter_ctx *ctx,
+					  struct maildir_uidlist_rec **rec_r);
+
+static int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist,
+					bool nonblock, bool refresh,
+					bool refresh_when_locked)
+{
+	struct mailbox *box = uidlist->box;
+	const struct mailbox_permissions *perm = mailbox_get_permissions(box);
+	const char *path = uidlist->path;
+	mode_t old_mask;
+	const enum dotlock_create_flags dotlock_flags =
+		nonblock ? DOTLOCK_CREATE_FLAG_NONBLOCK : 0;
+	int i, ret;
+
+	if (uidlist->lock_count > 0) {
+		if (!uidlist->locked_refresh && refresh_when_locked) {
+			if (maildir_uidlist_refresh(uidlist) < 0)
+				return -1;
+		}
+		uidlist->lock_count++;
+		return 1;
+	}
+
+        index_storage_lock_notify_reset(box);
+
+	for (i = 0;; i++) {
+		old_mask = umask(0777 & ~perm->file_create_mode);
+		ret = file_dotlock_create(&uidlist->dotlock_settings, path,
+					  dotlock_flags, &uidlist->dotlock);
+		umask(old_mask);
+		if (ret > 0)
+			break;
+
+		/* failure */
+		if (ret == 0) {
+			mail_storage_set_error(box->storage,
+				MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT);
+			return 0;
+		}
+		if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) {
+			if (errno == EACCES) {
+				mailbox_set_critical(box, "%s",
+					eacces_error_get_creating("file_dotlock_create", path));
+			} else {
+				mailbox_set_critical(box,
+					"file_dotlock_create(%s) failed: %m",
+					path);
+			}
+			return -1;
+		}
+		/* the control dir doesn't exist. create it unless the whole
+		   mailbox was just deleted. */
+		if (!maildir_set_deleted(uidlist->box))
+			return -1;
+	}
+
+	uidlist->lock_count++;
+	uidlist->locked_refresh = FALSE;
+
+	if (refresh) {
+		/* make sure we have the latest changes before
+		   changing anything */
+		if (maildir_uidlist_refresh(uidlist) < 0) {
+			maildir_uidlist_unlock(uidlist);
+			return -1;
+		}
+	}
+	return 1;
+}
+
+int maildir_uidlist_lock(struct maildir_uidlist *uidlist)
+{
+	return maildir_uidlist_lock_timeout(uidlist, FALSE, TRUE, FALSE);
+}
+
+int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)
+{
+	return maildir_uidlist_lock_timeout(uidlist, TRUE, TRUE, FALSE);
+}
+
+int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist)
+{
+	i_assert(UIDLIST_IS_LOCKED(uidlist));
+
+	return file_dotlock_touch(uidlist->dotlock);
+}
+
+bool maildir_uidlist_is_locked(struct maildir_uidlist *uidlist)
+{
+	return UIDLIST_IS_LOCKED(uidlist);
+}
+
+bool maildir_uidlist_is_read(struct maildir_uidlist *uidlist)
+{
+	return uidlist->initial_read;
+}
+
+bool maildir_uidlist_is_open(struct maildir_uidlist *uidlist)
+{
+	return uidlist->fd != -1;
+}
+
+void maildir_uidlist_unlock(struct maildir_uidlist *uidlist)
+{
+	i_assert(uidlist->lock_count > 0);
+
+	if (--uidlist->lock_count > 0)
+		return;
+
+	uidlist->locked_refresh = FALSE;
+	file_dotlock_delete(&uidlist->dotlock);
+}
+
+static bool dotlock_callback(unsigned int secs_left, bool stale, void *context)
+{
+	struct mailbox *box = context;
+
+	index_storage_lock_notify(box, stale ?
+				  MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE :
+				  MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT,
+				  secs_left);
+	return TRUE;
+}
+
+struct maildir_uidlist *maildir_uidlist_init(struct maildir_mailbox *mbox)
+{
+	struct mailbox *box = &mbox->box;
+	struct maildir_uidlist *uidlist;
+	const char *control_dir;
+
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL,
+				&control_dir) <= 0)
+		i_unreached();
+
+	uidlist = i_new(struct maildir_uidlist, 1);
+	uidlist->box = box;
+	uidlist->mhdr = &mbox->maildir_hdr;
+	uidlist->fd = -1;
+	uidlist->path = i_strconcat(control_dir, "/"MAILDIR_UIDLIST_NAME, NULL);
+	i_array_init(&uidlist->records, 128);
+	hash_table_create(&uidlist->files, default_pool, 4096,
+			  maildir_filename_base_hash,
+			  maildir_filename_base_cmp);
+	uidlist->next_uid = 1;
+	uidlist->hdr_extensions = str_new(default_pool, 128);
+
+	uidlist->dotlock_settings.use_io_notify = TRUE;
+	uidlist->dotlock_settings.use_excl_lock =
+		box->storage->set->dotlock_use_excl;
+	uidlist->dotlock_settings.nfs_flush =
+		box->storage->set->mail_nfs_storage;
+	uidlist->dotlock_settings.timeout =
+		mail_storage_get_lock_timeout(box->storage,
+			MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT + 2);
+	uidlist->dotlock_settings.stale_timeout =
+		MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT;
+	uidlist->dotlock_settings.callback = dotlock_callback;
+	uidlist->dotlock_settings.context = box;
+	uidlist->dotlock_settings.temp_prefix = mbox->storage->temp_prefix;
+	return uidlist;
+}
+
+static void maildir_uidlist_close(struct maildir_uidlist *uidlist)
+{
+	if (uidlist->fd != -1) {
+		if (close(uidlist->fd) < 0) {
+			mailbox_set_critical(uidlist->box,
+				"close(%s) failed: %m", uidlist->path);
+		}
+		uidlist->fd = -1;
+		uidlist->fd_ino = 0;
+	}
+	uidlist->last_read_offset = 0;
+	uidlist->read_line_count = 0;
+}
+
+static void maildir_uidlist_reset(struct maildir_uidlist *uidlist)
+{
+	maildir_uidlist_close(uidlist);
+	uidlist->last_seen_uid = 0;
+	uidlist->initial_hdr_read = FALSE;
+	uidlist->read_records_count = 0;
+
+	hash_table_clear(uidlist->files, FALSE);
+	array_clear(&uidlist->records);
+}
+
+void maildir_uidlist_deinit(struct maildir_uidlist **_uidlist)
+{
+	struct maildir_uidlist *uidlist = *_uidlist;
+
+	i_assert(!UIDLIST_IS_LOCKED(uidlist));
+
+	*_uidlist = NULL;
+	(void)maildir_uidlist_update(uidlist);
+	maildir_uidlist_close(uidlist);
+
+	hash_table_destroy(&uidlist->files);
+	pool_unref(&uidlist->record_pool);
+
+	array_free(&uidlist->records);
+	str_free(&uidlist->hdr_extensions);
+	i_free(uidlist->path);
+	i_free(uidlist);
+}
+
+static int maildir_uid_cmp(struct maildir_uidlist_rec *const *rec1,
+			   struct maildir_uidlist_rec *const *rec2)
+{
+	return (*rec1)->uid < (*rec2)->uid ? -1 :
+		(*rec1)->uid > (*rec2)->uid ? 1 : 0;
+}
+
+static void ATTR_FORMAT(2, 3)
+maildir_uidlist_set_corrupted(struct maildir_uidlist *uidlist,
+			      const char *fmt, ...)
+{
+	va_list args;
+
+	va_start(args, fmt);
+	if (uidlist->retry_rewind) {
+		mailbox_set_critical(uidlist->box,
+			"Broken or unexpectedly changed file %s "
+			"line %u: %s - re-reading from beginning",
+			uidlist->path, uidlist->read_line_count,
+			t_strdup_vprintf(fmt, args));
+	} else {
+		mailbox_set_critical(uidlist->box, "Broken file %s line %u: %s",
+			uidlist->path, uidlist->read_line_count,
+			t_strdup_vprintf(fmt, args));
+	}
+	va_end(args);
+}
+
+static void maildir_uidlist_update_hdr(struct maildir_uidlist *uidlist,
+				       const struct stat *st)
+{
+	struct maildir_index_header *mhdr = uidlist->mhdr;
+
+	if (mhdr->uidlist_mtime == 0 && uidlist->version != UIDLIST_VERSION) {
+		/* upgrading from older version. don't update the
+		   uidlist times until it uses the new format */
+		uidlist->recreate = TRUE;
+		return;
+	}
+	mhdr->uidlist_mtime = st->st_mtime;
+	mhdr->uidlist_mtime_nsecs = ST_MTIME_NSEC(*st);
+	mhdr->uidlist_size = st->st_size;
+}
+
+static unsigned int
+maildir_uidlist_records_array_delete(struct maildir_uidlist *uidlist,
+				     struct maildir_uidlist_rec *rec)
+{
+	struct maildir_uidlist_rec *const *recs, *const *pos;
+	unsigned int idx, count;
+
+	recs = array_get(&uidlist->records, &count);
+	if (!uidlist->unsorted) {
+		pos = array_bsearch(&uidlist->records, &rec, maildir_uid_cmp);
+		i_assert(pos != NULL);
+		idx = pos - recs;
+	} else {
+		for (idx = 0; idx < count; idx++) {
+			if (recs[idx]->uid == rec->uid)
+				break;
+		}
+		i_assert(idx != count);
+	}
+	array_delete(&uidlist->records, idx, 1);
+	return idx;
+}
+
+static bool
+maildir_uidlist_read_extended(struct maildir_uidlist *uidlist,
+			      const char **line_p,
+			      struct maildir_uidlist_rec *rec)
+{
+	const char *start, *line = *line_p;
+	buffer_t *buf;
+
+	buf = t_buffer_create(128);
+	while (*line != '\0' && *line != ':') {
+		/* skip over an extension field */
+		start = line;
+		while (*line != ' ' && *line != '\0') line++;
+		if (MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*start)) {
+			buffer_append(buf, start, line - start);
+			buffer_append_c(buf, '\0');
+		} else {
+			maildir_uidlist_set_corrupted(uidlist,
+				"Invalid extension record, removing: %s",
+				t_strdup_until(start, line));
+			uidlist->recreate = TRUE;
+		}
+		while (*line == ' ') line++;
+	}
+
+	if (buf->used > 0) {
+		/* save the extensions */
+		buffer_append_c(buf, '\0');
+		rec->extensions = p_malloc(uidlist->record_pool, buf->used);
+		memcpy(rec->extensions, buf->data, buf->used);
+	}
+
+	if (*line == ':')
+		line++;
+	if (*line == '\0')
+		return FALSE;
+
+	*line_p = line;
+	return TRUE;
+}
+
+static bool maildir_uidlist_next(struct maildir_uidlist *uidlist,
+				 const char *line)
+{
+	struct maildir_uidlist_rec *rec, *old_rec, *const *recs;
+	unsigned int count;
+	uint32_t uid;
+
+	uid = 0;
+	while (*line >= '0' && *line <= '9') {
+		uid = uid*10 + (*line - '0');
+		line++;
+	}
+
+	if (uid == 0 || *line != ' ') {
+		/* invalid file */
+		maildir_uidlist_set_corrupted(uidlist, "Invalid data: %s",
+					      line);
+		return FALSE;
+	}
+	if (uid <= uidlist->prev_read_uid) {
+		maildir_uidlist_set_corrupted(uidlist, 
+					      "UIDs not ordered (%u >= %u)",
+					      uid, uidlist->prev_read_uid);
+		return FALSE;
+	}
+	if (uid >= (uint32_t)-1) {
+		maildir_uidlist_set_corrupted(uidlist,
+					      "UID too high (%u)", uid);
+		return FALSE;
+	}
+	uidlist->prev_read_uid = uid;
+
+	if (uid <= uidlist->last_seen_uid) {
+		/* we already have this */
+		return TRUE;
+	}
+        uidlist->last_seen_uid = uid;
+
+	if (uid >= uidlist->next_uid && uidlist->version == 1) {
+		maildir_uidlist_set_corrupted(uidlist, 
+			"UID larger than next_uid (%u >= %u)",
+			uid, uidlist->next_uid);
+		return FALSE;
+	}
+
+	rec = p_new(uidlist->record_pool, struct maildir_uidlist_rec, 1);
+	rec->uid = uid;
+	rec->flags = MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+
+	while (*line == ' ') line++;
+
+	if (uidlist->version == UIDLIST_VERSION) {
+		/* read extended fields */
+		bool ret;
+
+		T_BEGIN {
+			ret = maildir_uidlist_read_extended(uidlist, &line,
+							    rec);
+		} T_END;
+		if (!ret) {
+			maildir_uidlist_set_corrupted(uidlist, 
+				"Invalid extended fields: %s", line);
+			return FALSE;
+		}
+	}
+
+	if (strchr(line, '/') != NULL) {
+		maildir_uidlist_set_corrupted(uidlist, 
+			"%s: Broken filename at line %u: %s",
+			uidlist->path, uidlist->read_line_count, line);
+		return FALSE;
+	}
+
+	old_rec = hash_table_lookup(uidlist->files, line);
+	if (old_rec == NULL) {
+		/* no conflicts */
+	} else if (old_rec->uid == uid) {
+		/* most likely this is a record we saved ourself, but couldn't
+		   update last_seen_uid because uidlist wasn't refreshed while
+		   it was locked.
+
+		   another possibility is a duplicate file record. currently
+		   it would be a bug, but not that big of a deal. also perhaps
+		   in future such duplicate lines could be used to update
+		   extended fields. so just let it through anyway.
+
+		   we'll waste a bit of memory here by allocating the record
+		   twice, but that's not really a problem.  */
+		rec->filename = old_rec->filename;
+		hash_table_update(uidlist->files, rec->filename, rec);
+		uidlist->unsorted = TRUE;
+		return TRUE;
+	} else {
+		/* This can happen if expunged file is moved back and the file
+		   was appended to uidlist. */
+		i_warning("%s: Duplicate file entry at line %u: "
+			  "%s (uid %u -> %u)%s",
+			  uidlist->path, uidlist->read_line_count, line,
+			  old_rec->uid, uid, uidlist->retry_rewind ?
+			  " - retrying by re-reading from beginning" : "");
+		if (uidlist->retry_rewind)
+			return FALSE;
+		/* Delete the old UID */
+		(void)maildir_uidlist_records_array_delete(uidlist, old_rec);
+		/* Replace the old record with this new one */
+		*old_rec = *rec;
+		rec = old_rec;
+		uidlist->recreate = TRUE;
+	}
+
+	recs = array_get(&uidlist->records, &count);
+	if (count > 0 && recs[count-1]->uid > uid) {
+		/* we most likely have some records in the array that we saved
+		   ourself without refreshing uidlist */
+		uidlist->unsorted = TRUE;
+	}
+
+	rec->filename = p_strdup(uidlist->record_pool, line);
+	hash_table_update(uidlist->files, rec->filename, rec);
+	array_push_back(&uidlist->records, &rec);
+	return TRUE;
+}
+
+static int
+maildir_uidlist_read_v3_header(struct maildir_uidlist *uidlist,
+			       const char *line,
+			       unsigned int *uid_validity_r,
+			       unsigned int *next_uid_r)
+{
+	char key;
+
+	str_truncate(uidlist->hdr_extensions, 0);
+	while (*line != '\0') {
+		const char *value;
+
+		key = *line;
+		value = ++line;
+		while (*line != '\0' && *line != ' ') line++;
+		value = t_strdup_until(value, line);
+
+		switch (key) {
+		case MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY:
+			if (str_to_uint(value, uid_validity_r) < 0) {
+				maildir_uidlist_set_corrupted(uidlist,
+					"Invalid mailbox UID_VALIDITY: %s", value);
+				return -1;
+			}
+			break;
+		case MAILDIR_UIDLIST_HDR_EXT_NEXT_UID:
+			if (str_to_uint(value, next_uid_r) < 0) {
+				maildir_uidlist_set_corrupted(uidlist,
+					"Invalid mailbox NEXT_UID: %s", value);
+				return -1;
+			}
+			break;
+		case MAILDIR_UIDLIST_HDR_EXT_GUID:
+			if (guid_128_from_string(value,
+						 uidlist->mailbox_guid) < 0) {
+				maildir_uidlist_set_corrupted(uidlist,
+					"Invalid mailbox GUID: %s", value);
+				return -1;
+			}
+			uidlist->have_mailbox_guid = TRUE;
+			break;
+		default:
+			if (str_len(uidlist->hdr_extensions) > 0)
+				str_append_c(uidlist->hdr_extensions, ' ');
+			str_printfa(uidlist->hdr_extensions,
+				    "%c%s", key, value);
+			break;
+		}
+
+		while (*line == ' ') line++;
+	}
+	return 0;
+}
+
+static int maildir_uidlist_read_header(struct maildir_uidlist *uidlist,
+				       struct istream *input)
+{
+	unsigned int uid_validity = 0, next_uid = 0;
+	const char *line;
+	int ret;
+
+	line = i_stream_read_next_line(input);
+        if (line == NULL) {
+                /* I/O error / empty file */
+                return input->stream_errno == 0 ? 0 : -1;
+	}
+	uidlist->read_line_count = 1;
+
+	if (*line < '0' || *line > '9' || line[1] != ' ') {
+		maildir_uidlist_set_corrupted(uidlist,
+			"Corrupted header (invalid version number)");
+		return 0;
+	}
+
+	uidlist->version = *line - '0';
+	line += 2;
+
+	switch (uidlist->version) {
+	case 1:
+		if (sscanf(line, "%u %u", &uid_validity, &next_uid) != 2) {
+			maildir_uidlist_set_corrupted(uidlist,
+				"Corrupted header (version 1)");
+			return 0;
+		}
+		break;
+	case UIDLIST_VERSION:
+		T_BEGIN {
+			ret = maildir_uidlist_read_v3_header(uidlist, line,
+							     &uid_validity,
+							     &next_uid);
+		} T_END;
+		if (ret < 0)
+			return 0;
+		break;
+	default:
+		maildir_uidlist_set_corrupted(uidlist, "Unsupported version %u",
+					      uidlist->version);
+		return 0;
+	}
+
+	if (uid_validity == 0 || next_uid == 0) {
+		maildir_uidlist_set_corrupted(uidlist,
+			"Broken header (uidvalidity = %u, next_uid=%u)",
+			uid_validity, next_uid);
+		return 0;
+	}
+
+	if (uid_validity == uidlist->uid_validity &&
+	    next_uid < uidlist->hdr_next_uid) {
+		maildir_uidlist_set_corrupted(uidlist,
+			"next_uid header was lowered (%u -> %u)",
+			uidlist->hdr_next_uid, next_uid);
+		return 0;
+	}
+
+	uidlist->uid_validity = uid_validity;
+	uidlist->next_uid = next_uid;
+	uidlist->hdr_next_uid = next_uid;
+	return 1;
+}
+
+static void maildir_uidlist_records_sort_by_uid(struct maildir_uidlist *uidlist)
+{
+	array_sort(&uidlist->records, maildir_uid_cmp);
+	uidlist->unsorted = FALSE;
+}
+
+static int
+maildir_uidlist_update_read(struct maildir_uidlist *uidlist,
+			    bool *retry_r, bool try_retry)
+{
+	const char *line;
+	uint32_t orig_next_uid, orig_uid_validity;
+	struct istream *input;
+	struct stat st;
+	uoff_t last_read_offset;
+	int fd, ret;
+	bool readonly = FALSE;
+
+	*retry_r = FALSE;
+
+	if (uidlist->fd == -1) {
+		fd = nfs_safe_open(uidlist->path, O_RDWR);
+		if (fd == -1 && errno == EACCES) {
+			fd = nfs_safe_open(uidlist->path, O_RDONLY);
+			readonly = TRUE;
+		}
+		if (fd == -1) {
+			if (errno != ENOENT) {
+				mailbox_set_critical(uidlist->box,
+					"open(%s) failed: %m", uidlist->path);
+				return -1;
+			}
+			return 0;
+		}
+		last_read_offset = 0;
+	} else {
+		/* the file was updated */
+		fd = uidlist->fd;
+		if (lseek(fd, 0, SEEK_SET) < 0) {
+			if (errno == ESTALE && try_retry) {
+				*retry_r = TRUE;
+				return -1;
+			}
+			mailbox_set_critical(uidlist->box,
+				"lseek(%s) failed: %m", uidlist->path);
+			return -1;
+		}
+		uidlist->fd = -1;
+		uidlist->fd_ino = 0;
+		last_read_offset = uidlist->last_read_offset;
+		uidlist->last_read_offset = 0;
+	}
+
+	if (fstat(fd, &st) < 0) {
+                i_close_fd(&fd);
+                if (errno == ESTALE && try_retry) {
+                        *retry_r = TRUE;
+                        return -1;
+                }
+                mailbox_set_critical(uidlist->box,
+			"fstat(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+
+	if (uidlist->record_pool == NULL) {
+		uidlist->record_pool =
+			pool_alloconly_create(MEMPOOL_GROWING
+					      "uidlist record_pool",
+					      nearest_power(st.st_size -
+							    st.st_size/8));
+	}
+
+	input = i_stream_create_fd(fd, SIZE_MAX);
+	i_stream_seek(input, last_read_offset);
+
+	orig_uid_validity = uidlist->uid_validity;
+	orig_next_uid = uidlist->next_uid;
+	ret = input->v_offset != 0 ? 1 :
+		maildir_uidlist_read_header(uidlist, input);
+	if (ret > 0) {
+		uidlist->prev_read_uid = 0;
+		uidlist->change_counter++;
+		uidlist->retry_rewind = last_read_offset != 0 && try_retry;
+
+		ret = 1;
+		while ((line = i_stream_read_next_line(input)) != NULL) {
+			uidlist->read_records_count++;
+			uidlist->read_line_count++;
+			if (!maildir_uidlist_next(uidlist, line)) {
+				if (!uidlist->retry_rewind)
+					ret = 0;
+				else {
+					ret = -1;
+					*retry_r = TRUE;
+				}
+				break;
+			}
+                }
+		uidlist->retry_rewind = FALSE;
+		if (input->stream_errno != 0)
+                        ret = -1;
+
+		if (uidlist->unsorted) {
+			uidlist->recreate_on_change = TRUE;
+			maildir_uidlist_records_sort_by_uid(uidlist);
+		}
+		if (uidlist->next_uid <= uidlist->prev_read_uid)
+			uidlist->next_uid = uidlist->prev_read_uid + 1;
+		if (ret > 0 && uidlist->uid_validity != orig_uid_validity &&
+		    orig_uid_validity != 0) {
+			uidlist->recreate = TRUE;
+		} else if (ret > 0 && uidlist->next_uid < orig_next_uid) {
+			mailbox_set_critical(uidlist->box,
+				"%s: next_uid was lowered (%u -> %u, hdr=%u)",
+				uidlist->path, orig_next_uid,
+				uidlist->next_uid, uidlist->hdr_next_uid);
+			uidlist->recreate = TRUE;
+			uidlist->next_uid = orig_next_uid;
+		}
+	}
+
+        if (ret == 0) {
+                /* file is broken */
+                i_unlink(uidlist->path);
+        } else if (ret > 0) {
+                /* success */
+		if (readonly)
+			uidlist->recreate_on_change = TRUE;
+		uidlist->fd = fd;
+		uidlist->fd_dev = st.st_dev;
+		uidlist->fd_ino = st.st_ino;
+		uidlist->fd_size = st.st_size;
+		uidlist->last_read_offset = input->v_offset;
+		maildir_uidlist_update_hdr(uidlist, &st);
+        } else if (!*retry_r) {
+                /* I/O error */
+                if (input->stream_errno == ESTALE && try_retry)
+			*retry_r = TRUE;
+		else {
+			mailbox_set_critical(uidlist->box,
+				"read(%s) failed: %s", uidlist->path,
+				i_stream_get_error(input));
+		}
+		uidlist->last_read_offset = 0;
+	}
+
+	i_stream_destroy(&input);
+	if (ret <= 0) {
+		if (close(fd) < 0) {
+			mailbox_set_critical(uidlist->box,
+				"close(%s) failed: %m", uidlist->path);
+		}
+	}
+	return ret;
+}
+
+static int
+maildir_uidlist_stat(struct maildir_uidlist *uidlist, struct stat *st_r)
+{
+	struct mail_storage *storage = uidlist->box->storage;
+
+	if (storage->set->mail_nfs_storage) {
+		nfs_flush_file_handle_cache(uidlist->path);
+		nfs_flush_attr_cache_unlocked(uidlist->path);
+	}
+	if (nfs_safe_stat(uidlist->path, st_r) < 0) {
+		if (errno != ENOENT) {
+			mailbox_set_critical(uidlist->box,
+				"stat(%s) failed: %m", uidlist->path);
+			return -1;
+		}
+		return 0;
+	}
+	return 1;
+}
+
+static int
+maildir_uidlist_has_changed(struct maildir_uidlist *uidlist, bool *recreated_r)
+{
+	struct mail_storage *storage = uidlist->box->storage;
+        struct stat st;
+	int ret;
+
+	*recreated_r = FALSE;
+
+	if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0)
+		return -1;
+	if (ret == 0) {
+		*recreated_r = TRUE;
+		return 1;
+	}
+
+	if (st.st_ino != uidlist->fd_ino ||
+	    !CMP_DEV_T(st.st_dev, uidlist->fd_dev)) {
+		/* file recreated */
+		*recreated_r = TRUE;
+		return 1;
+	}
+
+	if (storage->set->mail_nfs_storage) {
+		/* NFS: either the file hasn't been changed, or it has already
+		   been deleted and the inodes just happen to be the same.
+		   check if the fd is still valid. */
+		if (fstat(uidlist->fd, &st) < 0) {
+			if (errno == ESTALE) {
+				*recreated_r = TRUE;
+				return 1;
+			}
+			mailbox_set_critical(uidlist->box,
+				"fstat(%s) failed: %m", uidlist->path);
+			return -1;
+		}
+	}
+
+	if (st.st_size != uidlist->fd_size) {
+		/* file modified but not recreated */
+		return 1;
+	} else {
+		/* unchanged */
+		return 0;
+	}
+}
+
+static int maildir_uidlist_open_latest(struct maildir_uidlist *uidlist)
+{
+	bool recreated;
+	int ret;
+
+	if (uidlist->fd != -1) {
+		ret = maildir_uidlist_has_changed(uidlist, &recreated);
+		if (ret <= 0) {
+			if (UIDLIST_IS_LOCKED(uidlist))
+				uidlist->locked_refresh = TRUE;
+			return ret < 0 ? -1 : 1;
+		}
+
+		if (!recreated)
+			return 0;
+		maildir_uidlist_reset(uidlist);
+	}
+
+	uidlist->fd = nfs_safe_open(uidlist->path, O_RDWR);
+	if (uidlist->fd == -1 && errno == EACCES) {
+		uidlist->fd = nfs_safe_open(uidlist->path, O_RDONLY);
+		uidlist->recreate_on_change = TRUE;
+	}
+	if (uidlist->fd == -1 && errno != ENOENT) {
+		mailbox_set_critical(uidlist->box,
+			"open(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	return 0;
+}
+
+int maildir_uidlist_refresh(struct maildir_uidlist *uidlist)
+{
+        unsigned int i;
+        bool retry;
+        int ret;
+
+	if (maildir_uidlist_open_latest(uidlist) < 0)
+		return -1;
+
+        for (i = 0; ; i++) {
+		ret = maildir_uidlist_update_read(uidlist, &retry,
+						i < UIDLIST_ESTALE_RETRY_COUNT);
+		if (!retry)
+			break;
+		/* ESTALE - try reopening and rereading */
+		maildir_uidlist_close(uidlist);
+        }
+	if (ret >= 0) {
+		uidlist->initial_read = TRUE;
+		uidlist->initial_hdr_read = TRUE;
+		if (UIDLIST_IS_LOCKED(uidlist))
+			uidlist->locked_refresh = TRUE;
+		if (!uidlist->have_mailbox_guid) {
+			uidlist->recreate = TRUE;
+			(void)maildir_uidlist_update(uidlist);
+		}
+	}
+        return ret;
+}
+
+int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist)
+{
+	const struct maildir_index_header *mhdr = uidlist->mhdr;
+	struct mail_index *index = uidlist->box->index;
+	struct mail_index_view *view;
+	const struct mail_index_header *hdr;
+	struct stat st;
+	int ret;
+
+	i_assert(UIDLIST_IS_LOCKED(uidlist));
+
+	if (uidlist->fd != -1)
+		return maildir_uidlist_refresh(uidlist);
+
+	if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0)
+		return ret;
+
+	if (ret > 0 && st.st_size == mhdr->uidlist_size &&
+	    st.st_mtime == (time_t)mhdr->uidlist_mtime &&
+	    ST_NTIMES_EQUAL(ST_MTIME_NSEC(st), mhdr->uidlist_mtime_nsecs) &&
+	    (!mail_index_is_in_memory(index) || st.st_mtime < ioloop_time-1)) {
+		/* index is up-to-date. look up the uidvalidity and next-uid
+		   from it. we'll need to create a new view temporarily to
+		   make sure we get the latest values. */
+		view = mail_index_view_open(index);
+		hdr = mail_index_get_header(view);
+		uidlist->uid_validity = hdr->uid_validity;
+		uidlist->next_uid = hdr->next_uid;
+		uidlist->initial_hdr_read = TRUE;
+		mail_index_view_close(&view);
+
+		if (UIDLIST_IS_LOCKED(uidlist))
+			uidlist->locked_refresh = TRUE;
+		return 1;
+	} else {
+		return maildir_uidlist_refresh(uidlist);
+	}
+}
+
+static int
+maildir_uid_bsearch_cmp(const uint32_t *uidp,
+			struct maildir_uidlist_rec *const *recp)
+{
+	return *uidp < (*recp)->uid ? -1 :
+		*uidp > (*recp)->uid ? 1 : 0;
+}
+
+static int
+maildir_uidlist_lookup_rec(struct maildir_uidlist *uidlist, uint32_t uid,
+			   struct maildir_uidlist_rec **rec_r)
+{
+	struct maildir_uidlist_rec *const *pos;
+
+	if (!uidlist->initial_read) {
+		/* first time we need to read uidlist */
+		if (maildir_uidlist_refresh(uidlist) < 0)
+			return -1;
+	}
+
+	pos = array_bsearch(&uidlist->records, &uid,
+			    maildir_uid_bsearch_cmp);
+	if (pos == NULL) {
+		*rec_r = NULL;
+		return 0;
+	}
+	*rec_r = *pos;
+	return 1;
+}
+
+int maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid,
+			   enum maildir_uidlist_rec_flag *flags_r,
+			   const char **fname_r)
+{
+	struct maildir_uidlist_rec *rec;
+	int ret;
+
+	if ((ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec)) <= 0)
+		return ret;
+
+	*flags_r = rec->flags;
+	*fname_r = rec->filename;
+	return 1;
+}
+
+const char *
+maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid,
+			   enum maildir_uidlist_rec_ext_key key)
+{
+	struct maildir_uidlist_rec *rec;
+	const unsigned char *p;
+	int ret;
+
+	ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec);
+	if (ret <= 0 || rec->extensions == NULL)
+		return NULL;
+
+	p = rec->extensions;
+	while (*p != '\0') {
+		/* <key><value>\0 */
+		if (*p == (unsigned char)key)
+			return (const char *)p + 1;
+
+		p += strlen((const char *)p) + 1;
+	}
+	return NULL;
+}
+
+uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist)
+{
+	return uidlist->uid_validity;
+}
+
+uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist)
+{
+	return !uidlist->initial_hdr_read ? 0 : uidlist->next_uid;
+}
+
+int maildir_uidlist_get_mailbox_guid(struct maildir_uidlist *uidlist,
+				     guid_128_t mailbox_guid)
+{
+	if (!uidlist->initial_hdr_read) {
+		if (maildir_uidlist_refresh(uidlist) < 0)
+			return -1;
+	}
+	if (!uidlist->have_mailbox_guid) {
+		uidlist->recreate = TRUE;
+		if (maildir_uidlist_update(uidlist) < 0)
+			return -1;
+	}
+	memcpy(mailbox_guid, uidlist->mailbox_guid, GUID_128_SIZE);
+	return 0;
+}
+
+void maildir_uidlist_set_mailbox_guid(struct maildir_uidlist *uidlist,
+				      const guid_128_t mailbox_guid)
+{
+	if (memcmp(uidlist->mailbox_guid, mailbox_guid,
+		   sizeof(uidlist->mailbox_guid)) != 0) {
+		memcpy(uidlist->mailbox_guid, mailbox_guid,
+		       sizeof(uidlist->mailbox_guid));
+		uidlist->recreate = TRUE;
+	}
+}
+
+void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist,
+				      uint32_t uid_validity)
+{
+	i_assert(uid_validity != 0);
+
+	if (uid_validity != uidlist->uid_validity) {
+		uidlist->uid_validity = uid_validity;
+		uidlist->recreate = TRUE;
+	}
+}
+
+void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist,
+				  uint32_t next_uid, bool force)
+{
+	if (uidlist->next_uid < next_uid || force) {
+		i_assert(next_uid != 0);
+		uidlist->next_uid = next_uid;
+		uidlist->recreate = TRUE;
+	}
+}
+
+static void
+maildir_uidlist_rec_set_ext(struct maildir_uidlist_rec *rec, pool_t pool,
+			    enum maildir_uidlist_rec_ext_key key,
+			    const char *value)
+{
+	const unsigned char *p;
+	buffer_t *buf;
+	size_t len;
+
+	/* copy existing extensions, except for the one we're updating */
+	buf = t_buffer_create(128);
+	if (rec->extensions != NULL) {
+		p = rec->extensions;
+		while (*p != '\0') {
+			/* <key><value>\0 */
+			i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*p));
+
+			len = strlen((const char *)p) + 1;
+			if (*p != (unsigned char)key)
+				buffer_append(buf, p, len);
+			p += len;
+		}
+	}
+	if (value != NULL) {
+		buffer_append_c(buf, key);
+		buffer_append(buf, value, strlen(value) + 1);
+	}
+	buffer_append_c(buf, '\0');
+
+	rec->extensions = p_malloc(pool, buf->used);
+	memcpy(rec->extensions, buf->data, buf->used);
+}
+
+static void ATTR_NULL(4)
+maildir_uidlist_set_ext_internal(struct maildir_uidlist *uidlist, uint32_t uid,
+				 enum maildir_uidlist_rec_ext_key key,
+				 const char *value)
+{
+	struct maildir_uidlist_rec *rec;
+	int ret;
+
+	i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(key));
+
+	ret = maildir_uidlist_lookup_rec(uidlist, uid, &rec);
+	if (ret <= 0) {
+		if (ret < 0)
+			return;
+
+		/* maybe it's a new message */
+		if (maildir_uidlist_refresh(uidlist) < 0)
+			return;
+		if (maildir_uidlist_lookup_rec(uidlist, uid, &rec) <= 0) {
+			/* message is already expunged, ignore */
+			return;
+		}
+	}
+
+	T_BEGIN {
+		maildir_uidlist_rec_set_ext(rec, uidlist->record_pool,
+					    key, value);
+	} T_END;
+
+	if (rec->uid != (uint32_t)-1) {
+		/* message already exists in uidlist, need to recreate it */
+		uidlist->recreate = TRUE;
+	}
+}
+
+void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid,
+			     enum maildir_uidlist_rec_ext_key key,
+			     const char *value)
+{
+	maildir_uidlist_set_ext_internal(uidlist, uid, key, value);
+}
+
+void maildir_uidlist_unset_ext(struct maildir_uidlist *uidlist, uint32_t uid,
+			       enum maildir_uidlist_rec_ext_key key)
+{
+	maildir_uidlist_set_ext_internal(uidlist, uid, key, NULL);
+}
+
+static void
+maildir_uidlist_generate_uid_validity(struct maildir_uidlist *uidlist)
+{
+	const struct mail_index_header *hdr;
+
+	if (uidlist->box->opened) {
+		hdr = mail_index_get_header(uidlist->box->view);
+		if (hdr->uid_validity != 0) {
+			uidlist->uid_validity = hdr->uid_validity;
+			return;
+		}
+	}
+	uidlist->uid_validity =
+		maildir_get_uidvalidity_next(uidlist->box->list);
+}
+
+static int maildir_uidlist_write_fd(struct maildir_uidlist *uidlist, int fd,
+				    const char *path, unsigned int first_idx,
+				    uoff_t *file_size_r)
+{
+	struct mail_storage *storage = uidlist->box->storage;
+	struct maildir_uidlist_iter_ctx *iter;
+	struct ostream *output;
+	struct maildir_uidlist_rec *rec;
+	string_t *str;
+	const unsigned char *p;
+	const char *strp;
+	size_t len;
+
+	i_assert(fd != -1);
+
+	output = o_stream_create_fd_file(fd, UOFF_T_MAX, FALSE);
+	o_stream_cork(output);
+	str = t_str_new(512);
+
+	if (output->offset == 0) {
+		i_assert(first_idx == 0);
+		uidlist->version = UIDLIST_VERSION;
+
+		if (uidlist->uid_validity == 0)
+			maildir_uidlist_generate_uid_validity(uidlist);
+		if (!uidlist->have_mailbox_guid)
+			guid_128_generate(uidlist->mailbox_guid);
+
+		i_assert(uidlist->next_uid > 0);
+		str_printfa(str, "%u %c%u %c%u %c%s", uidlist->version,
+			    MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY,
+			    uidlist->uid_validity,
+			    MAILDIR_UIDLIST_HDR_EXT_NEXT_UID,
+			    uidlist->next_uid,
+			    MAILDIR_UIDLIST_HDR_EXT_GUID,
+			    guid_128_to_string(uidlist->mailbox_guid));
+		if (str_len(uidlist->hdr_extensions) > 0) {
+			str_append_c(str, ' ');
+			str_append_str(str, uidlist->hdr_extensions);
+		}
+		str_append_c(str, '\n');
+		o_stream_nsend(output, str_data(str), str_len(str));
+	}
+
+	iter = maildir_uidlist_iter_init(uidlist);
+	i_assert(first_idx <= array_count(&uidlist->records));
+	iter->next += first_idx;
+
+	while (maildir_uidlist_iter_next_rec(iter, &rec)) {
+		uidlist->read_records_count++;
+		str_truncate(str, 0);
+		str_printfa(str, "%u", rec->uid);
+		if (rec->extensions != NULL) {
+			for (p = rec->extensions; *p != '\0'; ) {
+				i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(*p));
+				len = strlen((const char *)p);
+				str_append_c(str, ' ');
+				str_append_data(str, p, len);
+				p += len + 1;
+			}
+		}
+		str_append(str, " :");
+		strp = strchr(rec->filename, *MAILDIR_INFO_SEP_S);
+		if (strp == NULL)
+			str_append(str, rec->filename);
+		else
+			str_append_data(str, rec->filename, strp - rec->filename);
+		str_append_c(str, '\n');
+		o_stream_nsend(output, str_data(str), str_len(str));
+	}
+	maildir_uidlist_iter_deinit(&iter);
+
+	if (o_stream_finish(output) < 0) {
+		mailbox_set_critical(uidlist->box, "write(%s) failed: %s",
+				     path, o_stream_get_error(output));
+		o_stream_unref(&output);
+		return -1;
+	}
+
+	*file_size_r = output->offset;
+	o_stream_unref(&output);
+
+	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
+		if (fdatasync(fd) < 0) {
+			mailbox_set_critical(uidlist->box,
+				"fdatasync(%s) failed: %m", path);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static void
+maildir_uidlist_records_drop_expunges(struct maildir_uidlist *uidlist)
+{
+	struct mail_index_view *view;
+	struct maildir_uidlist_rec *const *recs;
+	ARRAY_TYPE(maildir_uidlist_rec_p) new_records;
+	const struct mail_index_header *hdr;
+	const struct mail_index_record *rec;
+	unsigned int i, count;
+	uint32_t seq;
+
+	/* we could get here when opening and locking mailbox,
+	   before index files have been opened. */
+	if (!uidlist->box->opened)
+		return;
+
+	mail_index_refresh(uidlist->box->index);
+	view = mail_index_view_open(uidlist->box->index);
+	count = array_count(&uidlist->records);
+	hdr = mail_index_get_header(view);
+	if (count * UIDLIST_COMPRESS_PERCENTAGE / 100 <= hdr->messages_count) {
+		/* too much trouble to be worth it */
+		mail_index_view_close(&view);
+		return;
+	}
+
+	i_array_init(&new_records, hdr->messages_count + 64);
+	recs = array_get(&uidlist->records, &count);
+	for (i = 0, seq = 1; i < count && seq <= hdr->messages_count; ) {
+		rec = mail_index_lookup(view, seq);
+		if (recs[i]->uid < rec->uid) {
+			/* expunged entry */
+			hash_table_remove(uidlist->files, recs[i]->filename);
+			i++;
+		} else if (recs[i]->uid > rec->uid) {
+			/* index isn't up to date. we're probably just
+			   syncing it here. ignore this entry. */
+			seq++;
+		} else {
+			array_push_back(&new_records, &recs[i]);
+			seq++; i++;
+		}
+	}
+
+	/* drop messages expunged at the end of index */
+	while (i < count && recs[i]->uid < hdr->next_uid) {
+		hash_table_remove(uidlist->files, recs[i]->filename);
+		i++;
+	}
+	/* view might not be completely up-to-date, so preserve any
+	   messages left */
+	for (; i < count; i++)
+		array_push_back(&new_records, &recs[i]);
+
+	array_free(&uidlist->records);
+	uidlist->records = new_records;
+
+	mail_index_view_close(&view);
+}
+
+static int maildir_uidlist_recreate(struct maildir_uidlist *uidlist)
+{
+	struct mailbox *box = uidlist->box;
+	const struct mailbox_permissions *perm = mailbox_get_permissions(box);
+	const char *control_dir, *temp_path;
+	struct stat st;
+	mode_t old_mask;
+	uoff_t file_size;
+	int i, fd, ret;
+
+	i_assert(uidlist->initial_read);
+
+	maildir_uidlist_records_drop_expunges(uidlist);
+
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL,
+				&control_dir) <= 0)
+		i_unreached();
+	temp_path = t_strconcat(control_dir,
+				"/" MAILDIR_UIDLIST_NAME ".tmp", NULL);
+
+	for (i = 0;; i++) {
+		old_mask = umask(0777 & ~perm->file_create_mode);
+		fd = open(temp_path, O_RDWR | O_CREAT | O_TRUNC, 0777);
+		umask(old_mask);
+		if (fd != -1)
+			break;
+
+		if (errno != ENOENT || i == MAILDIR_DELETE_RETRY_COUNT) {
+			mailbox_set_critical(box,
+				"open(%s, O_CREAT) failed: %m", temp_path);
+			return -1;
+		}
+		/* the control dir doesn't exist. create it unless the whole
+		   mailbox was just deleted. */
+		if (!maildir_set_deleted(uidlist->box))
+			return -1;
+	}
+
+	if (perm->file_create_gid != (gid_t)-1 &&
+	    fchown(fd, (uid_t)-1, perm->file_create_gid) < 0) {
+		if (errno == EPERM) {
+			mailbox_set_critical(box, "%s",
+				eperm_error_get_chgrp("fchown", temp_path,
+						perm->file_create_gid,
+						perm->file_create_gid_origin));
+		} else {
+			mailbox_set_critical(box,
+				"fchown(%s) failed: %m", temp_path);
+		}
+	}
+
+	uidlist->read_records_count = 0;
+	ret = maildir_uidlist_write_fd(uidlist, fd, temp_path, 0, &file_size);
+	if (ret == 0) {
+		if (rename(temp_path, uidlist->path) < 0) {
+			mailbox_set_critical(box,
+				"rename(%s, %s) failed: %m",
+				temp_path, uidlist->path);
+			ret = -1;
+		}
+	}
+
+	if (ret < 0)
+		i_unlink(temp_path);
+	else if (fstat(fd, &st) < 0) {
+		mailbox_set_critical(box,
+			"fstat(%s) failed: %m", temp_path);
+		ret = -1;
+	} else if (file_size != (uoff_t)st.st_size) {
+		i_assert(!file_dotlock_is_locked(uidlist->dotlock));
+		mailbox_set_critical(box,
+			"Maildir uidlist dotlock overridden: %s",
+			uidlist->path);
+		ret = -1;
+	} else {
+		maildir_uidlist_close(uidlist);
+		uidlist->fd = fd;
+		uidlist->fd_dev = st.st_dev;
+		uidlist->fd_ino = st.st_ino;
+		uidlist->fd_size = st.st_size;
+		uidlist->last_read_offset = st.st_size;
+		uidlist->recreate = FALSE;
+		uidlist->recreate_on_change = FALSE;
+		uidlist->have_mailbox_guid = TRUE;
+		maildir_uidlist_update_hdr(uidlist, &st);
+	}
+	if (ret < 0)
+		i_close_fd(&fd);
+	return ret;
+}
+
+int maildir_uidlist_update(struct maildir_uidlist *uidlist)
+{
+	int ret;
+
+	if (!uidlist->recreate)
+		return 0;
+
+	if (maildir_uidlist_lock(uidlist) <= 0)
+		return -1;
+	ret = maildir_uidlist_recreate(uidlist);
+	maildir_uidlist_unlock(uidlist);
+	return ret;
+}
+
+static bool maildir_uidlist_want_compress(struct maildir_uidlist_sync_ctx *ctx)
+{
+	unsigned int min_rewrite_count;
+
+	if (!ctx->uidlist->locked_refresh)
+		return FALSE;
+	if (ctx->uidlist->recreate)
+		return TRUE;
+
+	min_rewrite_count =
+		(ctx->uidlist->read_records_count + ctx->new_files_count) *
+		UIDLIST_COMPRESS_PERCENTAGE / 100;
+	return min_rewrite_count >= array_count(&ctx->uidlist->records);
+}
+
+static bool maildir_uidlist_want_recreate(struct maildir_uidlist_sync_ctx *ctx)
+{
+	struct maildir_uidlist *uidlist = ctx->uidlist;
+
+	if (!uidlist->locked_refresh || !uidlist->initial_read)
+		return FALSE;
+
+	if (ctx->finish_change_counter != uidlist->change_counter)
+		return TRUE;
+	if (uidlist->fd == -1 || uidlist->version != UIDLIST_VERSION ||
+	    !uidlist->have_mailbox_guid)
+		return TRUE;
+	return maildir_uidlist_want_compress(ctx);
+}
+
+static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx)
+{
+	struct maildir_uidlist *uidlist = ctx->uidlist;
+	struct stat st;
+	uoff_t file_size;
+
+	if (maildir_uidlist_want_recreate(ctx) || uidlist->recreate_on_change)
+		return maildir_uidlist_recreate(uidlist);
+
+	if (!uidlist->locked_refresh || uidlist->fd == -1) {
+		/* make sure we have the latest file (e.g. NOREFRESH used) */
+		i_assert(uidlist->initial_hdr_read);
+		if (maildir_uidlist_open_latest(uidlist) < 0)
+			return -1;
+		if (uidlist->recreate_on_change)
+			return maildir_uidlist_recreate(uidlist);
+	}
+	i_assert(ctx->first_unwritten_pos != UINT_MAX);
+
+	if (lseek(uidlist->fd, 0, SEEK_END) < 0) {
+		mailbox_set_critical(uidlist->box,
+			"lseek(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+
+	if (maildir_uidlist_write_fd(uidlist, uidlist->fd, uidlist->path,
+				     ctx->first_unwritten_pos, &file_size) < 0)
+		return -1;
+
+	if (fstat(uidlist->fd, &st) < 0) {
+		mailbox_set_critical(uidlist->box,
+			"fstat(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	if ((uoff_t)st.st_size != file_size) {
+		i_warning("%s: file size changed unexpectedly after write",
+			  uidlist->path);
+	} else if (uidlist->locked_refresh) {
+		uidlist->fd_size = st.st_size;
+		uidlist->last_read_offset = st.st_size;
+		maildir_uidlist_update_hdr(uidlist, &st);
+	}
+	return 0;
+}
+
+static void maildir_uidlist_mark_all(struct maildir_uidlist *uidlist,
+				     bool nonsynced)
+{
+	struct maildir_uidlist_rec **recs;
+	unsigned int i, count;
+
+	recs = array_get_modifiable(&uidlist->records, &count);
+	if (nonsynced) {
+		for (i = 0; i < count; i++)
+			recs[i]->flags |= MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+	} else {
+		for (i = 0; i < count; i++)
+			recs[i]->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED);
+	}
+}
+
+static int maildir_uidlist_sync_lock(struct maildir_uidlist *uidlist,
+				     enum maildir_uidlist_sync_flags sync_flags,
+				     bool *locked_r)
+{
+	bool nonblock, refresh;
+	int ret;
+
+	*locked_r = FALSE;
+
+	if ((sync_flags & MAILDIR_UIDLIST_SYNC_NOLOCK) != 0) {
+		if (maildir_uidlist_refresh(uidlist) < 0)
+			return -1;
+		return 1;
+	}
+
+	nonblock = (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0;
+	refresh = (sync_flags & MAILDIR_UIDLIST_SYNC_NOREFRESH) == 0;
+
+	ret = maildir_uidlist_lock_timeout(uidlist, nonblock, refresh, refresh);
+	if (ret <= 0) {
+		if (ret < 0 || !nonblock)
+			return ret;
+
+		/* couldn't lock it */
+		if ((sync_flags & MAILDIR_UIDLIST_SYNC_FORCE) == 0)
+			return 0;
+		/* forcing the sync anyway */
+		if (maildir_uidlist_refresh(uidlist) < 0)
+			return -1;
+	} else {
+		*locked_r = TRUE;
+	}
+	return 1;
+}
+
+void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist)
+{
+	maildir_uidlist_mark_all(uidlist, TRUE);
+}
+
+int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,
+			      enum maildir_uidlist_sync_flags sync_flags,
+			      struct maildir_uidlist_sync_ctx **sync_ctx_r)
+{
+	struct maildir_uidlist_sync_ctx *ctx;
+	bool locked;
+	int ret;
+
+	ret = maildir_uidlist_sync_lock(uidlist, sync_flags, &locked);
+	if (ret <= 0)
+		return ret;
+
+	*sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1);
+	ctx->uidlist = uidlist;
+	ctx->sync_flags = sync_flags;
+	ctx->partial = !locked ||
+		(sync_flags & MAILDIR_UIDLIST_SYNC_PARTIAL) != 0;
+	ctx->locked = locked;
+	ctx->first_unwritten_pos = UINT_MAX;
+	ctx->first_new_pos = UINT_MAX;
+
+	if (ctx->partial) {
+		if ((sync_flags & MAILDIR_UIDLIST_SYNC_KEEP_STATE) == 0) {
+			/* initially mark all nonsynced */
+			maildir_uidlist_mark_all(uidlist, TRUE);
+		}
+		return 1;
+	}
+	i_assert(uidlist->locked_refresh);
+
+	ctx->record_pool = pool_alloconly_create(MEMPOOL_GROWING
+						 "maildir_uidlist_sync", 16384);
+	hash_table_create(&ctx->files, ctx->record_pool, 4096,
+			  maildir_filename_base_hash,
+			  maildir_filename_base_cmp);
+
+	i_array_init(&ctx->records, array_count(&uidlist->records));
+	return 1;
+}
+
+static int
+maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx,
+				  const char *filename, uint32_t uid,
+				  enum maildir_uidlist_rec_flag flags,
+				  struct maildir_uidlist_rec **rec_r)
+{
+	struct maildir_uidlist *uidlist = ctx->uidlist;
+	struct maildir_uidlist_rec *rec, *const *recs;
+	unsigned int count;
+
+	/* we'll update uidlist directly */
+	rec = hash_table_lookup(uidlist->files, filename);
+	if (rec == NULL) {
+		/* doesn't exist in uidlist */
+		if (!ctx->locked) {
+			/* we can't add it, so just ignore it */
+			return 1;
+		}
+		if (ctx->first_new_pos == UINT_MAX)
+			ctx->first_new_pos = array_count(&uidlist->records);
+		ctx->new_files_count++;
+		ctx->changed = TRUE;
+
+		if (uidlist->record_pool == NULL) {
+			uidlist->record_pool =
+				pool_alloconly_create(MEMPOOL_GROWING
+						      "uidlist record_pool",
+						      1024);
+		}
+
+		rec = p_new(uidlist->record_pool,
+			    struct maildir_uidlist_rec, 1);
+		rec->uid = (uint32_t)-1;
+		rec->filename = p_strdup(uidlist->record_pool, filename);
+		array_push_back(&uidlist->records, &rec);
+		uidlist->change_counter++;
+
+		hash_table_insert(uidlist->files, rec->filename, rec);
+	} else if (strcmp(rec->filename, filename) != 0) {
+		rec->filename = p_strdup(uidlist->record_pool, filename);
+	}
+	if (uid != 0) {
+		if (rec->uid != uid && rec->uid != (uint32_t)-1) {
+			mailbox_set_critical(uidlist->box,
+				"Maildir: %s changed UID %u -> %u",
+				filename, rec->uid, uid);
+			return -1;
+		}
+		rec->uid = uid;
+		if (uidlist->next_uid <= uid)
+			uidlist->next_uid = uid + 1;
+		else {
+			recs = array_get(&uidlist->records, &count);
+			if (count > 1 && uid < recs[count-1]->uid)
+				uidlist->unsorted = TRUE;
+		}
+	}
+
+	rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR);
+	rec->flags = (rec->flags | flags) &
+		ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED);
+
+	ctx->finished = FALSE;
+	*rec_r = rec;
+	return 1;
+}
+
+static unsigned char *ext_dup(pool_t pool, const unsigned char *extensions)
+{
+	unsigned char *ret;
+
+	if (extensions == NULL)
+		return NULL;
+
+	T_BEGIN {
+		unsigned int len;
+
+		for (len = 0; extensions[len] != '\0'; len++) {
+			while (extensions[len] != '\0') len++;
+		}
+		ret = p_malloc(pool, len + 1);
+		memcpy(ret, extensions, len);
+	} T_END;
+	return ret;
+}
+
+int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
+			      const char *filename,
+			      enum maildir_uidlist_rec_flag flags)
+{
+	struct maildir_uidlist_rec *rec;
+
+	return maildir_uidlist_sync_next_uid(ctx, filename, 0, flags, &rec);
+}
+
+int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx,
+				  const char *filename, uint32_t uid,
+				  enum maildir_uidlist_rec_flag flags,
+				  struct maildir_uidlist_rec **rec_r)
+{
+	struct maildir_uidlist *uidlist = ctx->uidlist;
+	struct maildir_uidlist_rec *rec, *old_rec;
+	const char *p;
+
+	*rec_r = NULL;
+
+	if (ctx->failed)
+		return -1;
+	for (p = filename; *p != '\0'; p++) {
+		if (*p == 13 || *p == 10) {
+			i_warning("Maildir %s: Ignoring a file with #0x%x: %s",
+				  mailbox_get_path(uidlist->box), *p, filename);
+			return 1;
+		}
+	}
+
+	if (ctx->partial) {
+		return maildir_uidlist_sync_next_partial(ctx, filename,
+							 uid, flags, rec_r);
+	}
+
+	rec = hash_table_lookup(ctx->files, filename);
+	if (rec != NULL) {
+		if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+				   MAILDIR_UIDLIST_REC_FLAG_MOVED)) == 0) {
+			/* possibly duplicate */
+			return 0;
+		}
+
+		/* probably was in new/ and now we're seeing it in cur/.
+		   remove new/moved flags so if this happens again we'll know
+		   to check for duplicates. */
+		rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+					  MAILDIR_UIDLIST_REC_FLAG_MOVED);
+		if (strcmp(rec->filename, filename) != 0)
+			rec->filename = p_strdup(ctx->record_pool, filename);
+	} else {
+		old_rec = hash_table_lookup(uidlist->files, filename);
+		i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist));
+
+		rec = p_new(ctx->record_pool, struct maildir_uidlist_rec, 1);
+
+		if (old_rec != NULL) {
+			*rec = *old_rec;
+			rec->extensions =
+				ext_dup(ctx->record_pool, rec->extensions);
+		} else {
+			rec->uid = (uint32_t)-1;
+			ctx->new_files_count++;
+			ctx->changed = TRUE;
+			/* didn't exist in uidlist, it's recent */
+			flags |= MAILDIR_UIDLIST_REC_FLAG_RECENT;
+		}
+		rec->filename = p_strdup(ctx->record_pool, filename);
+		hash_table_insert(ctx->files, rec->filename, rec);
+
+		array_push_back(&ctx->records, &rec);
+	}
+	if (uid != 0) {
+		rec->uid = uid;
+		if (uidlist->next_uid <= uid)
+			uidlist->next_uid = uid + 1;
+	}
+
+	rec->flags = (rec->flags | flags) & ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED);
+	*rec_r = rec;
+	return 1;
+}
+
+void maildir_uidlist_sync_remove(struct maildir_uidlist_sync_ctx *ctx,
+				 const char *filename)
+{
+	struct maildir_uidlist_rec *rec;
+	unsigned int idx;
+
+	i_assert(ctx->partial);
+	i_assert(ctx->uidlist->locked_refresh);
+
+	rec = hash_table_lookup(ctx->uidlist->files, filename);
+	i_assert(rec != NULL);
+	i_assert(rec->uid != (uint32_t)-1);
+
+	hash_table_remove(ctx->uidlist->files, filename);
+	idx = maildir_uidlist_records_array_delete(ctx->uidlist, rec);
+
+	if (ctx->first_unwritten_pos != UINT_MAX) {
+		i_assert(ctx->first_unwritten_pos > idx);
+		ctx->first_unwritten_pos--;
+	}
+	if (ctx->first_new_pos != UINT_MAX) {
+		i_assert(ctx->first_new_pos > idx);
+		ctx->first_new_pos--;
+	}
+
+	ctx->changed = TRUE;
+	ctx->uidlist->recreate = TRUE;
+}
+
+void maildir_uidlist_sync_set_ext(struct maildir_uidlist_sync_ctx *ctx,
+				  struct maildir_uidlist_rec *rec,
+				  enum maildir_uidlist_rec_ext_key key,
+				  const char *value)
+{
+	pool_t pool = ctx->partial ?
+		ctx->uidlist->record_pool : ctx->record_pool;
+
+	i_assert(MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(key));
+
+	maildir_uidlist_rec_set_ext(rec, pool, key, value);
+}
+
+const char *
+maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx,
+				       const char *filename)
+{
+	struct maildir_uidlist_rec *rec;
+
+	rec = hash_table_lookup(ctx->files, filename);
+	return rec == NULL ? NULL : rec->filename;
+}
+
+bool maildir_uidlist_get_uid(struct maildir_uidlist *uidlist,
+			     const char *filename, uint32_t *uid_r)
+{
+	struct maildir_uidlist_rec *rec;
+
+	rec = hash_table_lookup(uidlist->files, filename);
+	if (rec == NULL)
+		return FALSE;
+
+	*uid_r = rec->uid;
+	return TRUE;
+}
+
+void maildir_uidlist_update_fname(struct maildir_uidlist *uidlist,
+				  const char *filename)
+{
+	struct maildir_uidlist_rec *rec;
+
+	rec = hash_table_lookup(uidlist->files, filename);
+	if (rec == NULL)
+		return;
+
+	rec->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_NONSYNCED);
+	if (strcmp(rec->filename, filename) != 0)
+		rec->filename = p_strdup(uidlist->record_pool, filename);
+}
+
+const char *
+maildir_uidlist_get_full_filename(struct maildir_uidlist *uidlist,
+				  const char *filename)
+{
+	struct maildir_uidlist_rec *rec;
+
+	rec = hash_table_lookup(uidlist->files, filename);
+	return rec == NULL ? NULL : rec->filename;
+}
+
+static int maildir_assign_uid_cmp(const void *p1, const void *p2)
+{
+	const struct maildir_uidlist_rec *const *rec1 = p1, *const *rec2 = p2;
+
+	if ((*rec1)->uid != (*rec2)->uid) {
+		if ((*rec1)->uid < (*rec2)->uid)
+			return -1;
+		else
+			return 1;
+	}
+	return maildir_filename_sort_cmp((*rec1)->filename, (*rec2)->filename);
+}
+
+static void maildir_uidlist_assign_uids(struct maildir_uidlist_sync_ctx *ctx)
+{
+	struct maildir_uidlist_rec **recs;
+	unsigned int dest, count;
+
+	i_assert(UIDLIST_IS_LOCKED(ctx->uidlist));
+	i_assert(ctx->first_new_pos != UINT_MAX);
+
+	if (ctx->first_unwritten_pos == UINT_MAX)
+		ctx->first_unwritten_pos = ctx->first_new_pos;
+
+	/* sort new files and assign UIDs for them */
+	recs = array_get_modifiable(&ctx->uidlist->records, &count);
+	qsort(recs + ctx->first_new_pos, count - ctx->first_new_pos,
+	      sizeof(*recs), maildir_assign_uid_cmp);
+
+	for (dest = ctx->first_new_pos; dest < count; dest++) {
+		if (recs[dest]->uid == (uint32_t)-1)
+			break;
+	}
+
+	for (; dest < count; dest++) {
+		i_assert(recs[dest]->uid == (uint32_t)-1);
+		i_assert(ctx->uidlist->next_uid < (uint32_t)-1);
+		recs[dest]->uid = ctx->uidlist->next_uid++;
+		recs[dest]->flags &= ENUM_NEGATE(MAILDIR_UIDLIST_REC_FLAG_MOVED);
+	}
+
+	if (ctx->uidlist->locked_refresh && ctx->uidlist->initial_read)
+		ctx->uidlist->last_seen_uid = ctx->uidlist->next_uid-1;
+
+	ctx->new_files_count = 0;
+	ctx->first_new_pos = UINT_MAX;
+	ctx->uidlist->change_counter++;
+	ctx->finish_change_counter = ctx->uidlist->change_counter;
+}
+
+static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx)
+{
+	struct maildir_uidlist *uidlist = ctx->uidlist;
+
+	/* buffer is unsorted, sort it by UID */
+	array_sort(&ctx->records, maildir_uid_cmp);
+
+	array_free(&uidlist->records);
+	uidlist->records = ctx->records;
+	ctx->records.arr.buffer = NULL;
+	i_assert(array_is_created(&uidlist->records));
+
+	hash_table_destroy(&uidlist->files);
+	uidlist->files = ctx->files;
+	i_zero(&ctx->files);
+
+	pool_unref(&uidlist->record_pool);
+	uidlist->record_pool = ctx->record_pool;
+	ctx->record_pool = NULL;
+
+	if (ctx->new_files_count != 0) {
+		ctx->first_new_pos = array_count(&uidlist->records) -
+			ctx->new_files_count;
+		maildir_uidlist_assign_uids(ctx);
+	} else {
+		ctx->uidlist->change_counter++;
+	}
+}
+
+void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx)
+{
+	ctx->uidlist->recreate = TRUE;
+}
+
+void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx)
+{
+	if (!ctx->partial) {
+		if (!ctx->failed)
+			maildir_uidlist_swap(ctx);
+	} else {
+		if (ctx->new_files_count != 0 && !ctx->failed) {
+			i_assert(ctx->changed);
+			i_assert(ctx->locked);
+			maildir_uidlist_assign_uids(ctx);
+		}
+	}
+
+	ctx->finished = TRUE;
+
+	/* mbox=NULL means we're coming from dbox rebuilding code.
+	   the dbox is already locked, so allow uidlist recreation */
+	i_assert(ctx->locked || !ctx->changed);
+	if ((ctx->changed || maildir_uidlist_want_compress(ctx)) &&
+	    !ctx->failed && ctx->locked) {
+		T_BEGIN {
+			if (maildir_uidlist_sync_update(ctx) < 0) {
+				/* we couldn't write everything we wanted. make
+				   sure we don't continue using those UIDs */
+				maildir_uidlist_reset(ctx->uidlist);
+				ctx->failed = TRUE;
+			}
+		} T_END;
+	}
+}
+
+int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **_ctx,
+				bool success)
+{
+	struct maildir_uidlist_sync_ctx *ctx = *_ctx;
+	int ret;
+
+	*_ctx = NULL;
+
+	if (!success)
+		ctx->failed = TRUE;
+	ret = ctx->failed ? -1 : 0;
+
+	if (!ctx->finished)
+		maildir_uidlist_sync_finish(ctx);
+	if (ctx->partial)
+		maildir_uidlist_mark_all(ctx->uidlist, FALSE);
+	if (ctx->locked)
+		maildir_uidlist_unlock(ctx->uidlist);
+
+	hash_table_destroy(&ctx->files);
+	pool_unref(&ctx->record_pool);
+	if (array_is_created(&ctx->records))
+		array_free(&ctx->records);
+	i_free(ctx);
+	return ret;
+}
+
+void maildir_uidlist_add_flags(struct maildir_uidlist *uidlist,
+			       const char *filename,
+			       enum maildir_uidlist_rec_flag flags)
+{
+	struct maildir_uidlist_rec *rec;
+
+	rec = hash_table_lookup(uidlist->files, filename);
+	i_assert(rec != NULL);
+
+	rec->flags |= flags;
+}
+
+struct maildir_uidlist_iter_ctx *
+maildir_uidlist_iter_init(struct maildir_uidlist *uidlist)
+{
+	struct maildir_uidlist_iter_ctx *ctx;
+	unsigned int count;
+
+	ctx = i_new(struct maildir_uidlist_iter_ctx, 1);
+	ctx->uidlist = uidlist;
+	ctx->next = array_get(&uidlist->records, &count);
+	ctx->end = ctx->next + count;
+	ctx->change_counter = ctx->uidlist->change_counter;
+	return ctx;
+}
+
+static void
+maildir_uidlist_iter_update_idx(struct maildir_uidlist_iter_ctx *ctx)
+{
+	unsigned int old_rev_idx, idx, count;
+
+	old_rev_idx = ctx->end - ctx->next;
+	ctx->next = array_get(&ctx->uidlist->records, &count);
+	ctx->end = ctx->next + count;
+
+	idx = old_rev_idx >= count ? 0 :
+		count - old_rev_idx;
+	while (idx < count && ctx->next[idx]->uid <= ctx->prev_uid)
+		idx++;
+	while (idx > 0 && ctx->next[idx-1]->uid > ctx->prev_uid)
+		idx--;
+
+	ctx->next += idx;
+}
+
+static bool maildir_uidlist_iter_next_rec(struct maildir_uidlist_iter_ctx *ctx,
+					  struct maildir_uidlist_rec **rec_r)
+{
+	struct maildir_uidlist_rec *rec;
+
+	if (ctx->change_counter != ctx->uidlist->change_counter)
+		maildir_uidlist_iter_update_idx(ctx);
+
+	if (ctx->next == ctx->end)
+		return FALSE;
+
+	rec = *ctx->next;
+	i_assert(rec->uid != (uint32_t)-1);
+
+	ctx->prev_uid = rec->uid;
+	ctx->next++;
+
+	*rec_r = rec;
+	return TRUE;
+}
+
+bool maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx,
+			       uint32_t *uid_r,
+			       enum maildir_uidlist_rec_flag *flags_r,
+			       const char **filename_r)
+{
+	struct maildir_uidlist_rec *rec;
+
+	if (!maildir_uidlist_iter_next_rec(ctx, &rec))
+		return FALSE;
+
+	*uid_r = rec->uid;
+	*flags_r = rec->flags;
+	*filename_r = rec->filename;
+	return TRUE;
+}
+
+void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx **_ctx)
+{
+	i_free(*_ctx);
+	*_ctx = NULL;
+}
diff --git a/src/lib-storage/index/maildir/maildir-uidlist.h b/src/lib-storage/index/maildir/maildir-uidlist.h
new file mode 100644
index 0000000..8b45723
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-uidlist.h
@@ -0,0 +1,161 @@
+#ifndef MAILDIR_UIDLIST_H
+#define MAILDIR_UIDLIST_H
+
+#include "mail-storage.h"
+
+#define MAILDIR_UIDLIST_NAME "dovecot-uidlist"
+/* how many seconds to wait before overriding uidlist.lock */
+#define MAILDIR_UIDLIST_LOCK_STALE_TIMEOUT (60*2)
+
+struct maildir_mailbox;
+struct maildir_uidlist;
+struct maildir_uidlist_sync_ctx;
+struct maildir_uidlist_rec;
+
+enum maildir_uidlist_sync_flags {
+	MAILDIR_UIDLIST_SYNC_PARTIAL	= 0x01,
+	MAILDIR_UIDLIST_SYNC_KEEP_STATE	= 0x02,
+	MAILDIR_UIDLIST_SYNC_FORCE	= 0x04,
+	MAILDIR_UIDLIST_SYNC_TRYLOCK	= 0x08,
+	MAILDIR_UIDLIST_SYNC_NOREFRESH	= 0x10,
+	MAILDIR_UIDLIST_SYNC_NOLOCK	= 0x20
+};
+
+enum maildir_uidlist_rec_flag {
+	MAILDIR_UIDLIST_REC_FLAG_NEW_DIR	= 0x01,
+	MAILDIR_UIDLIST_REC_FLAG_MOVED		= 0x02,
+	MAILDIR_UIDLIST_REC_FLAG_RECENT		= 0x04,
+	MAILDIR_UIDLIST_REC_FLAG_NONSYNCED	= 0x08,
+	MAILDIR_UIDLIST_REC_FLAG_RACING		= 0x10
+};
+
+enum maildir_uidlist_hdr_ext_key {
+	MAILDIR_UIDLIST_HDR_EXT_UID_VALIDITY		= 'V',
+	MAILDIR_UIDLIST_HDR_EXT_NEXT_UID		= 'N',
+	MAILDIR_UIDLIST_HDR_EXT_GUID			= 'G',
+	/* POP3 UIDL format unless overridden by records */
+	MAILDIR_UIDLIST_HDR_EXT_POP3_UIDL_FORMAT	= 'P'
+};
+
+#define MAILDIR_UIDLIST_REC_EXT_KEY_IS_VALID(c) \
+	((c) >= 'A' && (c) <= 'Z')
+enum maildir_uidlist_rec_ext_key {
+	/* Physical message size. If filename also contains ,S=<vsize> this
+	   isn't written to uidlist. */
+	MAILDIR_UIDLIST_REC_EXT_PSIZE		= 'S',
+	/* Virtual message size. If filename also contains ,W=<vsize> this
+	   isn't written to uidlist. */
+	MAILDIR_UIDLIST_REC_EXT_VSIZE		= 'W',
+	/* POP3 UIDL overriding the default format */
+	MAILDIR_UIDLIST_REC_EXT_POP3_UIDL	= 'P',
+	/* POP3 message ordering number. Lower numbered messages are listed
+	   first. Messages without ordering number are listed after them.
+	   The idea is to be able to preserve POP3 UIDL list and IMAP UIDs
+	   perfectly when migrating from other servers. */
+	MAILDIR_UIDLIST_REC_EXT_POP3_ORDER	= 'O',
+	/* Message GUID (default is the base filename) */
+	MAILDIR_UIDLIST_REC_EXT_GUID		= 'G'
+};
+
+int maildir_uidlist_lock(struct maildir_uidlist *uidlist);
+int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist);
+int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist);
+void maildir_uidlist_unlock(struct maildir_uidlist *uidlist);
+bool maildir_uidlist_is_locked(struct maildir_uidlist *uidlist);
+bool maildir_uidlist_is_read(struct maildir_uidlist *uidlist);
+/* Returns TRUE if uidlist file is currently open */
+bool maildir_uidlist_is_open(struct maildir_uidlist *uidlist);
+
+struct maildir_uidlist *maildir_uidlist_init(struct maildir_mailbox *mbox);
+void maildir_uidlist_deinit(struct maildir_uidlist **uidlist);
+
+/* Returns -1 if error, 0 if file is broken or lost, 1 if ok. If nfs_flush=TRUE
+   and storage has NFS_FLUSH flag set, the NFS attribute cache is flushed to
+   make sure that we see the latest uidlist file. */
+int maildir_uidlist_refresh(struct maildir_uidlist *uidlist);
+/* Like maildir_uidlist_refresh(), but if uidlist isn't opened yet, try to
+   fill in the uidvalidity/nextuid from index file instead. */
+int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist);
+
+/* Look up uidlist record for given filename. Returns 1 if found,
+   0 if not found, -1 if error */
+int maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid,
+			   enum maildir_uidlist_rec_flag *flags_r,
+			   const char **fname_r);
+/* Returns extension's value or NULL if it doesn't exist. */
+const char *
+maildir_uidlist_lookup_ext(struct maildir_uidlist *uidlist, uint32_t uid,
+			   enum maildir_uidlist_rec_ext_key key);
+
+uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist);
+uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist);
+int maildir_uidlist_get_mailbox_guid(struct maildir_uidlist *uidlist,
+				     guid_128_t mailbox_guid);
+void maildir_uidlist_set_mailbox_guid(struct maildir_uidlist *uidlist,
+				      const guid_128_t mailbox_guid);
+
+void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist,
+				      uint32_t uid_validity);
+void maildir_uidlist_set_next_uid(struct maildir_uidlist *uidlist,
+				  uint32_t next_uid, bool force);
+
+/* Update extended record. */
+void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid,
+			     enum maildir_uidlist_rec_ext_key key,
+			     const char *value);
+void maildir_uidlist_unset_ext(struct maildir_uidlist *uidlist, uint32_t uid,
+			       enum maildir_uidlist_rec_ext_key key);
+
+/* If uidlist has changed, update it. This is mostly meant to be used with
+   maildir_uidlist_set_ext() */
+int maildir_uidlist_update(struct maildir_uidlist *uidlist);
+
+void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist);
+/* Sync uidlist with what's actually on maildir. Returns same as
+   maildir_uidlist_lock(). */
+int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,
+			      enum maildir_uidlist_sync_flags sync_flags,
+			      struct maildir_uidlist_sync_ctx **sync_ctx_r);
+int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
+			      const char *filename,
+			      enum maildir_uidlist_rec_flag flags);
+int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx,
+				  const char *filename, uint32_t uid,
+				  enum maildir_uidlist_rec_flag flags,
+				  struct maildir_uidlist_rec **rec_r);
+void maildir_uidlist_sync_remove(struct maildir_uidlist_sync_ctx *ctx,
+				 const char *filename);
+void maildir_uidlist_sync_set_ext(struct maildir_uidlist_sync_ctx *ctx,
+				  struct maildir_uidlist_rec *rec,
+				  enum maildir_uidlist_rec_ext_key key,
+				  const char *value);
+void maildir_uidlist_update_fname(struct maildir_uidlist *uidlist,
+				  const char *filename);
+const char *
+maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx,
+				       const char *filename);
+void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx);
+void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx);
+int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **ctx,
+				bool success);
+
+bool maildir_uidlist_get_uid(struct maildir_uidlist *uidlist,
+			     const char *filename, uint32_t *uid_r);
+const char *
+maildir_uidlist_get_full_filename(struct maildir_uidlist *uidlist,
+				  const char *filename);
+
+void maildir_uidlist_add_flags(struct maildir_uidlist *uidlist,
+			       const char *filename,
+			       enum maildir_uidlist_rec_flag flags);
+
+/* List all maildir files. */
+struct maildir_uidlist_iter_ctx *
+maildir_uidlist_iter_init(struct maildir_uidlist *uidlist);
+bool maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx,
+			       uint32_t *uid_r,
+			       enum maildir_uidlist_rec_flag *flags_r,
+			       const char **filename_r);
+void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx **ctx);
+
+#endif
diff --git a/src/lib-storage/index/maildir/maildir-util.c b/src/lib-storage/index/maildir/maildir-util.c
new file mode 100644
index 0000000..c03546e
--- /dev/null
+++ b/src/lib-storage/index/maildir/maildir-util.c
@@ -0,0 +1,323 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "str.h"
+#include "mkdir-parents.h"
+#include "mailbox-list-private.h"
+#include "maildir-storage.h"
+#include "maildir-uidlist.h"
+#include "maildir-keywords.h"
+#include "maildir-filename-flags.h"
+#include "maildir-sync.h"
+#include "mailbox-recent-flags.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <utime.h>
+#include <sys/stat.h>
+
+#define MAILDIR_RESYNC_RETRY_COUNT 10
+
+static const char *
+maildir_filename_guess(struct maildir_mailbox *mbox, uint32_t uid,
+		       const char *fname,
+		       enum maildir_uidlist_rec_flag *uidlist_flags,
+		       bool *have_flags_r)
+
+{
+	struct mail_index_view *view = mbox->flags_view;
+	struct maildir_keywords_sync_ctx *kw_ctx;
+	enum mail_flags flags;
+	ARRAY_TYPE(keyword_indexes) keywords;
+	const char *p;
+	uint32_t seq;
+
+	if (view == NULL || !mail_index_lookup_seq(view, uid, &seq)) {
+		*have_flags_r = FALSE;
+		return fname;
+	}
+
+	t_array_init(&keywords, 32);
+	mail_index_lookup_view_flags(view, seq, &flags, &keywords);
+	if (array_count(&keywords) == 0) {
+		*have_flags_r = (flags & MAIL_FLAGS_NONRECENT) != 0;
+		fname = maildir_filename_flags_set(fname, flags);
+	} else {
+		*have_flags_r = TRUE;
+		kw_ctx = maildir_keywords_sync_init_readonly(mbox->keywords,
+							     mbox->box.index);
+		fname = maildir_filename_flags_kw_set(kw_ctx, fname,
+						      flags, &keywords);
+		maildir_keywords_sync_deinit(&kw_ctx);
+	}
+
+	if (*have_flags_r) {
+		/* don't even bother looking into new/ dir */
+		*uidlist_flags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+	} else if ((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0 &&
+		   ((*uidlist_flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 ||
+		    mailbox_recent_flags_have_uid(&mbox->box, uid))) {
+		/* probably in new/ dir, drop ":2," from fname */
+		*uidlist_flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+		p = strrchr(fname, MAILDIR_INFO_SEP);
+		if (p != NULL)
+			fname = t_strdup_until(fname, p);
+	}
+
+	return fname;
+}
+
+static int maildir_file_do_try(struct maildir_mailbox *mbox, uint32_t uid,
+			       maildir_file_do_func *callback, void *context)
+{
+	const char *path, *fname;
+	enum maildir_uidlist_rec_flag flags;
+	bool have_flags;
+	int ret;
+
+	ret = maildir_sync_lookup(mbox, uid, &flags, &fname);
+	if (ret <= 0)
+		return ret == 0 ? -2 : -1;
+
+	if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
+		/* let's see if we can guess the filename based on index */
+		fname = maildir_filename_guess(mbox, uid, fname,
+					       &flags, &have_flags);
+	}
+	/* make a copy, just in case callback refreshes uidlist and
+	   the pointer becomes invalid. */
+	fname = t_strdup(fname);
+
+	ret = 0;
+	if ((flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) {
+		/* probably in new/ dir */
+		path = t_strconcat(mailbox_get_path(&mbox->box),
+				   "/new/", fname, NULL);
+		ret = callback(mbox, path, context);
+	}
+	if (ret == 0) {
+		path = t_strconcat(mailbox_get_path(&mbox->box), "/cur/",
+				   fname, NULL);
+		ret = callback(mbox, path, context);
+	}
+	if (ret > 0 && (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
+		/* file was found. make sure we remember its latest name. */
+		maildir_uidlist_update_fname(mbox->uidlist, fname);
+	} else if (ret == 0 &&
+		   (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) {
+		/* file wasn't found. mark this message nonsynced, so we can
+		   retry the lookup by guessing the flags */
+		maildir_uidlist_add_flags(mbox->uidlist, fname,
+					  MAILDIR_UIDLIST_REC_FLAG_NONSYNCED);
+	}
+	return ret;
+}
+
+static int do_racecheck(struct maildir_mailbox *mbox, const char *path,
+			void *context)
+{
+	const uint32_t *uidp = context;
+	struct stat st;
+	int ret;
+
+	ret = lstat(path, &st);
+	if (ret == 0 && (st.st_mode & S_IFMT) == S_IFLNK) {
+		/* most likely a symlink pointing to a nonexistent file */
+		mailbox_set_critical(&mbox->box,
+			"Maildir: Symlink destination doesn't exist for UID=%u: %s", *uidp, path);
+		return -2;
+	} else if (ret < 0 && errno != ENOENT) {
+		mailbox_set_critical(&mbox->box, "lstat(%s) failed: %m", path);
+		return -1;
+	} else {
+		/* success or ENOENT, either way we're done */
+		mailbox_set_critical(&mbox->box,
+			"maildir_file_do(%s): Filename keeps changing for UID=%u", path, *uidp);
+		return -1;
+	}
+}
+
+#undef maildir_file_do
+int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid,
+		    maildir_file_do_func *callback, void *context)
+{
+	int i, ret;
+
+	T_BEGIN {
+		ret = maildir_file_do_try(mbox, uid, callback, context);
+	} T_END;
+	if (ret == 0 && mbox->storage->set->maildir_very_dirty_syncs) T_BEGIN {
+		/* try guessing again with refreshed flags */
+		if (maildir_sync_refresh_flags_view(mbox) == 0)
+			ret = maildir_file_do_try(mbox, uid, callback, context);
+	} T_END;
+	for (i = 0; i < MAILDIR_RESYNC_RETRY_COUNT && ret == 0; i++) {
+		/* file is either renamed or deleted. sync the maildir and
+		   see which one. if file appears to be renamed constantly,
+		   don't try to open it more than 10 times. */
+		if (maildir_storage_sync_force(mbox, uid) < 0)
+			return -1;
+
+		T_BEGIN {
+			ret = maildir_file_do_try(mbox, uid, callback, context);
+		} T_END;
+	}
+
+	if (i == MAILDIR_RESYNC_RETRY_COUNT) T_BEGIN {
+		ret = maildir_file_do_try(mbox, uid, do_racecheck, &uid);
+	} T_END;
+
+	return ret == -2 ? 0 : ret;
+}
+
+static int maildir_create_path(struct mailbox *box, const char *path,
+			       enum mailbox_list_path_type type, bool retry)
+{
+	const struct mailbox_permissions *perm = mailbox_get_permissions(box);
+	const char *p, *parent;
+
+	if (mkdir_chgrp(path, perm->dir_create_mode, perm->file_create_gid,
+			perm->file_create_gid_origin) == 0)
+		return 0;
+
+	switch (errno) {
+	case EEXIST:
+		return 0;
+	case ENOENT:
+		p = strrchr(path, '/');
+		if (type == MAILBOX_LIST_PATH_TYPE_MAILBOX ||
+		    p == NULL || !retry) {
+			/* mailbox was being deleted just now */
+			mailbox_set_deleted(box);
+			return -1;
+		}
+		/* create index/control root directory */
+		parent = t_strdup_until(path, p);
+		if (mailbox_list_mkdir_root(box->list, parent, type) < 0) {
+			mail_storage_copy_list_error(box->storage, box->list);
+			return -1;
+		}
+		/* should work now, try again */
+		return maildir_create_path(box, path, type, FALSE);
+	default:
+		mailbox_set_critical(box, "mkdir(%s) failed: %m", path);
+		return -1;
+	}
+}
+
+static int maildir_create_subdirs(struct mailbox *box)
+{
+	static const char *subdirs[] = { "cur", "new", "tmp" };
+	const char *dirs[N_ELEMENTS(subdirs) + 2];
+	enum mailbox_list_path_type types[N_ELEMENTS(subdirs) + 2];
+	struct stat st;
+	const char *path;
+	unsigned int i, count;
+
+	/* @UNSAFE: get a list of directories we want to create */
+	for (i = 0; i < N_ELEMENTS(subdirs); i++) {
+		types[i] = MAILBOX_LIST_PATH_TYPE_MAILBOX;
+		dirs[i] = t_strconcat(mailbox_get_path(box),
+				      "/", subdirs[i], NULL);
+	}
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_CONTROL, &path) > 0) {
+		types[i] = MAILBOX_LIST_PATH_TYPE_CONTROL;
+		dirs[i++] = path;
+	}
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) > 0) {
+		types[i] = MAILBOX_LIST_PATH_TYPE_INDEX;
+		dirs[i++] = path;
+	}
+	count = i;
+	i_assert(count <= N_ELEMENTS(dirs));
+
+	for (i = 0; i < count; i++) {
+		path = dirs[i];
+		if (stat(path, &st) == 0)
+			continue;
+		if (errno != ENOENT) {
+			mailbox_set_critical(box, "stat(%s) failed: %m", path);
+			break;
+		}
+		if (maildir_create_path(box, path, types[i], TRUE) < 0)
+			break;
+	}
+	return i == N_ELEMENTS(dirs) ? 0 : -1;
+}
+
+bool maildir_set_deleted(struct mailbox *box)
+{
+	struct stat st;
+	int ret;
+
+	if (stat(mailbox_get_path(box), &st) < 0) {
+		if (errno == ENOENT)
+			mailbox_set_deleted(box);
+		else {
+			mailbox_set_critical(box,
+				"stat(%s) failed: %m", mailbox_get_path(box));
+		}
+		return FALSE;
+	}
+	/* maildir itself exists. create all of its subdirectories in case
+	   they got lost. */
+	T_BEGIN {
+		ret = maildir_create_subdirs(box);
+	} T_END;
+	return ret < 0 ? FALSE : TRUE;
+}
+
+int maildir_lose_unexpected_dir(struct mail_storage *storage, const char *path)
+{
+	const char *dest, *fname, *p;
+
+	/* There's a directory in maildir, get rid of it.
+
+	   In some installations this was caused by a messed up configuration
+	   where e.g. mails was initially delivered to new/new/ directory.
+	   Also Dovecot v2.0.0 - v2.0.4 sometimes may have renamed tmp/
+	   directory under new/ or cur/. */
+	if (rmdir(path) == 0) {
+		mail_storage_set_critical(storage,
+			"Maildir: rmdir()ed unwanted empty directory: %s",
+			path);
+		return 1;
+	} else if (errno == ENOENT) {
+		/* someone else rmdired or renamed it */
+		return 0;
+	} else if (errno != ENOTEMPTY) {
+		mail_storage_set_critical(storage,
+			"Maildir: Found unwanted directory %s, "
+			"but rmdir() failed: %m", path);
+		return -1;
+	}
+
+	/* It's not safe to delete this directory since it has some files in it,
+	   but it's also not helpful to log this message over and over again.
+	   Get rid of this error by renaming the directory elsewhere */
+	p = strrchr(path, '/');
+	i_assert(p != NULL);
+	fname = p + 1;
+	while (p != path && p[-1] != '/') p--;
+	i_assert(p != NULL);
+
+	dest = t_strconcat(t_strdup_until(path, p), "extra-", fname, NULL);
+	if (rename(path, dest) == 0) {
+		mail_storage_set_critical(storage,
+			"Maildir: renamed unwanted directory %s to %s",
+			path, dest);
+		return 1;
+	} else if (errno == ENOENT) {
+		/* someone else renamed it (could have been flag change) */
+		return 0;
+	} else {
+		mail_storage_set_critical(storage,
+			"Maildir: Found unwanted directory, "
+			"but rename(%s, %s) failed: %m", path, dest);
+		return -1;
+	}
+}
diff --git a/src/lib-storage/index/mbox/Makefile.am b/src/lib-storage/index/mbox/Makefile.am
new file mode 100644
index 0000000..4165a20
--- /dev/null
+++ b/src/lib-storage/index/mbox/Makefile.am
@@ -0,0 +1,39 @@
+noinst_LTLIBRARIES = libstorage_mbox.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_mbox_la_SOURCES = \
+	istream-raw-mbox.c \
+	mbox-file.c \
+	mbox-lock.c \
+	mbox-mail.c \
+	mbox-md5-apop3d.c \
+	mbox-md5-all.c \
+	mbox-save.c \
+	mbox-settings.c \
+	mbox-sync-list-index.c \
+	mbox-sync-parse.c \
+	mbox-sync-rewrite.c \
+	mbox-sync-update.c \
+	mbox-sync.c \
+	mbox-storage.c
+
+headers = \
+	istream-raw-mbox.h \
+	mbox-file.h \
+	mbox-lock.h \
+	mbox-md5.h \
+	mbox-settings.h \
+	mbox-storage.h \
+	mbox-sync-private.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/mbox/Makefile.in b/src/lib-storage/index/mbox/Makefile.in
new file mode 100644
index 0000000..924b48f
--- /dev/null
+++ b/src/lib-storage/index/mbox/Makefile.in
@@ -0,0 +1,884 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/mbox
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_mbox_la_LIBADD =
+am_libstorage_mbox_la_OBJECTS = istream-raw-mbox.lo mbox-file.lo \
+	mbox-lock.lo mbox-mail.lo mbox-md5-apop3d.lo mbox-md5-all.lo \
+	mbox-save.lo mbox-settings.lo mbox-sync-list-index.lo \
+	mbox-sync-parse.lo mbox-sync-rewrite.lo mbox-sync-update.lo \
+	mbox-sync.lo mbox-storage.lo
+libstorage_mbox_la_OBJECTS = $(am_libstorage_mbox_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/istream-raw-mbox.Plo \
+	./$(DEPDIR)/mbox-file.Plo ./$(DEPDIR)/mbox-lock.Plo \
+	./$(DEPDIR)/mbox-mail.Plo ./$(DEPDIR)/mbox-md5-all.Plo \
+	./$(DEPDIR)/mbox-md5-apop3d.Plo ./$(DEPDIR)/mbox-save.Plo \
+	./$(DEPDIR)/mbox-settings.Plo ./$(DEPDIR)/mbox-storage.Plo \
+	./$(DEPDIR)/mbox-sync-list-index.Plo \
+	./$(DEPDIR)/mbox-sync-parse.Plo \
+	./$(DEPDIR)/mbox-sync-rewrite.Plo \
+	./$(DEPDIR)/mbox-sync-update.Plo ./$(DEPDIR)/mbox-sync.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_mbox_la_SOURCES)
+DIST_SOURCES = $(libstorage_mbox_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_mbox.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_mbox_la_SOURCES = \
+	istream-raw-mbox.c \
+	mbox-file.c \
+	mbox-lock.c \
+	mbox-mail.c \
+	mbox-md5-apop3d.c \
+	mbox-md5-all.c \
+	mbox-save.c \
+	mbox-settings.c \
+	mbox-sync-list-index.c \
+	mbox-sync-parse.c \
+	mbox-sync-rewrite.c \
+	mbox-sync-update.c \
+	mbox-sync.c \
+	mbox-storage.c
+
+headers = \
+	istream-raw-mbox.h \
+	mbox-file.h \
+	mbox-lock.h \
+	mbox-md5.h \
+	mbox-settings.h \
+	mbox-storage.h \
+	mbox-sync-private.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/mbox/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/mbox/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_mbox.la: $(libstorage_mbox_la_OBJECTS) $(libstorage_mbox_la_DEPENDENCIES) $(EXTRA_libstorage_mbox_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_mbox_la_OBJECTS) $(libstorage_mbox_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-raw-mbox.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-file.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-lock.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-md5-all.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-md5-apop3d.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-save.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-settings.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-list-index.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-parse.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-rewrite.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync-update.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mbox-sync.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/istream-raw-mbox.Plo
+	-rm -f ./$(DEPDIR)/mbox-file.Plo
+	-rm -f ./$(DEPDIR)/mbox-lock.Plo
+	-rm -f ./$(DEPDIR)/mbox-mail.Plo
+	-rm -f ./$(DEPDIR)/mbox-md5-all.Plo
+	-rm -f ./$(DEPDIR)/mbox-md5-apop3d.Plo
+	-rm -f ./$(DEPDIR)/mbox-save.Plo
+	-rm -f ./$(DEPDIR)/mbox-settings.Plo
+	-rm -f ./$(DEPDIR)/mbox-storage.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-list-index.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-parse.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-rewrite.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-update.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/istream-raw-mbox.Plo
+	-rm -f ./$(DEPDIR)/mbox-file.Plo
+	-rm -f ./$(DEPDIR)/mbox-lock.Plo
+	-rm -f ./$(DEPDIR)/mbox-mail.Plo
+	-rm -f ./$(DEPDIR)/mbox-md5-all.Plo
+	-rm -f ./$(DEPDIR)/mbox-md5-apop3d.Plo
+	-rm -f ./$(DEPDIR)/mbox-save.Plo
+	-rm -f ./$(DEPDIR)/mbox-settings.Plo
+	-rm -f ./$(DEPDIR)/mbox-storage.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-list-index.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-parse.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-rewrite.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync-update.Plo
+	-rm -f ./$(DEPDIR)/mbox-sync.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.c b/src/lib-storage/index/mbox/istream-raw-mbox.c
new file mode 100644
index 0000000..e91e581
--- /dev/null
+++ b/src/lib-storage/index/mbox/istream-raw-mbox.c
@@ -0,0 +1,821 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "istream-private.h"
+#include "istream-raw-mbox.h"
+#include "mbox-from.h"
+
+struct raw_mbox_istream {
+	struct istream_private istream;
+
+	time_t received_time, next_received_time;
+	char *sender, *next_sender;
+
+	uoff_t from_offset, hdr_offset, body_offset, mail_size;
+	uoff_t input_peak_offset;
+
+	bool locked:1;
+	bool seeked:1;
+	bool crlf_ending:1;
+	bool corrupted:1;
+	bool mail_size_forced:1;
+	bool eof:1;
+	bool header_missing_eoh:1;
+};
+
+static void mbox_istream_log_read_error(struct raw_mbox_istream *rstream)
+{
+	if (rstream->istream.parent->stream_errno != 0) {
+		/* Log e.g. compression istream error */
+		i_error("Failed to read mbox file %s: %s",
+			i_stream_get_name(&rstream->istream.istream),
+			i_stream_get_error(rstream->istream.parent));
+	}
+}
+
+static void i_stream_raw_mbox_destroy(struct iostream_private *stream)
+{
+	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
+
+	i_free(rstream->sender);
+	i_free(rstream->next_sender);
+
+	i_stream_seek(rstream->istream.parent,
+		      rstream->istream.istream.v_offset);
+}
+
+static int mbox_read_from_line(struct raw_mbox_istream *rstream)
+{
+	const unsigned char *buf, *p;
+	char *sender;
+	time_t received_time;
+	size_t pos, line_pos;
+	ssize_t ret;
+	unsigned int skip;
+	int tz;
+
+	buf = i_stream_get_data(rstream->istream.parent, &pos);
+	i_assert(pos > 0);
+
+	/* from_offset points to "\nFrom ", so unless we're at the beginning
+	   of the file, skip the initial \n */
+	if (rstream->from_offset == 0)
+		skip = 0;
+	else {
+		skip = 1;
+		if (*buf == '\r')
+			skip++;
+	}
+
+	while ((p = memchr(buf+skip, '\n', pos-skip)) == NULL) {
+		ret = i_stream_read_memarea(rstream->istream.parent);
+		buf = i_stream_get_data(rstream->istream.parent, &pos);
+		if (ret < 0) {
+			if (ret == -2) {
+				/* From_-line is too long, but we should be
+				   able to parse what we have so far. */
+				break;
+			}
+			/* EOF shouldn't happen */
+			rstream->istream.istream.eof =
+				rstream->istream.parent->eof;
+			rstream->istream.istream.stream_errno =
+				rstream->istream.parent->stream_errno;
+			return -1;
+		}
+		i_assert(pos > 0);
+	}
+	line_pos = p == NULL ? 0 : (size_t)(p - buf);
+
+	/* beginning of mbox */
+	if (memcmp(buf+skip, "From ", 5) != 0 ||
+	    mbox_from_parse((buf+skip)+5, (pos-skip)-5,
+			    &received_time, &tz, &sender) < 0) {
+		/* broken From - should happen only at beginning of
+		   file if this isn't a mbox.. */
+		io_stream_set_error(&rstream->istream.iostream,
+			"mbox file doesn't begin with 'From ' line");
+		rstream->istream.istream.stream_errno = EINVAL;
+		return -1;
+	}
+
+	if (rstream->istream.istream.v_offset == rstream->from_offset) {
+		rstream->received_time = received_time;
+		i_free(rstream->sender);
+		rstream->sender = sender;
+	} else {
+		rstream->next_received_time = received_time;
+		i_free(rstream->next_sender);
+		rstream->next_sender = sender;
+	}
+
+	/* skip over From-line */
+	if (line_pos == 0) {
+		/* line was too long. skip the input until we find LF. */
+		rstream->istream.istream.v_offset += pos;
+		i_stream_skip(rstream->istream.parent, pos);
+
+		while ((ret = i_stream_read_memarea(rstream->istream.parent)) > 0) {
+			p = memchr(buf, '\n', pos);
+			if (p != NULL)
+				break;
+			rstream->istream.istream.v_offset += pos;
+			i_stream_skip(rstream->istream.parent, pos);
+		}
+		if (ret <= 0) {
+			i_assert(ret == -1);
+			/* EOF shouldn't happen */
+			rstream->istream.istream.eof =
+				rstream->istream.parent->eof;
+			rstream->istream.istream.stream_errno =
+				rstream->istream.parent->stream_errno;
+			return -1;
+		}
+		line_pos = (size_t)(p - buf);
+	}
+	rstream->istream.istream.v_offset += line_pos+1;
+	i_stream_skip(rstream->istream.parent, line_pos+1);
+
+	rstream->hdr_offset = rstream->istream.istream.v_offset;
+	return 0;
+}
+
+static void handle_end_of_mail(struct raw_mbox_istream *rstream, size_t pos)
+{
+	rstream->mail_size = rstream->istream.istream.v_offset + pos -
+		rstream->hdr_offset;
+
+	if (rstream->hdr_offset + rstream->mail_size < rstream->body_offset) {
+		uoff_t new_body_offset =
+			rstream->hdr_offset + rstream->mail_size;
+
+		if (rstream->body_offset != UOFF_T_MAX) {
+			/* Header didn't have ending \n */
+			rstream->header_missing_eoh = TRUE;
+		} else {
+			/* "headers\n\nFrom ..", the second \n belongs to next
+			   message which we didn't know at the time yet. */
+		}
+
+		/* The +2 check is for CR+LF linefeeds */
+		i_assert(rstream->body_offset == UOFF_T_MAX ||
+			 rstream->body_offset == new_body_offset + 1 ||
+			 rstream->body_offset == new_body_offset + 2);
+		rstream->body_offset = new_body_offset;
+	}
+}
+
+static ssize_t i_stream_raw_mbox_read(struct istream_private *stream)
+{
+	static const char *mbox_from = "\nFrom ";
+	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
+	const unsigned char *buf;
+	const char *fromp;
+	char *sender;
+	time_t received_time;
+	size_t i, pos, new_pos, from_start_pos, from_after_pos;
+	ssize_t ret = 0;
+	int eoh_char, tz;
+	bool crlf_ending = FALSE;
+
+	i_assert(rstream->seeked);
+	i_assert(stream->istream.v_offset >= rstream->from_offset);
+
+	if (stream->istream.eof)
+		return -1;
+	if (rstream->corrupted) {
+		rstream->istream.istream.stream_errno = EINVAL;
+		return -1;
+	}
+
+	i_stream_seek(stream->parent, stream->istream.v_offset);
+
+	stream->pos -= stream->skip;
+	stream->skip = 0;
+	stream->buffer = NULL;
+
+	do {
+		buf = i_stream_get_data(stream->parent, &pos);
+		if (pos > 1 && stream->istream.v_offset + pos >
+		    rstream->input_peak_offset) {
+			/* fake our read count. needed because if in the end
+			   we have only one character in buffer and we skip it
+			   (as potential CR), we want to get back to this
+			   i_stream_raw_mbox_read() to read more data. */
+			ret = pos;
+			break;
+		}
+		ret = i_stream_read_memarea(stream->parent);
+	} while (ret > 0);
+	stream->istream.stream_errno = stream->parent->stream_errno;
+
+	if (ret < 0) {
+		if (ret == -1)
+			mbox_istream_log_read_error(rstream);
+		if (ret == -2) {
+			if (stream->skip == stream->pos) {
+				/* From_-line is longer than our input buffer.
+				   finish the check without seeing the LF. */
+			} else if (stream->istream.v_offset + pos ==
+				   rstream->input_peak_offset) {
+				/* we've read everything our parent stream
+				   has to offer. */
+				stream->buffer = buf;
+				return -2;
+			}
+			/* parent stream is full, but we haven't returned
+			   all its bytes to our caller yet. */
+		} else if (stream->istream.v_offset != 0 || pos == 0) {
+			/* we've read the whole file, final byte should be
+			   the \n trailer */
+			if (pos > 0 && buf[pos-1] == '\n') {
+				pos--;
+				if (pos > 0 && buf[pos-1] == '\r') {
+					crlf_ending = TRUE;
+					pos--;
+				}
+			}
+
+			i_assert(pos >= stream->pos);
+			ret = pos == stream->pos ? -1 :
+				(ssize_t)(pos - stream->pos);
+
+			stream->buffer = buf;
+			stream->pos = pos;
+
+			if (stream->istream.v_offset == rstream->from_offset) {
+				/* haven't seen From-line yet, so this mbox
+				   stream is now at EOF */
+				rstream->eof = TRUE;
+			}
+			stream->istream.eof = TRUE;
+			rstream->crlf_ending = crlf_ending;
+			handle_end_of_mail(rstream, pos);
+			return ret < 0 ? i_stream_raw_mbox_read(stream) : ret;
+		}
+	}
+
+	if (stream->istream.v_offset == rstream->from_offset) {
+		/* beginning of message, we haven't yet read our From-line */
+		if (pos == 2 && ret > 0) {
+			/* we're at the end of file with CR+LF linefeeds?
+			   need more data to verify it. */
+			rstream->input_peak_offset =
+				stream->istream.v_offset + pos;
+			return i_stream_raw_mbox_read(stream);
+		}
+		if (mbox_read_from_line(rstream) < 0) {
+			io_stream_set_error(&stream->iostream,
+				"Next message unexpectedly corrupted in mbox file "
+				"%s at %"PRIuUOFF_T,
+				i_stream_get_name(&stream->istream),
+				stream->istream.v_offset);
+			if (stream->istream.v_offset != 0)
+				i_error("%s", stream->iostream.error);
+			stream->pos = 0;
+			rstream->eof = TRUE;
+			rstream->corrupted = TRUE;
+			return -1;
+		}
+
+		/* got it. we don't want to return it however,
+		   so start again from headers */
+		buf = i_stream_get_data(stream->parent, &pos);
+		if (pos == 0)
+			return i_stream_raw_mbox_read(stream);
+	}
+
+	/* See if we have From-line here - note that it works right only
+	   because all characters are different in mbox_from. */
+        fromp = mbox_from; from_start_pos = from_after_pos = SIZE_MAX;
+	eoh_char = rstream->body_offset == UOFF_T_MAX ? '\n' : -1;
+	for (i = stream->pos; i < pos; i++) {
+		if (buf[i] == eoh_char &&
+		    ((i > 0 && buf[i-1] == '\n') ||
+                     (i > 1 && buf[i-1] == '\r' && buf[i-2] == '\n') ||
+		     stream->istream.v_offset + i == rstream->hdr_offset)) {
+			rstream->body_offset = stream->istream.v_offset + i + 1;
+			eoh_char = -1;
+		}
+		if ((char)buf[i] == *fromp) {
+			if (*++fromp == '\0') {
+				/* potential From-line, see if we have the
+				   rest of the line buffered. */
+				i++;
+				if (i >= 7 && buf[i-7] == '\r') {
+					/* CR also belongs to it. */
+					crlf_ending = TRUE;
+					from_start_pos = i - 7;
+				} else {
+					crlf_ending = FALSE;
+					from_start_pos = i - 6;
+				}
+
+				if (rstream->mail_size == UOFF_T_MAX ||
+				    rstream->hdr_offset + rstream->mail_size ==
+				    stream->istream.v_offset + from_start_pos) {
+					from_after_pos = i;
+					if (ret == -2) {
+						/* even if we don't have the
+						   whole line, we need to
+						   finish this check now. */
+						goto mbox_verify;
+					}
+				}
+				fromp = mbox_from;
+			} else if (from_after_pos != SIZE_MAX) {
+				/* we have the whole From-line here now.
+				   See if it's a valid one. */
+			mbox_verify:
+				if (mbox_from_parse(buf + from_after_pos,
+						    pos - from_after_pos,
+						    &received_time, &tz,
+						    &sender) == 0) {
+					/* yep, we stop here. */
+					rstream->next_received_time =
+						received_time;
+					i_free(rstream->next_sender);
+					rstream->next_sender = sender;
+					stream->istream.eof = TRUE;
+
+					rstream->crlf_ending = crlf_ending;
+					handle_end_of_mail(rstream,
+							   from_start_pos);
+					break;
+				}
+				from_after_pos = SIZE_MAX;
+			}
+		} else {
+			fromp = mbox_from;
+			if ((char)buf[i] == *fromp)
+				fromp++;
+		}
+	}
+
+	/* we want to go at least one byte further next time */
+	rstream->input_peak_offset = stream->istream.v_offset + i;
+
+	if (from_after_pos != SIZE_MAX) {
+		/* we're waiting for the \n at the end of From-line */
+		new_pos = from_start_pos;
+	} else {
+		/* leave out the beginnings of potential From-line + CR */
+		new_pos = i - (fromp - mbox_from);
+		if (new_pos > 0)
+			new_pos--;
+	}
+
+	if (stream->istream.v_offset -
+	    rstream->hdr_offset + new_pos > rstream->mail_size) {
+		/* istream_raw_mbox_set_next_offset() used invalid
+		   cached next_offset? */
+		io_stream_set_error(&stream->iostream,
+			"Next message unexpectedly lost from mbox file "
+			"%s at %"PRIuUOFF_T" (%s)",
+			i_stream_get_name(&stream->istream),
+			rstream->hdr_offset + rstream->mail_size,
+			rstream->mail_size_forced ? "cached" : "noncached");
+		i_error("%s", stream->iostream.error);
+		rstream->eof = TRUE;
+		rstream->corrupted = TRUE;
+		rstream->istream.istream.stream_errno = EINVAL;
+		stream->pos = 0;
+		return -1;
+	}
+
+	stream->buffer = buf;
+	if (new_pos == stream->pos) {
+		if (stream->istream.eof || ret > 0)
+			return i_stream_raw_mbox_read(stream);
+		i_assert(new_pos > 0);
+		ret = -2;
+	} else {
+		i_assert(new_pos > stream->pos);
+		ret = new_pos - stream->pos;
+		stream->pos = new_pos;
+	}
+	return ret;
+}
+
+static void i_stream_raw_mbox_seek(struct istream_private *stream,
+				   uoff_t v_offset, bool mark ATTR_UNUSED)
+{
+	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
+
+	stream->istream.v_offset = v_offset;
+	stream->skip = stream->pos = 0;
+	stream->buffer = NULL;
+
+        rstream->input_peak_offset = 0;
+	rstream->eof = FALSE;
+}
+
+static void i_stream_raw_mbox_sync(struct istream_private *stream)
+{
+	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
+
+	i_stream_sync(stream->parent);
+
+	rstream->istream.skip = 0;
+	rstream->istream.pos = 0;
+	rstream->input_peak_offset = 0;
+}
+
+static int
+i_stream_raw_mbox_stat(struct istream_private *stream, bool exact)
+{
+	const struct stat *st;
+	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
+
+	if (i_stream_stat(stream->parent, exact, &st) < 0) {
+		stream->istream.stream_errno = stream->parent->stream_errno;
+		return -1;
+	}
+
+	stream->statbuf = *st;
+	stream->statbuf.st_size =
+		!exact && rstream->seeked && rstream->mail_size != UOFF_T_MAX ?
+		(off_t)rstream->mail_size : -1;
+	return 0;
+}
+
+struct istream *i_stream_create_raw_mbox(struct istream *input)
+{
+	struct raw_mbox_istream *rstream;
+
+	i_assert(input->v_offset == 0);
+
+	rstream = i_new(struct raw_mbox_istream, 1);
+
+	rstream->body_offset = UOFF_T_MAX;
+	rstream->mail_size = UOFF_T_MAX;
+	rstream->received_time = (time_t)-1;
+	rstream->next_received_time = (time_t)-1;
+
+	rstream->istream.iostream.destroy = i_stream_raw_mbox_destroy;
+	rstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+	rstream->istream.read = i_stream_raw_mbox_read;
+	rstream->istream.seek = i_stream_raw_mbox_seek;
+	rstream->istream.sync = i_stream_raw_mbox_sync;
+	rstream->istream.stat = i_stream_raw_mbox_stat;
+
+	rstream->istream.istream.readable_fd = input->readable_fd;
+	rstream->istream.istream.blocking = input->blocking;
+	rstream->istream.istream.seekable = input->seekable;
+
+	return i_stream_create(&rstream->istream, input, -1, 0);
+}
+
+static int istream_raw_mbox_is_valid_from(struct raw_mbox_istream *rstream)
+{
+	const unsigned char *data;
+	size_t size = 0;
+	time_t received_time;
+	char *sender;
+	int tz;
+	ssize_t ret = 0;
+
+	/* minimal: "From x Thu Nov 29 22:33:52 2001" = 31 chars */
+	do {
+		data = i_stream_get_data(rstream->istream.parent, &size);
+		if (size >= 31)
+			break;
+	} while ((ret = i_stream_read_memarea(rstream->istream.parent)) > 0);
+	if (ret == -1)
+		mbox_istream_log_read_error(rstream);
+
+	if ((size == 1 && data[0] == '\n') ||
+	    (size == 2 && data[0] == '\r' && data[1] == '\n')) {
+		/* EOF */
+		return 1;
+	}
+
+	if (size > 31 && memcmp(data, "\nFrom ", 6) == 0) {
+		data += 6;
+		size -= 6;
+	} else if (size > 32 && memcmp(data, "\r\nFrom ", 7) == 0) {
+		data += 7;
+		size -= 7;
+	} else {
+		return 0;
+	}
+
+	while (memchr(data, '\n', size) == NULL) {
+		ret = i_stream_read_bytes(rstream->istream.parent,
+					  &data, &size, size+1);
+		if (ret < 0) {
+			if (ret == -1)
+				mbox_istream_log_read_error(rstream);
+			break;
+		}
+	}
+
+	if (mbox_from_parse(data, size, &received_time, &tz, &sender) < 0)
+		return 0;
+
+	rstream->next_received_time = received_time;
+	i_free(rstream->next_sender);
+	rstream->next_sender = sender;
+	return 1;
+}
+
+uoff_t istream_raw_mbox_get_start_offset(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	i_assert(rstream->seeked);
+
+	return rstream->from_offset;
+}
+
+int istream_raw_mbox_get_header_offset(struct istream *stream,
+				       uoff_t *hdr_offset_r)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	i_assert(rstream->seeked);
+
+	if (rstream->hdr_offset == rstream->from_offset)
+		(void)i_stream_read(stream);
+
+	if (rstream->corrupted) {
+		i_error("Unexpectedly lost From-line from mbox file %s at "
+			"%"PRIuUOFF_T, i_stream_get_name(stream),
+			rstream->from_offset);
+		return -1;
+	}
+	if (stream->stream_errno != 0)
+		return -1;
+
+	*hdr_offset_r = rstream->hdr_offset;
+	return 0;
+}
+
+int istream_raw_mbox_get_body_offset(struct istream *stream,
+				     uoff_t *body_offset_r)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+	uoff_t offset;
+
+	i_assert(rstream->seeked);
+
+	if (rstream->body_offset != UOFF_T_MAX) {
+		*body_offset_r = rstream->body_offset;
+		return 0;
+	}
+
+	offset = stream->v_offset;
+	i_stream_seek(stream, rstream->hdr_offset);
+	while (rstream->body_offset == UOFF_T_MAX) {
+		i_stream_skip(stream, i_stream_get_data_size(stream));
+
+		if (i_stream_read(stream) < 0) {
+			if (rstream->corrupted) {
+				i_error("Unexpectedly lost From-line from mbox file "
+					"%s at %"PRIuUOFF_T,
+					i_stream_get_name(stream),
+					rstream->from_offset);
+			} else {
+				i_assert(rstream->body_offset != UOFF_T_MAX);
+			}
+			return -1;
+		}
+	}
+
+	i_stream_seek(stream, offset);
+	*body_offset_r = rstream->body_offset;
+	return 0;
+}
+
+int istream_raw_mbox_get_body_size(struct istream *stream,
+				   uoff_t expected_body_size,
+				   uoff_t *body_size_r)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+	const unsigned char *data;
+	size_t size;
+	uoff_t old_offset, body_offset, body_size, next_body_offset;
+
+	i_assert(rstream->seeked);
+	i_assert(rstream->hdr_offset != UOFF_T_MAX);
+
+	if (istream_raw_mbox_get_body_offset(stream, &body_offset) < 0)
+		return -1;
+	body_size = rstream->mail_size == UOFF_T_MAX ? UOFF_T_MAX :
+		rstream->mail_size - (rstream->body_offset -
+				      rstream->hdr_offset);
+	old_offset = stream->v_offset;
+	if (expected_body_size != UOFF_T_MAX) {
+		/* if we already have the existing body size, use it as long as
+		   it's >= expected body_size. otherwise the previous parsing
+		   may have stopped at a From_-line that belongs to the body. */
+		if (body_size != UOFF_T_MAX && body_size >= expected_body_size) {
+			*body_size_r = body_size;
+			return 0;
+		}
+
+		next_body_offset = rstream->body_offset + expected_body_size;
+		/* If header_missing_eoh is set, the message body begins with
+		   a From_-line and the body_offset is pointing to the line
+		   *before* the first line of the body, i.e. the empty line
+		   separating the headers from the body. If that is the case,
+		   we'll have to skip over the empty line to get the correct
+		   next_body_offset. */
+		if (rstream->header_missing_eoh) {
+			i_assert(body_size == 0);
+			next_body_offset += rstream->crlf_ending ? 2 : 1;
+		}
+
+		i_stream_seek(rstream->istream.parent, next_body_offset);
+		if (istream_raw_mbox_is_valid_from(rstream) > 0) {
+			rstream->mail_size =
+				next_body_offset - rstream->hdr_offset;
+			i_stream_seek(stream, old_offset);
+			*body_size_r = expected_body_size;
+			return 0;
+		}
+		/* invalid expected_body_size */
+	}
+	if (body_size != UOFF_T_MAX) {
+		*body_size_r = body_size;
+		return 0;
+	}
+
+	/* have to read through the message body */
+	while (i_stream_read_more(stream, &data, &size) > 0)
+		i_stream_skip(stream, size);
+	i_stream_seek(stream, old_offset);
+	if (stream->stream_errno != 0)
+		return -1;
+
+	i_assert(rstream->mail_size != UOFF_T_MAX);
+	*body_size_r = rstream->mail_size -
+		(rstream->body_offset - rstream->hdr_offset);
+	return 0;
+}
+
+time_t istream_raw_mbox_get_received_time(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	i_assert(rstream->seeked);
+
+	if (rstream->received_time == (time_t)-1)
+		(void)i_stream_read(stream);
+	return rstream->received_time;
+}
+
+const char *istream_raw_mbox_get_sender(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	i_assert(rstream->seeked);
+
+	if (rstream->sender == NULL)
+		(void)i_stream_read(stream);
+	return rstream->sender == NULL ? "" : rstream->sender;
+}
+
+bool istream_raw_mbox_has_crlf_ending(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	i_assert(rstream->seeked);
+
+	return rstream->crlf_ending;
+}
+
+int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+	uoff_t body_size;
+
+	if (istream_raw_mbox_get_body_size(stream, expected_body_size,
+					   &body_size) < 0)
+		return -1;
+	rstream->mail_size = UOFF_T_MAX;
+
+	rstream->received_time = rstream->next_received_time;
+	rstream->next_received_time = (time_t)-1;
+
+	i_free(rstream->sender);
+	rstream->sender = rstream->next_sender;
+	rstream->next_sender = NULL;
+
+	rstream->from_offset = rstream->body_offset + body_size;
+	rstream->hdr_offset = rstream->from_offset;
+	rstream->body_offset = UOFF_T_MAX;
+	rstream->header_missing_eoh = FALSE;
+
+	if (stream->v_offset != rstream->from_offset)
+		i_stream_seek_mark(stream, rstream->from_offset);
+	i_stream_seek_mark(rstream->istream.parent, rstream->from_offset);
+
+	rstream->eof = FALSE;
+	rstream->istream.istream.eof = FALSE;
+	return 0;
+}
+
+int istream_raw_mbox_seek(struct istream *stream, uoff_t offset)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+	bool check;
+
+	i_assert(rstream->locked);
+
+	/* reset any (corruption) errors */
+	stream->stream_errno = 0;
+	i_free_and_null(stream->real_stream->iostream.error);
+	rstream->corrupted = FALSE;
+	rstream->eof = FALSE;
+	rstream->istream.istream.eof = FALSE;
+
+	/* if seeked is FALSE, we unlocked in the middle. don't try to use
+	   any cached state then. */
+	if (rstream->mail_size != UOFF_T_MAX && rstream->seeked &&
+	    rstream->hdr_offset + rstream->mail_size == offset)
+		return istream_raw_mbox_next(stream, UOFF_T_MAX);
+
+	if (offset == rstream->from_offset && rstream->seeked) {
+		/* back to beginning of current message */
+		offset = rstream->hdr_offset;
+		check = offset == 0;
+	} else {
+		rstream->body_offset = UOFF_T_MAX;
+		rstream->mail_size = UOFF_T_MAX;
+		rstream->received_time = (time_t)-1;
+		rstream->next_received_time = (time_t)-1;
+		rstream->header_missing_eoh = FALSE;
+
+		i_free(rstream->sender);
+		rstream->sender = NULL;
+		i_free(rstream->next_sender);
+		rstream->next_sender = NULL;
+
+                rstream->from_offset = offset;
+		rstream->hdr_offset = offset;
+		check = TRUE;
+	}
+	rstream->seeked = TRUE;
+
+	i_stream_seek_mark(stream, offset);
+	i_stream_seek_mark(rstream->istream.parent, offset);
+
+	if (check)
+		(void)i_stream_read(stream);
+	return rstream->corrupted ? -1 : 0;
+}
+
+void istream_raw_mbox_set_next_offset(struct istream *stream, uoff_t offset)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	i_assert(rstream->hdr_offset != UOFF_T_MAX);
+
+	rstream->mail_size_forced = TRUE;
+	rstream->mail_size = offset - rstream->hdr_offset;
+}
+
+bool istream_raw_mbox_is_eof(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	return rstream->eof;
+}
+
+bool istream_raw_mbox_is_corrupted(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	return rstream->corrupted;
+}
+
+void istream_raw_mbox_set_locked(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	rstream->locked = TRUE;
+}
+
+void istream_raw_mbox_set_unlocked(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	rstream->locked = FALSE;
+	rstream->seeked = FALSE;
+}
diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.h b/src/lib-storage/index/mbox/istream-raw-mbox.h
new file mode 100644
index 0000000..4543841
--- /dev/null
+++ b/src/lib-storage/index/mbox/istream-raw-mbox.h
@@ -0,0 +1,56 @@
+#ifndef ISTREAM_RAW_MBOX_H
+#define ISTREAM_RAW_MBOX_H
+
+/* Create a mbox stream for parsing mbox. Reading stops before From-line,
+   you'll have to call istream_raw_mbox_next() to get to next message.
+   path is used only for logging purposes. */
+struct istream *i_stream_create_raw_mbox(struct istream *input);
+
+/* Return offset to beginning of the "\nFrom"-line. */
+uoff_t istream_raw_mbox_get_start_offset(struct istream *stream);
+/* Return offset to beginning of the headers. */
+int istream_raw_mbox_get_header_offset(struct istream *stream,
+				       uoff_t *hdr_offset_r);
+/* Return offset to beginning of the body. */
+int istream_raw_mbox_get_body_offset(struct istream *stream,
+				     uoff_t *body_offset_r);
+
+/* Return the number of bytes in the body of this message. If
+   expected_body_size isn't UOFF_T_MAX, we'll use it as potentially valid body
+   size to avoid actually reading through the whole message. */
+int istream_raw_mbox_get_body_size(struct istream *stream,
+				   uoff_t expected_body_size,
+				   uoff_t *body_size_r);
+
+/* Return received time of current message, or (time_t)-1 if the timestamp is
+   broken. */
+time_t istream_raw_mbox_get_received_time(struct istream *stream);
+
+/* Return sender of current message. */
+const char *istream_raw_mbox_get_sender(struct istream *stream);
+/* Return TRUE if the empty line between this and the next mail contains CR. */
+bool istream_raw_mbox_has_crlf_ending(struct istream *stream);
+
+/* Jump to next message. If expected_body_size isn't UOFF_T_MAX, we'll use it
+   as potentially valid body size. */
+int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size);
+
+/* Seek to message at given offset. offset must point to beginning of
+   "\nFrom ", or 0 for beginning of file. Returns -1 if it offset doesn't
+   contain a valid From-line. */
+int istream_raw_mbox_seek(struct istream *stream, uoff_t offset);
+/* Set next message's start offset. If this isn't set, read stops at the next
+   valid From_-line, even if it belongs to the current message's body
+   (Content-Length: header can be used to determine that). */
+void istream_raw_mbox_set_next_offset(struct istream *stream, uoff_t offset);
+
+/* Returns TRUE if we've read the whole mbox. */
+bool istream_raw_mbox_is_eof(struct istream *stream);
+/* Returns TRUE if we've noticed corruption in used offsets/sizes. */
+bool istream_raw_mbox_is_corrupted(struct istream *stream);
+/* Change stream's locking state. We'll assert-crash if stream is tried to be
+   read while it's unlocked. */
+void istream_raw_mbox_set_locked(struct istream *stream);
+void istream_raw_mbox_set_unlocked(struct istream *stream);
+
+#endif
diff --git a/src/lib-storage/index/mbox/mbox-file.c b/src/lib-storage/index/mbox/mbox-file.c
new file mode 100644
index 0000000..b71abb8
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-file.c
@@ -0,0 +1,207 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "mbox-storage.h"
+#include "mbox-sync-private.h"
+#include "mbox-file.h"
+#include "istream-raw-mbox.h"
+
+#include <sys/stat.h>
+#include <utime.h>
+
+#define MBOX_READ_BLOCK_SIZE IO_BLOCK_SIZE
+
+int mbox_file_open(struct mbox_mailbox *mbox)
+{
+	struct stat st;
+	int fd;
+
+	i_assert(mbox->mbox_fd == -1);
+
+	if (mbox->mbox_file_stream != NULL) {
+		/* read-only mbox stream */
+		i_assert(mbox_is_backend_readonly(mbox));
+		return 0;
+	}
+
+	fd = open(mailbox_get_path(&mbox->box),
+		  mbox_is_backend_readonly(mbox) ? O_RDONLY : O_RDWR);
+	if (fd == -1 && errno == EACCES && !mbox->backend_readonly) {
+		mbox->backend_readonly = TRUE;
+		fd = open(mailbox_get_path(&mbox->box), O_RDONLY);
+	}
+
+	if (fd == -1) {
+		mbox_set_syscall_error(mbox, "open()");
+		return -1;
+	}
+
+	if (fstat(fd, &st) < 0) {
+		mbox_set_syscall_error(mbox, "fstat()");
+		i_close_fd(&fd);
+		return -1;
+	}
+
+	mbox->mbox_writeonly = S_ISFIFO(st.st_mode);
+	mbox->mbox_fd = fd;
+	mbox->mbox_dev = st.st_dev;
+	mbox->mbox_ino = st.st_ino;
+	return 0;
+}
+
+void mbox_file_close(struct mbox_mailbox *mbox)
+{
+	mbox_file_close_stream(mbox);
+
+	if (mbox->mbox_fd != -1) {
+		if (close(mbox->mbox_fd) < 0)
+			mbox_set_syscall_error(mbox, "close()");
+		mbox->mbox_fd = -1;
+	}
+}
+
+int mbox_file_open_stream(struct mbox_mailbox *mbox)
+{
+	if (mbox->mbox_stream != NULL)
+		return 0;
+
+	if (mbox->mbox_file_stream != NULL) {
+		/* read-only mbox stream */
+		i_assert(mbox->mbox_fd == -1 && mbox_is_backend_readonly(mbox));
+	} else {
+		if (mbox->mbox_fd == -1) {
+			if (mbox_file_open(mbox) < 0)
+				return -1;
+		}
+
+		if (mbox->mbox_writeonly) {
+			mbox->mbox_file_stream =
+				i_stream_create_from_data("", 0);
+		} else {
+			mbox->mbox_file_stream =
+				i_stream_create_fd(mbox->mbox_fd,
+						   MBOX_READ_BLOCK_SIZE);
+			i_stream_set_init_buffer_size(mbox->mbox_file_stream,
+						      MBOX_READ_BLOCK_SIZE);
+		}
+		i_stream_set_name(mbox->mbox_file_stream,
+				  mailbox_get_path(&mbox->box));
+	}
+
+	mbox->mbox_stream = i_stream_create_raw_mbox(mbox->mbox_file_stream);
+	if (mbox->mbox_lock_type != F_UNLCK)
+		istream_raw_mbox_set_locked(mbox->mbox_stream);
+	return 0;
+}
+
+static void mbox_file_fix_atime(struct mbox_mailbox *mbox)
+{
+	struct utimbuf buf;
+	struct stat st;
+
+	if (mbox->box.recent_flags_count > 0 &&
+	    (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0 &&
+	    mbox->mbox_fd != -1 && !mbox_is_backend_readonly(mbox)) {
+		/* we've seen recent messages which we want to keep recent.
+		   keep file's atime lower than mtime so \Marked status
+		   gets shown while listing */
+		if (fstat(mbox->mbox_fd, &st) < 0) {
+			mbox_set_syscall_error(mbox, "fstat()");
+			return;
+		}
+		if (st.st_atime >= st.st_mtime) {
+			buf.modtime = st.st_mtime;
+			buf.actime = buf.modtime - 1;
+			/* EPERM can happen with shared mailboxes */
+			if (utime(mailbox_get_path(&mbox->box), &buf) < 0 &&
+			    errno != EPERM)
+				mbox_set_syscall_error(mbox, "utime()");
+		}
+	}
+}
+void mbox_file_close_stream(struct mbox_mailbox *mbox)
+{
+	/* if we read anything, fix the atime if needed */
+	mbox_file_fix_atime(mbox);
+
+	i_stream_destroy(&mbox->mbox_stream);
+
+	if (mbox->mbox_file_stream != NULL) {
+		if (mbox->mbox_fd == -1) {
+			/* read-only mbox stream */
+			i_assert(mbox_is_backend_readonly(mbox));
+			i_stream_seek(mbox->mbox_file_stream, 0);
+		} else {
+			i_stream_destroy(&mbox->mbox_file_stream);
+		}
+	}
+}
+
+int mbox_file_lookup_offset(struct mbox_mailbox *mbox,
+			    struct mail_index_view *view,
+			    uint32_t seq, uoff_t *offset_r)
+{
+	const void *data;
+	bool deleted;
+
+	mail_index_lookup_ext(view, seq, mbox->mbox_ext_idx, &data, &deleted);
+	if (deleted)
+		return -1;
+
+	if (data == NULL) {
+		mailbox_set_critical(&mbox->box,
+			"Cached message offset lost for seq %u in mbox", seq);
+		mbox->mbox_hdr.dirty_flag = 1;
+                mbox->mbox_broken_offsets = TRUE;
+		return 0;
+	}
+
+	*offset_r = *((const uint64_t *)data);
+	return 1;
+}
+
+int mbox_file_seek(struct mbox_mailbox *mbox, struct mail_index_view *view,
+		   uint32_t seq, bool *deleted_r)
+{
+	uoff_t offset;
+	int ret;
+
+	ret = mbox_file_lookup_offset(mbox, view, seq, &offset);
+	if (ret <= 0) {
+		*deleted_r = ret < 0;
+		return ret;
+	}
+	*deleted_r = FALSE;
+
+	if (istream_raw_mbox_seek(mbox->mbox_stream, offset) < 0) {
+		if (offset == 0) {
+			mbox->invalid_mbox_file = TRUE;
+			mail_storage_set_error(&mbox->storage->storage,
+				MAIL_ERROR_NOTPOSSIBLE,
+				"Mailbox isn't a valid mbox file");
+			return -1;
+		}
+
+		if (mbox->mbox_hdr.dirty_flag != 0)
+			return 0;
+
+		mailbox_set_critical(&mbox->box,
+			"Cached message offset %s is invalid for mbox",
+			dec2str(offset));
+		mbox->mbox_hdr.dirty_flag = 1;
+		mbox->mbox_broken_offsets = TRUE;
+		return 0;
+	}
+
+	if (mbox->mbox_hdr.dirty_flag != 0) {
+		/* we're dirty - make sure this is the correct mail */
+		if (!mbox_sync_parse_match_mail(mbox, view, seq))
+			return 0;
+
+		ret = istream_raw_mbox_seek(mbox->mbox_stream, offset);
+		i_assert(ret == 0);
+	}
+
+	return 1;
+}
diff --git a/src/lib-storage/index/mbox/mbox-file.h b/src/lib-storage/index/mbox/mbox-file.h
new file mode 100644
index 0000000..529a2e0
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-file.h
@@ -0,0 +1,16 @@
+#ifndef MBOX_FILE_H
+#define MBOX_FILE_H
+
+int mbox_file_open(struct mbox_mailbox *mbox);
+void mbox_file_close(struct mbox_mailbox *mbox);
+
+int mbox_file_open_stream(struct mbox_mailbox *mbox);
+void mbox_file_close_stream(struct mbox_mailbox *mbox);
+
+int mbox_file_lookup_offset(struct mbox_mailbox *mbox,
+			    struct mail_index_view *view,
+			    uint32_t seq, uoff_t *offset_r);
+int mbox_file_seek(struct mbox_mailbox *mbox, struct mail_index_view *view,
+		   uint32_t seq, bool *deleted_r);
+
+#endif
diff --git a/src/lib-storage/index/mbox/mbox-lock.c b/src/lib-storage/index/mbox/mbox-lock.c
new file mode 100644
index 0000000..cebff48
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-lock.c
@@ -0,0 +1,900 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "eacces-error.h"
+#include "restrict-access.h"
+#include "nfs-workarounds.h"
+#include "ipwd.h"
+#include "mail-index-private.h"
+#include "mbox-storage.h"
+#include "istream-raw-mbox.h"
+#include "mbox-file.h"
+#include "mbox-lock.h"
+
+#include <time.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+#ifdef HAVE_FLOCK
+#  include <sys/file.h>
+#endif
+
+/* 0.1 .. 0.2msec */
+#define LOCK_RANDOM_USLEEP_TIME (100000 + (unsigned int)i_rand() % 100000)
+
+enum mbox_lock_type {
+	MBOX_LOCK_DOTLOCK,
+	MBOX_LOCK_DOTLOCK_TRY,
+	MBOX_LOCK_FCNTL,
+	MBOX_LOCK_FLOCK,
+	MBOX_LOCK_LOCKF,
+
+	MBOX_LOCK_COUNT
+};
+
+enum mbox_dotlock_op {
+	MBOX_DOTLOCK_OP_LOCK,
+	MBOX_DOTLOCK_OP_UNLOCK,
+	MBOX_DOTLOCK_OP_TOUCH
+};
+
+struct mbox_lock_context {
+	struct mbox_mailbox *mbox;
+	bool locked_status[MBOX_LOCK_COUNT];
+	bool checked_file;
+
+	int lock_type;
+	bool dotlock_last_stale;
+	bool fcntl_locked;
+	bool using_privileges;
+};
+
+struct mbox_lock_data {
+	enum mbox_lock_type type;
+	const char *name;
+	int (*func)(struct mbox_lock_context *ctx, int lock_type,
+		    time_t max_wait_time);
+};
+
+static int mbox_lock_dotlock(struct mbox_lock_context *ctx, int lock_type,
+			     time_t max_wait_time);
+static int mbox_lock_dotlock_try(struct mbox_lock_context *ctx, int lock_type,
+				 time_t max_wait_time);
+static int mbox_lock_fcntl(struct mbox_lock_context *ctx, int lock_type,
+			   time_t max_wait_time);
+#ifdef HAVE_FLOCK
+static int mbox_lock_flock(struct mbox_lock_context *ctx, int lock_type,
+			   time_t max_wait_time);
+#else
+#  define mbox_lock_flock NULL
+#endif
+#ifdef HAVE_LOCKF
+static int mbox_lock_lockf(struct mbox_lock_context *ctx, int lock_type,
+			   time_t max_wait_time);
+#else
+#  define mbox_lock_lockf NULL
+#endif
+
+static struct mbox_lock_data lock_data[] = {
+	{ MBOX_LOCK_DOTLOCK, "dotlock", mbox_lock_dotlock },
+	{ MBOX_LOCK_DOTLOCK_TRY, "dotlock_try", mbox_lock_dotlock_try },
+	{ MBOX_LOCK_FCNTL, "fcntl", mbox_lock_fcntl },
+	{ MBOX_LOCK_FLOCK, "flock", mbox_lock_flock },
+	{ MBOX_LOCK_LOCKF, "lockf", mbox_lock_lockf },
+	{ 0, NULL, NULL }
+};
+
+static int ATTR_NOWARN_UNUSED_RESULT
+mbox_lock_list(struct mbox_lock_context *ctx, int lock_type,
+	       time_t max_wait_time, int idx);
+static int ATTR_NOWARN_UNUSED_RESULT
+mbox_unlock_files(struct mbox_lock_context *ctx);
+
+static void mbox_read_lock_methods(const char *str, const char *env,
+				   enum mbox_lock_type *locks)
+{
+        enum mbox_lock_type type;
+	const char *const *lock;
+	int i, dest;
+
+	for (lock = t_strsplit(str, " "), dest = 0; *lock != NULL; lock++) {
+		for (type = 0; lock_data[type].name != NULL; type++) {
+			if (strcasecmp(*lock, lock_data[type].name) == 0) {
+				type = lock_data[type].type;
+				break;
+			}
+		}
+		if (lock_data[type].name == NULL)
+			i_fatal("%s: Invalid value %s", env, *lock);
+		if (lock_data[type].func == NULL) {
+			i_fatal("%s: Support for lock type %s "
+				"not compiled into binary", env, *lock);
+		}
+
+		for (i = 0; i < dest; i++) {
+			if (locks[i] == type)
+				i_fatal("%s: Duplicated value %s", env, *lock);
+		}
+
+		/* @UNSAFE */
+		locks[dest++] = type;
+	}
+	locks[dest] = (enum mbox_lock_type)-1;
+}
+
+static void mbox_init_lock_settings(struct mbox_storage *storage)
+{
+	enum mbox_lock_type read_locks[MBOX_LOCK_COUNT+1];
+	enum mbox_lock_type write_locks[MBOX_LOCK_COUNT+1];
+	int r, w;
+
+	mbox_read_lock_methods(storage->set->mbox_read_locks,
+			       "mbox_read_locks", read_locks);
+	mbox_read_lock_methods(storage->set->mbox_write_locks,
+			       "mbox_write_locks", write_locks);
+
+	/* check that read/write list orders match. write_locks must contain
+	   at least read_locks and possibly more. */
+	for (r = w = 0; write_locks[w] != (enum mbox_lock_type)-1; w++) {
+		if (read_locks[r] == (enum mbox_lock_type)-1)
+			break;
+		if (read_locks[r] == write_locks[w])
+			r++;
+	}
+	if (read_locks[r] != (enum mbox_lock_type)-1) {
+		i_fatal("mbox read/write lock list settings are invalid. "
+			"Lock ordering must be the same with both, "
+			"and write locks must contain all read locks "
+			"(and possibly more)");
+	}
+
+	storage->read_locks = p_new(storage->storage.pool,
+				    enum mbox_lock_type, MBOX_LOCK_COUNT+1);
+	memcpy(storage->read_locks, read_locks,
+	       sizeof(*storage->read_locks) * (MBOX_LOCK_COUNT+1));
+
+	storage->write_locks = p_new(storage->storage.pool,
+				     enum mbox_lock_type, MBOX_LOCK_COUNT+1);
+	memcpy(storage->write_locks, write_locks,
+	       sizeof(*storage->write_locks) * (MBOX_LOCK_COUNT+1));
+
+        storage->lock_settings_initialized = TRUE;
+}
+
+static int mbox_file_open_latest(struct mbox_lock_context *ctx, int lock_type)
+{
+	struct mbox_mailbox *mbox = ctx->mbox;
+	struct stat st;
+
+	if (ctx->checked_file || lock_type == F_UNLCK)
+		return 0;
+
+	if (mbox->mbox_fd != -1) {
+		/* we could flush NFS file handle cache here if we wanted to
+		   be sure that the file is latest, but mbox files get rarely
+		   deleted and the flushing might cause errors (e.g. EBUSY for
+		   trying to flush a /var/mail mountpoint) */
+		if (nfs_safe_stat(mailbox_get_path(&mbox->box), &st) < 0) {
+			if (errno == ENOENT)
+				mailbox_set_deleted(&mbox->box);
+			else
+				mbox_set_syscall_error(mbox, "stat()");
+			return -1;
+		}
+
+		if (st.st_ino != mbox->mbox_ino ||
+		    !CMP_DEV_T(st.st_dev, mbox->mbox_dev))
+			mbox_file_close(mbox);
+	}
+
+	if (mbox->mbox_fd == -1) {
+		if (mbox_file_open(mbox) < 0)
+			return -1;
+	}
+
+	ctx->checked_file = TRUE;
+	return 0;
+}
+
+static bool dotlock_callback(unsigned int secs_left, bool stale, void *context)
+{
+        struct mbox_lock_context *ctx = context;
+	enum mbox_lock_type *lock_types;
+	int i;
+
+	if (ctx->using_privileges)
+		restrict_access_drop_priv_gid();
+
+	if (stale && !ctx->dotlock_last_stale) {
+		/* get next index we wish to try locking. it's the one after
+		   dotlocking. */
+		lock_types = ctx->lock_type == F_WRLCK ||
+			(ctx->lock_type == F_UNLCK &&
+			 ctx->mbox->mbox_lock_type == F_WRLCK) ?
+			ctx->mbox->storage->write_locks :
+			ctx->mbox->storage->read_locks;
+
+		for (i = 0; lock_types[i] != (enum mbox_lock_type)-1; i++) {
+			if (lock_types[i] == MBOX_LOCK_DOTLOCK)
+				break;
+		}
+
+		if (lock_types[i] != (enum mbox_lock_type)-1 &&
+		    lock_types[i+1] != (enum mbox_lock_type)-1) {
+			i++;
+			if (mbox_lock_list(ctx, ctx->lock_type, 0, i) <= 0) {
+				/* we couldn't get fd lock -
+				   it's really locked */
+				ctx->dotlock_last_stale = TRUE;
+				return FALSE;
+			}
+			mbox_lock_list(ctx, F_UNLCK, 0, i);
+		}
+	}
+	ctx->dotlock_last_stale = stale;
+
+	index_storage_lock_notify(&ctx->mbox->box, stale ?
+				  MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE :
+				  MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT,
+				  secs_left);
+	if (ctx->using_privileges) {
+		if (restrict_access_use_priv_gid() < 0) {
+			/* shouldn't get here */
+			return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+static int ATTR_NULL(2) ATTR_NOWARN_UNUSED_RESULT
+mbox_dotlock_privileged_op(struct mbox_mailbox *mbox,
+			   struct dotlock_settings *set,
+			   enum mbox_dotlock_op op)
+{
+	const char *box_path, *dir, *fname;
+	int ret = -1, orig_dir_fd, orig_errno;
+
+	orig_dir_fd = open(".", O_RDONLY);
+	if (orig_dir_fd == -1) {
+		mailbox_set_critical(&mbox->box, "open(.) failed: %m");
+		return -1;
+	}
+
+	/* allow dotlocks to be created only for files we can read while we're
+	   unprivileged. to make sure there are no race conditions we first
+	   have to chdir to the mbox file's directory and then use relative
+	   paths. unless this is done, users could:
+	    - create *.lock files to any directory writable by the
+	      privileged group
+	    - DoS other users by dotlocking their mailboxes infinitely
+	*/
+	box_path = mailbox_get_path(&mbox->box);
+	fname = strrchr(box_path, '/');
+	if (fname == NULL) {
+		/* already relative */
+		fname = box_path;
+	} else {
+		dir = t_strdup_until(box_path, fname);
+		if (chdir(dir) < 0) {
+			mailbox_set_critical(&mbox->box,
+				"chdir(%s) failed: %m", dir);
+			i_close_fd(&orig_dir_fd);
+			return -1;
+		}
+		fname++;
+	}
+	if (op == MBOX_DOTLOCK_OP_LOCK) {
+		if (access(fname, R_OK) < 0) {
+			mailbox_set_critical(&mbox->box,
+				"access(%s) failed: %m", box_path);
+			i_close_fd(&orig_dir_fd);
+			return -1;
+		}
+	}
+
+	if (restrict_access_use_priv_gid() < 0) {
+		i_close_fd(&orig_dir_fd);
+		return -1;
+	}
+
+	switch (op) {
+	case MBOX_DOTLOCK_OP_LOCK:
+		/* we're now privileged - avoid doing as much as possible */
+		ret = file_dotlock_create(set, fname, 0, &mbox->mbox_dotlock);
+		if (ret > 0)
+			mbox->mbox_used_privileges = TRUE;
+		else if (ret < 0 && errno == EACCES) {
+			const char *errmsg =
+				eacces_error_get_creating("file_dotlock_create",
+							  fname);
+			mailbox_set_critical(&mbox->box, "%s", errmsg);
+		} else {
+			mbox_set_syscall_error(mbox, "file_dotlock_create()");
+		}
+		break;
+	case MBOX_DOTLOCK_OP_UNLOCK:
+		/* we're now privileged - avoid doing as much as possible */
+		ret = file_dotlock_delete(&mbox->mbox_dotlock);
+		if (ret < 0)
+			mbox_set_syscall_error(mbox, "file_dotlock_delete()");
+		mbox->mbox_used_privileges = FALSE;
+		break;
+	case MBOX_DOTLOCK_OP_TOUCH:
+		ret = file_dotlock_touch(mbox->mbox_dotlock);
+		if (ret < 0)
+			mbox_set_syscall_error(mbox, "file_dotlock_touch()");
+		break;
+	}
+
+	orig_errno = errno;
+	restrict_access_drop_priv_gid();
+
+	if (fchdir(orig_dir_fd) < 0) {
+		mailbox_set_critical(&mbox->box, "fchdir() failed: %m");
+	}
+	i_close_fd(&orig_dir_fd);
+	errno = orig_errno;
+	return ret;
+}
+
+static void
+mbox_dotlock_log_eacces_error(struct mbox_mailbox *mbox, const char *path)
+{
+	const char *dir, *errmsg, *name;
+	struct stat st;
+	struct group group;
+	int orig_errno = errno;
+
+	errmsg = eacces_error_get_creating("file_dotlock_create", path);
+	dir = strrchr(path, '/');
+	dir = dir == NULL ? "." : t_strdup_until(path, dir);
+	/* allow privileged locking for
+	   a) user's own INBOX,
+	   b) another user's shared INBOX, and
+	   c) anything called INBOX (in inbox=no namespace) */
+	if (!mbox->box.inbox_any && strcmp(mbox->box.name, "INBOX") != 0) {
+		mailbox_set_critical(&mbox->box,
+			"%s (not INBOX -> no privileged locking)", errmsg);
+	} else if (!mbox->mbox_privileged_locking) {
+		dir = mailbox_list_get_root_forced(mbox->box.list,
+						   MAILBOX_LIST_PATH_TYPE_DIR);
+		mailbox_set_critical(&mbox->box,
+			"%s (under root dir %s -> no privileged locking)",
+			errmsg, dir);
+	} else if (stat(dir, &st) == 0 &&
+		   (st.st_mode & 02) == 0 && /* not world-writable */
+		   (st.st_mode & 020) != 0) { /* group-writable */
+		if (i_getgrgid(st.st_gid, &group) <= 0)
+			name = dec2str(st.st_gid);
+		else
+			name = group.gr_name;
+		mailbox_set_critical(&mbox->box,
+			"%s (set mail_privileged_group=%s)", errmsg, name);
+	} else {
+		mailbox_set_critical(&mbox->box,
+			"%s (nonstandard permissions in %s)", errmsg, dir);
+	}
+	errno = orig_errno;
+}
+
+static int
+mbox_lock_dotlock_int(struct mbox_lock_context *ctx, int lock_type, bool try)
+{
+	struct mbox_mailbox *mbox = ctx->mbox;
+	struct dotlock_settings set;
+	int ret;
+
+	if (lock_type == F_UNLCK) {
+		if (!mbox->mbox_dotlocked)
+			return 1;
+
+		if (!mbox->mbox_used_privileges) {
+			if (file_dotlock_delete(&mbox->mbox_dotlock) <= 0) {
+				mbox_set_syscall_error(mbox,
+						       "file_dotlock_delete()");
+			}
+		} else {
+			ctx->using_privileges = TRUE;
+			mbox_dotlock_privileged_op(mbox, NULL,
+						   MBOX_DOTLOCK_OP_UNLOCK);
+			ctx->using_privileges = FALSE;
+		}
+                mbox->mbox_dotlocked = FALSE;
+		return 1;
+	}
+
+	if (mbox->mbox_dotlocked)
+		return 1;
+
+        ctx->dotlock_last_stale = TRUE;
+
+	i_zero(&set);
+	set.use_excl_lock = mbox->storage->storage.set->dotlock_use_excl;
+	set.nfs_flush = mbox->storage->storage.set->mail_nfs_storage;
+	set.timeout = mail_storage_get_lock_timeout(&mbox->storage->storage,
+				mbox->storage->set->mbox_lock_timeout);
+	set.stale_timeout = mbox->storage->set->mbox_dotlock_change_timeout;
+	set.callback = dotlock_callback;
+	set.context = ctx;
+
+	ret = file_dotlock_create(&set, mailbox_get_path(&mbox->box), 0,
+				  &mbox->mbox_dotlock);
+	if (ret >= 0) {
+		/* success / timeout */
+	} else if (errno == EACCES && restrict_access_have_priv_gid() &&
+		   mbox->mbox_privileged_locking) {
+		/* try again, this time with extra privileges */
+		ret = mbox_dotlock_privileged_op(mbox, &set,
+						 MBOX_DOTLOCK_OP_LOCK);
+	} else if (errno == EACCES)
+		mbox_dotlock_log_eacces_error(mbox, mailbox_get_path(&mbox->box));
+	else
+		mbox_set_syscall_error(mbox, "file_dotlock_create()");
+
+	if (ret < 0) {
+		if ((ENOSPACE(errno) || errno == EACCES) && try)
+			return 1;
+		return -1;
+	}
+	if (ret == 0) {
+		mail_storage_set_error(&mbox->storage->storage,
+			MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT);
+		return 0;
+	}
+	mbox->mbox_dotlocked = TRUE;
+
+	if (mbox_file_open_latest(ctx, lock_type) < 0)
+		return -1;
+	return 1;
+}
+
+static int mbox_lock_dotlock(struct mbox_lock_context *ctx, int lock_type,
+			     time_t max_wait_time ATTR_UNUSED)
+{
+	return mbox_lock_dotlock_int(ctx, lock_type, FALSE);
+}
+
+static int mbox_lock_dotlock_try(struct mbox_lock_context *ctx, int lock_type,
+				 time_t max_wait_time ATTR_UNUSED)
+{
+	return mbox_lock_dotlock_int(ctx, lock_type, TRUE);
+}
+
+#ifdef HAVE_FLOCK
+static int mbox_lock_flock(struct mbox_lock_context *ctx, int lock_type,
+			   time_t max_wait_time)
+{
+	time_t now;
+	unsigned int next_alarm;
+
+	if (mbox_file_open_latest(ctx, lock_type) < 0)
+		return -1;
+
+	if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1)
+		return 1;
+
+	if (lock_type == F_WRLCK)
+		lock_type = LOCK_EX;
+	else if (lock_type == F_RDLCK)
+		lock_type = LOCK_SH;
+	else
+		lock_type = LOCK_UN;
+
+	if (max_wait_time == 0) {
+		/* usually we're waiting here, but if we came from
+		   mbox_lock_dotlock(), we just want to try locking */
+		lock_type |= LOCK_NB;
+	} else {
+		now = time(NULL);
+		if (now >= max_wait_time)
+			alarm(1);
+		else
+			alarm(I_MIN(max_wait_time - now, 5));
+	}
+
+	while (flock(ctx->mbox->mbox_fd, lock_type) < 0) {
+		if (errno != EINTR) {
+			if (errno == EWOULDBLOCK && max_wait_time == 0) {
+				/* non-blocking lock trying failed */
+				return 0;
+			}
+			alarm(0);
+			mbox_set_syscall_error(ctx->mbox, "flock()");
+			return -1;
+		}
+
+		now = time(NULL);
+		if (now >= max_wait_time) {
+			alarm(0);
+			return 0;
+		}
+
+		/* notify locks once every 5 seconds.
+		   try to use rounded values. */
+		next_alarm = (max_wait_time - now) % 5;
+		if (next_alarm == 0)
+			next_alarm = 5;
+		alarm(next_alarm);
+
+		index_storage_lock_notify(&ctx->mbox->box,
+					  MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT,
+					  max_wait_time - now);
+	}
+
+	alarm(0);
+	return 1;
+}
+#endif
+
+#ifdef HAVE_LOCKF
+static int mbox_lock_lockf(struct mbox_lock_context *ctx, int lock_type,
+			   time_t max_wait_time)
+{
+	time_t now;
+	unsigned int next_alarm;
+
+	if (mbox_file_open_latest(ctx, lock_type) < 0)
+		return -1;
+
+	if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1)
+		return 1;
+
+	if (lock_type == F_UNLCK)
+		lock_type = F_ULOCK;
+	else if (max_wait_time == 0) {
+		/* usually we're waiting here, but if we came from
+		   mbox_lock_dotlock(), we just want to try locking */
+		lock_type = F_TLOCK;
+	} else {
+		now = time(NULL);
+		if (now >= max_wait_time)
+			alarm(1);
+		else
+			alarm(I_MIN(max_wait_time - now, 5));
+		lock_type = F_LOCK;
+	}
+
+	while (lockf(ctx->mbox->mbox_fd, lock_type, 0) < 0) {
+		if (errno != EINTR) {
+			if ((errno == EACCES || errno == EAGAIN) &&
+			    max_wait_time == 0) {
+				/* non-blocking lock trying failed */
+				return 0;
+			}
+			alarm(0);
+			mbox_set_syscall_error(ctx->mbox, "lockf()");
+			return -1;
+		}
+
+		now = time(NULL);
+		if (now >= max_wait_time) {
+			alarm(0);
+			return 0;
+		}
+
+		/* notify locks once every 5 seconds.
+		   try to use rounded values. */
+		next_alarm = (max_wait_time - now) % 5;
+		if (next_alarm == 0)
+			next_alarm = 5;
+		alarm(next_alarm);
+
+		index_storage_lock_notify(&ctx->mbox->box,
+					  MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT,
+					  max_wait_time - now);
+	}
+
+	alarm(0);
+	return 1;
+}
+#endif
+
+static int mbox_lock_fcntl(struct mbox_lock_context *ctx, int lock_type,
+			   time_t max_wait_time)
+{
+	struct flock fl;
+	time_t now;
+	unsigned int next_alarm;
+	int wait_type;
+
+	if (mbox_file_open_latest(ctx, lock_type) < 0)
+		return -1;
+
+	if (lock_type == F_UNLCK && ctx->mbox->mbox_fd == -1)
+		return 1;
+
+	i_zero(&fl);
+	fl.l_type = lock_type;
+	fl.l_whence = SEEK_SET;
+	fl.l_start = 0;
+	fl.l_len = 0;
+
+	if (max_wait_time == 0) {
+		/* usually we're waiting here, but if we came from
+		   mbox_lock_dotlock(), we just want to try locking */
+		wait_type = F_SETLK;
+	} else {
+		wait_type = F_SETLKW;
+		now = time(NULL);
+		if (now >= max_wait_time)
+			alarm(1);
+		else
+			alarm(I_MIN(max_wait_time - now, 5));
+	}
+
+	while (fcntl(ctx->mbox->mbox_fd, wait_type, &fl) < 0) {
+		if (errno != EINTR) {
+			if ((errno == EACCES || errno == EAGAIN) &&
+			    wait_type == F_SETLK) {
+				/* non-blocking lock trying failed */
+				return 0;
+			}
+			alarm(0);
+			if (errno != EACCES) {
+				mbox_set_syscall_error(ctx->mbox, "fcntl()");
+				return -1;
+			}
+			mailbox_set_critical(&ctx->mbox->box,
+				"fcntl() failed with mbox file %s: "
+				"File is locked by another process (EACCES)",
+				mailbox_get_path(&ctx->mbox->box));
+			return -1;
+		}
+
+		now = time(NULL);
+		if (now >= max_wait_time) {
+			alarm(0);
+			return 0;
+		}
+
+		/* notify locks once every 5 seconds.
+		   try to use rounded values. */
+		next_alarm = (max_wait_time - now) % 5;
+		if (next_alarm == 0)
+			next_alarm = 5;
+		alarm(next_alarm);
+
+		index_storage_lock_notify(&ctx->mbox->box,
+					  MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT,
+					  max_wait_time - now);
+	}
+
+	alarm(0);
+	ctx->fcntl_locked = TRUE;
+	return 1;
+}
+
+static int ATTR_NOWARN_UNUSED_RESULT
+mbox_lock_list(struct mbox_lock_context *ctx, int lock_type,
+	       time_t max_wait_time, int idx)
+{
+	enum mbox_lock_type *lock_types;
+        enum mbox_lock_type type;
+	int i, ret = 0;
+	bool locked_status;
+
+	ctx->lock_type = lock_type;
+
+	lock_types = lock_type == F_WRLCK ||
+		(lock_type == F_UNLCK && ctx->mbox->mbox_lock_type == F_WRLCK) ?
+		ctx->mbox->storage->write_locks :
+		ctx->mbox->storage->read_locks;
+	for (i = idx; lock_types[i] != (enum mbox_lock_type)-1; i++) {
+		type = lock_types[i];
+		locked_status = lock_type != F_UNLCK;
+
+		if (ctx->locked_status[type] == locked_status)
+			continue;
+		ctx->locked_status[type] = locked_status;
+
+		ret = lock_data[type].func(ctx, lock_type, max_wait_time);
+		if (ret <= 0)
+			break;
+	}
+	return ret;
+}
+
+static int mbox_update_locking(struct mbox_mailbox *mbox, int lock_type,
+			       bool *fcntl_locked_r)
+{
+	struct mbox_lock_context ctx;
+	time_t max_wait_time;
+	int ret, i;
+	bool drop_locks;
+
+	*fcntl_locked_r = FALSE;
+
+        index_storage_lock_notify_reset(&mbox->box);
+
+	if (!mbox->storage->lock_settings_initialized)
+                mbox_init_lock_settings(mbox->storage);
+
+	if (mbox->mbox_fd == -1 && mbox->mbox_file_stream != NULL) {
+		/* read-only mbox stream. no need to lock. */
+		i_assert(mbox_is_backend_readonly(mbox));
+		mbox->mbox_lock_type = lock_type;
+		return 1;
+	}
+
+	max_wait_time = time(NULL) +
+		mail_storage_get_lock_timeout(&mbox->storage->storage,
+			mbox->storage->set->mbox_lock_timeout);
+
+	i_zero(&ctx);
+	ctx.mbox = mbox;
+
+	if (mbox->mbox_lock_type == F_WRLCK) {
+		/* dropping to shared lock. first drop those that we
+		   don't remove completely. */
+		const enum mbox_lock_type *read_locks =
+                        mbox->storage->read_locks;
+
+		for (i = 0; i < MBOX_LOCK_COUNT; i++)
+			ctx.locked_status[i] = TRUE;
+		for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++)
+			ctx.locked_status[read_locks[i]] = FALSE;
+		drop_locks = TRUE;
+	} else {
+		drop_locks = FALSE;
+	}
+
+	mbox->mbox_lock_type = lock_type;
+	ret = mbox_lock_list(&ctx, lock_type, max_wait_time, 0);
+	if (ret <= 0) {
+		if (!drop_locks)
+			mbox_unlock_files(&ctx);
+		if (ret == 0) {
+			mail_storage_set_error(&mbox->storage->storage,
+				MAIL_ERROR_TEMP, MAIL_ERRSTR_LOCK_TIMEOUT);
+		}
+		return ret;
+	}
+
+	if (drop_locks) {
+		/* dropping to shared lock: drop the locks that are only
+		   in write list */
+		const enum mbox_lock_type *read_locks =
+                        mbox->storage->read_locks;
+		const enum mbox_lock_type *write_locks =
+			mbox->storage->write_locks;
+
+		memset(ctx.locked_status, 0, sizeof(ctx.locked_status));
+		for (i = 0; write_locks[i] != (enum mbox_lock_type)-1; i++)
+			ctx.locked_status[write_locks[i]] = TRUE;
+		for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++)
+			ctx.locked_status[read_locks[i]] = FALSE;
+
+		mbox->mbox_lock_type = F_WRLCK;
+		mbox_lock_list(&ctx, F_UNLCK, 0, 0);
+		mbox->mbox_lock_type = F_RDLCK;
+	}
+
+	*fcntl_locked_r = ctx.fcntl_locked;
+	return 1;
+}
+
+int mbox_lock(struct mbox_mailbox *mbox, int lock_type,
+	      unsigned int *lock_id_r)
+{
+	const char *path = mailbox_get_path(&mbox->box);
+	int mbox_fd = mbox->mbox_fd;
+	bool fcntl_locked;
+	int ret;
+
+	if (lock_type == F_RDLCK && mbox->external_transactions > 0 &&
+	    mbox->mbox_lock_type != F_RDLCK) {
+		/* we have a transaction open that is going to save mails
+		   and apparently also wants to read from the same mailbox
+		   (copy, move, catenate). we need to write lock the mailbox,
+		   since we can't later upgrade a read lock to write lock. */
+		lock_type = F_WRLCK;
+	}
+
+	/* allow only unlock -> shared/exclusive or exclusive -> shared */
+	i_assert(lock_type == F_RDLCK || lock_type == F_WRLCK);
+	i_assert(lock_type == F_RDLCK || mbox->mbox_lock_type != F_RDLCK);
+
+	if (mbox->mbox_lock_type == F_UNLCK) {
+		ret = mbox_update_locking(mbox, lock_type, &fcntl_locked);
+		if (ret <= 0)
+			return ret;
+
+		if (mbox->storage->storage.set->mail_nfs_storage) {
+			if (fcntl_locked) {
+				nfs_flush_attr_cache_fd_locked(path, mbox_fd);
+				nfs_flush_read_cache_locked(path, mbox_fd);
+			} else {
+				nfs_flush_attr_cache_unlocked(path);
+				nfs_flush_read_cache_unlocked(path, mbox_fd);
+			}
+		}
+
+		mbox->mbox_lock_id += 2;
+	}
+
+	if (lock_type == F_RDLCK) {
+		mbox->mbox_shared_locks++;
+		*lock_id_r = mbox->mbox_lock_id;
+	} else {
+		mbox->mbox_excl_locks++;
+		*lock_id_r = mbox->mbox_lock_id + 1;
+	}
+	if (mbox->mbox_stream != NULL)
+		istream_raw_mbox_set_locked(mbox->mbox_stream);
+	return 1;
+}
+
+static int mbox_unlock_files(struct mbox_lock_context *ctx)
+{
+	int ret = 0;
+
+	if (mbox_lock_list(ctx, F_UNLCK, 0, 0) < 0)
+		ret = -1;
+
+	ctx->mbox->mbox_lock_id += 2;
+	ctx->mbox->mbox_lock_type = F_UNLCK;
+	return ret;
+}
+
+int mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id)
+{
+	struct mbox_lock_context ctx;
+	bool fcntl_locked;
+	int i;
+
+	i_assert(mbox->mbox_lock_id == (lock_id & ~1U));
+
+	if ((lock_id & 1) != 0) {
+		/* dropping exclusive lock */
+		i_assert(mbox->mbox_excl_locks > 0);
+		if (--mbox->mbox_excl_locks > 0)
+			return 0;
+		if (mbox->mbox_shared_locks > 0) {
+			/* drop to shared lock */
+			if (mbox_update_locking(mbox, F_RDLCK,
+						&fcntl_locked) < 0)
+				return -1;
+			return 0;
+		}
+	} else {
+		/* dropping shared lock */
+		i_assert(mbox->mbox_shared_locks > 0);
+		if (--mbox->mbox_shared_locks > 0)
+			return 0;
+		if (mbox->mbox_excl_locks > 0)
+			return 0;
+	}
+	/* all locks gone */
+
+	/* make sure we don't read the stream while unlocked */
+	if (mbox->mbox_stream != NULL)
+		istream_raw_mbox_set_unlocked(mbox->mbox_stream);
+
+	i_zero(&ctx);
+	ctx.mbox = mbox;
+
+	for (i = 0; i < MBOX_LOCK_COUNT; i++)
+		ctx.locked_status[i] = TRUE;
+
+	return mbox_unlock_files(&ctx);
+}
+
+unsigned int mbox_get_cur_lock_id(struct mbox_mailbox *mbox)
+{
+	return mbox->mbox_lock_id +
+		(mbox->mbox_excl_locks > 0 ? 1 : 0);
+}
+
+void mbox_dotlock_touch(struct mbox_mailbox *mbox)
+{
+	if (mbox->mbox_dotlock == NULL)
+		return;
+
+	if (!mbox->mbox_used_privileges)
+		(void)file_dotlock_touch(mbox->mbox_dotlock);
+	else {
+		mbox_dotlock_privileged_op(mbox, NULL,
+						 MBOX_DOTLOCK_OP_TOUCH);
+	}
+}
diff --git a/src/lib-storage/index/mbox/mbox-lock.h b/src/lib-storage/index/mbox/mbox-lock.h
new file mode 100644
index 0000000..2175908
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-lock.h
@@ -0,0 +1,15 @@
+#ifndef MBOX_LOCK_H
+#define MBOX_LOCK_H
+
+/* NOTE: if mbox file is not open, it's opened. if it is open but file has
+   been overwritten (ie. inode has changed), it's reopened. */
+int mbox_lock(struct mbox_mailbox *mbox, int lock_type,
+	      unsigned int *lock_id_r);
+int ATTR_NOWARN_UNUSED_RESULT
+mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id);
+
+unsigned int mbox_get_cur_lock_id(struct mbox_mailbox *mbox);
+
+void mbox_dotlock_touch(struct mbox_mailbox *mbox);
+
+#endif
diff --git a/src/lib-storage/index/mbox/mbox-mail.c b/src/lib-storage/index/mbox/mbox-mail.c
new file mode 100644
index 0000000..09223b0
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-mail.c
@@ -0,0 +1,439 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "hex-binary.h"
+#include "index-mail.h"
+#include "mbox-storage.h"
+#include "mbox-file.h"
+#include "mbox-lock.h"
+#include "mbox-sync-private.h"
+#include "istream-raw-mbox.h"
+#include "istream-header-filter.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+static void mbox_prepare_resync(struct mail *mail)
+{
+	struct mbox_transaction_context *t = MBOX_TRANSCTX(mail->transaction);
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->box);
+
+	if (mbox->mbox_lock_type == F_RDLCK) {
+		if (mbox->mbox_lock_id == t->read_lock_id)
+			t->read_lock_id = 0;
+		mbox_unlock(mbox, mbox->mbox_lock_id);
+		i_assert(mbox->mbox_lock_type == F_UNLCK);
+	}
+}
+
+static int mbox_mail_seek(struct index_mail *mail)
+{
+	struct mail *_mail = &mail->mail.mail;
+	struct mbox_transaction_context *t = MBOX_TRANSCTX(_mail->transaction);
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box);
+	enum mbox_sync_flags sync_flags = 0;
+	int ret, try;
+	bool deleted;
+
+	if (_mail->expunged || mbox->syncing)
+		return -1;
+
+	if (!mail_stream_access_start(_mail))
+		return -1;
+
+	if (mbox->mbox_stream != NULL &&
+	    istream_raw_mbox_is_corrupted(mbox->mbox_stream)) {
+		/* clear the corruption by forcing a full resync */
+		sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
+	}
+
+	for (try = 0; try < 2; try++) {
+		if ((sync_flags & MBOX_SYNC_FORCE_SYNC) != 0) {
+			/* dirty offsets are broken. make sure we can sync. */
+			mbox_prepare_resync(_mail);
+		}
+		if (mbox->mbox_lock_type == F_UNLCK) {
+			i_assert(t->read_lock_id == 0);
+			sync_flags |= MBOX_SYNC_LOCK_READING;
+			if (mbox_sync(mbox, sync_flags) < 0)
+				return -1;
+			t->read_lock_id = mbox_get_cur_lock_id(mbox);
+			i_assert(t->read_lock_id != 0);
+
+			/* refresh index file after mbox has been locked to
+			   make sure we get only up-to-date mbox offsets. */
+			if (mail_index_refresh(mbox->box.index) < 0) {
+				mailbox_set_index_error(&mbox->box);
+				return -1;
+			}
+
+			i_assert(mbox->mbox_lock_type != F_UNLCK);
+		} else if (t->read_lock_id == 0) {
+			/* file is already locked by another transaction, but
+			   we must keep it locked for the entire transaction,
+			   so increase the lock counter. */
+			if (mbox_lock(mbox, mbox->mbox_lock_type,
+				      &t->read_lock_id) < 0)
+				i_unreached();
+		}
+
+		if (mbox_file_open_stream(mbox) < 0)
+			return -1;
+
+		ret = mbox_file_seek(mbox, _mail->transaction->view,
+				     _mail->seq, &deleted);
+		if (ret > 0) {
+			/* success */
+			break;
+		}
+		if (ret < 0) {
+			if (deleted)
+				mail_set_expunged(_mail);
+			return -1;
+		}
+
+		/* we'll need to re-sync it completely */
+		sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
+	}
+	if (ret == 0) {
+		mail_set_critical(_mail, "mbox: Losing sync");
+	}
+	return 0;
+}
+
+static int mbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box);
+
+	if (index_mail_get_received_date(_mail, date_r) == 0)
+		return 0;
+
+	if (mbox_mail_seek(mail) < 0)
+		return -1;
+	data->received_date =
+		istream_raw_mbox_get_received_time(mbox->mbox_stream);
+	if (data->received_date == (time_t)-1) {
+		/* it's broken and conflicts with our "not found"
+		   return value. change it. */
+		data->received_date = 0;
+	}
+
+	*date_r = data->received_date;
+	return 0;
+}
+
+static int mbox_mail_get_save_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	if (index_mail_get_save_date(_mail, date_r) > 0)
+		return 0;
+
+	/* no way to know this. save the current time into cache and use
+	   that from now on. this works only as long as the index files
+	   are permanent */
+	data->save_date = ioloop_time;
+	*date_r = data->save_date;
+	return 0;
+}
+
+static int
+mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r)
+{
+	struct mail *_mail = &mail->mail.mail;
+	static uint8_t empty_md5[16] =
+		{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box);
+	const void *ext_data;
+
+	if (mail->data.guid != NULL) {
+		*value_r = mail->data.guid;
+		return 1;
+	}
+
+	mail_index_lookup_ext(_mail->transaction->view, _mail->seq,
+			      mbox->md5hdr_ext_idx, &ext_data, NULL);
+	if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) {
+		mail->data.guid = p_strdup(mail->mail.data_pool,
+					   binary_to_hex(ext_data, 16));
+		*value_r = mail->data.guid;
+		return 1;
+	} else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) {
+		mail_set_expunged(_mail);
+		return -1;
+	} else {
+		return 0;
+	}
+}
+
+static int
+mbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+		      const char **value_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box);
+	uoff_t offset;
+	bool move_offset;
+	int ret;
+
+	switch (field) {
+	case MAIL_FETCH_FROM_ENVELOPE:
+		if (mbox_mail_seek(mail) < 0)
+			return -1;
+
+		*value_r = istream_raw_mbox_get_sender(mbox->mbox_stream);
+		return 0;
+	case MAIL_FETCH_GUID:
+	case MAIL_FETCH_HEADER_MD5:
+		if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0)
+			return ret < 0 ? -1 : 0;
+
+		/* i guess in theory the empty_md5 is valid and can happen,
+		   but it's almost guaranteed that it means the MD5 sum is
+		   missing. recalculate it. */
+		if (mbox->mbox_lock_type == F_UNLCK ||
+		    mbox->mbox_stream == NULL) {
+			offset = 0;
+			move_offset = FALSE;
+		} else {
+			offset = istream_raw_mbox_get_start_offset(mbox->mbox_stream);
+			move_offset = TRUE;
+		}
+		mbox->mbox_save_md5 = TRUE;
+		if (mbox_sync(mbox, MBOX_SYNC_FORCE_SYNC |
+			      MBOX_SYNC_READONLY) < 0)
+			return -1;
+		if (move_offset) {
+			if (istream_raw_mbox_seek(mbox->mbox_stream,
+						  offset) < 0) {
+				i_error("mbox %s sync lost during MD5 syncing",
+					_mail->box->name);
+				return -1;
+			}
+		}
+
+		if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) {
+			i_error("mbox %s resyncing didn't save header MD5 values",
+				_mail->box->name);
+			return -1;
+		}
+		return ret < 0 ? -1 : 0;
+	default:
+		break;
+	}
+
+	return index_mail_get_special(_mail, field, value_r);
+}
+
+static int
+mbox_mail_get_next_offset(struct index_mail *mail, uoff_t *next_offset_r)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->mail.mail.box);
+	struct mail_index_view *view;
+	const struct mail_index_header *hdr;
+	uint32_t seq;
+	int trailer_size;
+	int ret = 1;
+
+	*next_offset_r = UOFF_T_MAX;
+
+	hdr = mail_index_get_header(mail->mail.mail.transaction->view);
+	if (mail->mail.mail.seq > hdr->messages_count) {
+		/* we're appending a new message */
+		return 0;
+	}
+
+	/* We can't really trust trans_view. The next message may already be
+	   expunged from it. Also hdr.messages_count may be incorrect there.
+	   So refresh the index to get the latest changes and get the next
+	   message's offset using a new view. */
+	i_assert(mbox->mbox_lock_type != F_UNLCK);
+	if (mbox_sync_header_refresh(mbox) < 0)
+		return -1;
+
+	view = mail_index_view_open(mail->mail.mail.box->index);
+	hdr = mail_index_get_header(view);
+	if (!mail_index_lookup_seq(view, mail->mail.mail.uid, &seq))
+		i_panic("Message unexpectedly expunged from index");
+
+	if (seq < hdr->messages_count) {
+		if (mbox_file_lookup_offset(mbox, view, seq + 1,
+					    next_offset_r) <= 0)
+			ret = -1;
+	} else if (mail->mail.mail.box->input != NULL) {
+		/* opened the mailbox as input stream. we can't trust the
+		   sync_size, since it's wrong with compressed mailboxes */
+		ret = 0;
+	} else {
+		/* last message, use the synced mbox size */
+		trailer_size =
+			mbox->storage->storage.set->mail_save_crlf ? 2 : 1;
+		*next_offset_r = mbox->mbox_hdr.sync_size - trailer_size;
+	}
+	mail_index_view_close(&view);
+	return ret;
+}
+
+static int mbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(_mail->box);
+	struct istream *input;
+	struct message_size hdr_size;
+	uoff_t old_offset, body_offset, body_size, next_offset;
+
+	if (index_mail_get_physical_size(_mail, size_r) == 0)
+		return 0;
+
+	/* we want to return the header size as seen by mail_get_stream(). */
+	old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
+	if (mail_get_stream(_mail, &hdr_size, NULL, &input) < 0)
+		return -1;
+
+	/* our header size varies, so don't do any caching */
+	if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) {
+		mail_set_critical(_mail, "mbox: Couldn't get body offset");
+		return -1;
+	}
+
+	/* use the next message's offset to avoid reading through the entire
+	   message body to find out its size */
+	if (mbox_mail_get_next_offset(mail, &next_offset) > 0)
+		body_size = next_offset - body_offset;
+	else
+		body_size = UOFF_T_MAX;
+
+	/* verify that the calculated body size is correct */
+	if (istream_raw_mbox_get_body_size(mbox->mbox_stream,
+					   body_size, &body_size) < 0) {
+		mail_set_critical(_mail, "mbox: Couldn't get body size");
+		return -1;
+	}
+
+	data->physical_size = hdr_size.physical_size + body_size;
+	*size_r = data->physical_size;
+
+	i_stream_seek(input, old_offset);
+	return 0;
+}
+
+static int mbox_mail_init_stream(struct index_mail *mail)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(mail->mail.mail.box);
+	struct istream *raw_stream;
+	uoff_t hdr_offset, next_offset;
+	int ret;
+
+	if (mbox_mail_seek(mail) < 0)
+		return -1;
+
+	ret = mbox_mail_get_next_offset(mail, &next_offset);
+	if (ret < 0) {
+		if (mbox_mail_seek(mail) < 0)
+			return -1;
+		ret = mbox_mail_get_next_offset(mail, &next_offset);
+		if (ret < 0) {
+			i_warning("mbox %s: Can't find next message offset "
+				  "for uid=%u", mailbox_get_path(&mbox->box),
+				  mail->mail.mail.uid);
+		}
+	}
+
+	raw_stream = mbox->mbox_stream;
+	if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) {
+		mail_set_critical(&mail->mail.mail,
+			"mbox: Couldn't get header offset");
+		return -1;
+	}
+	i_stream_seek(raw_stream, hdr_offset);
+
+	if (next_offset != UOFF_T_MAX)
+		istream_raw_mbox_set_next_offset(raw_stream, next_offset);
+
+	raw_stream = i_stream_create_limit(raw_stream, UOFF_T_MAX);
+	mail->data.stream =
+		i_stream_create_header_filter(raw_stream,
+				HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
+				mbox_hide_headers, mbox_hide_headers_count,
+				*null_header_filter_callback, NULL);
+	i_stream_unref(&raw_stream);
+	return 0;
+}
+
+static int mbox_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
+				struct message_size *hdr_size,
+				struct message_size *body_size,
+				struct istream **stream_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	if (mail->data.stream == NULL) {
+		if (mbox_mail_init_stream(mail) < 0)
+			return -1;
+	}
+
+	return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
+}
+
+static void mbox_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	index_mail_set_seq(_mail, seq, saving);
+	mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
+}
+
+static bool mbox_mail_set_uid(struct mail *_mail, uint32_t uid)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	bool ret;
+
+	ret = index_mail_set_uid(_mail, uid);
+	mail->data.dont_cache_fetch_fields |= MAIL_FETCH_PHYSICAL_SIZE;
+	return ret;
+}
+
+struct mail_vfuncs mbox_mail_vfuncs = {
+	index_mail_close,
+	index_mail_free,
+	mbox_mail_set_seq,
+	mbox_mail_set_uid,
+	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
+	index_mail_precache,
+	index_mail_add_temp_wanted_fields,
+
+	index_mail_get_flags,
+	index_mail_get_keywords,
+	index_mail_get_keyword_indexes,
+	index_mail_get_modseq,
+	index_mail_get_pvt_modseq,
+	index_mail_get_parts,
+	index_mail_get_date,
+	mbox_mail_get_received_date,
+	mbox_mail_get_save_date,
+	index_mail_get_virtual_size,
+	mbox_mail_get_physical_size,
+	index_mail_get_first_header,
+	index_mail_get_headers,
+	index_mail_get_header_stream,
+	mbox_mail_get_stream,
+	index_mail_get_binary_stream,
+	mbox_mail_get_special,
+	index_mail_get_backend_mail,
+	index_mail_update_flags,
+	index_mail_update_keywords,
+	index_mail_update_modseq,
+	index_mail_update_pvt_modseq,
+	NULL,
+	index_mail_expunge,
+	index_mail_set_cache_corrupted,
+	index_mail_opened,
+};
diff --git a/src/lib-storage/index/mbox/mbox-md5-all.c b/src/lib-storage/index/mbox/mbox-md5-all.c
new file mode 100644
index 0000000..9a09fb2
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-md5-all.c
@@ -0,0 +1,39 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "md5.h"
+#include "message-parser.h"
+#include "mbox-md5.h"
+
+
+struct mbox_md5_context {
+	struct md5_context hdr_md5_ctx;
+};
+
+static struct mbox_md5_context *mbox_md5_all_init(void)
+{
+	struct mbox_md5_context *ctx;
+
+	ctx = i_new(struct mbox_md5_context, 1);
+	md5_init(&ctx->hdr_md5_ctx);
+	return ctx;
+}
+
+static void mbox_md5_all_more(struct mbox_md5_context *ctx,
+			      struct message_header_line *hdr)
+{
+	md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+}
+
+static void mbox_md5_all_finish(struct mbox_md5_context *ctx,
+				unsigned char result[STATIC_ARRAY 16])
+{
+	md5_final(&ctx->hdr_md5_ctx, result);
+	i_free(ctx);
+}
+
+struct mbox_md5_vfuncs mbox_md5_all = {
+	mbox_md5_all_init,
+	mbox_md5_all_more,
+	mbox_md5_all_finish
+};
diff --git a/src/lib-storage/index/mbox/mbox-md5-apop3d.c b/src/lib-storage/index/mbox/mbox-md5-apop3d.c
new file mode 100644
index 0000000..56f6f91
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-md5-apop3d.c
@@ -0,0 +1,119 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "md5.h"
+#include "message-parser.h"
+#include "mbox-md5.h"
+
+
+struct mbox_md5_context {
+	struct md5_context hdr_md5_ctx;
+	bool seen_received_hdr;
+};
+
+struct mbox_md5_header_func {
+	const char *header;
+	bool (*func)(struct mbox_md5_context *ctx,
+		     struct message_header_line *hdr);
+};
+
+static bool parse_date(struct mbox_md5_context *ctx,
+		       struct message_header_line *hdr)
+{
+	if (!ctx->seen_received_hdr) {
+		/* Received-header contains date too, and more trusted one */
+		md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+	}
+	return TRUE;
+}
+
+static bool parse_delivered_to(struct mbox_md5_context *ctx,
+			       struct message_header_line *hdr)
+{
+	md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+	return TRUE;
+}
+
+static bool parse_message_id(struct mbox_md5_context *ctx,
+			     struct message_header_line *hdr)
+{
+	if (!ctx->seen_received_hdr) {
+		/* Received-header contains unique ID too,
+		   and more trusted one */
+		md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+	}
+	return TRUE;
+}
+
+static bool parse_received(struct mbox_md5_context *ctx,
+			   struct message_header_line *hdr)
+{
+	if (!ctx->seen_received_hdr) {
+		/* get only the first received-header */
+		md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+		if (!hdr->continues)
+			ctx->seen_received_hdr = TRUE;
+	}
+	return TRUE;
+}
+
+static bool parse_x_delivery_id(struct mbox_md5_context *ctx,
+				struct message_header_line *hdr)
+{
+	/* Let the local delivery agent help generate unique ID's but don't
+	   blindly trust this header alone as it could just as easily come from
+	   the remote. */
+	md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+	return TRUE;
+}
+
+
+static struct mbox_md5_header_func md5_header_funcs[] = {
+	{ "Date", parse_date },
+	{ "Delivered-To", parse_delivered_to },
+	{ "Message-ID", parse_message_id },
+	{ "Received", parse_received },
+	{ "X-Delivery-ID", parse_x_delivery_id }
+};
+
+static int bsearch_header_func_cmp(const void *p1, const void *p2)
+{
+	const char *key = p1;
+	const struct mbox_md5_header_func *func = p2;
+
+	return strcasecmp(key, func->header);
+}
+
+static struct mbox_md5_context *mbox_md5_apop3d_init(void)
+{
+	struct mbox_md5_context *ctx;
+
+	ctx = i_new(struct mbox_md5_context, 1);
+	md5_init(&ctx->hdr_md5_ctx);
+	return ctx;
+}
+
+static void mbox_md5_apop3d_more(struct mbox_md5_context *ctx,
+				 struct message_header_line *hdr)
+{
+	struct mbox_md5_header_func *func;
+
+	func = bsearch(hdr->name, md5_header_funcs,
+		       N_ELEMENTS(md5_header_funcs), sizeof(*md5_header_funcs),
+		       bsearch_header_func_cmp);
+	if (func != NULL)
+		(void)func->func(ctx, hdr);
+}
+
+static void mbox_md5_apop3d_finish(struct mbox_md5_context *ctx,
+				   unsigned char result[STATIC_ARRAY 16])
+{
+	md5_final(&ctx->hdr_md5_ctx, result);
+	i_free(ctx);
+}
+
+struct mbox_md5_vfuncs mbox_md5_apop3d = {
+	mbox_md5_apop3d_init,
+	mbox_md5_apop3d_more,
+	mbox_md5_apop3d_finish
+};
diff --git a/src/lib-storage/index/mbox/mbox-md5.h b/src/lib-storage/index/mbox/mbox-md5.h
new file mode 100644
index 0000000..7584052
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-md5.h
@@ -0,0 +1,17 @@
+#ifndef MBOX_MD5_H
+#define MBOX_MD5_H
+
+struct message_header_line;
+
+struct mbox_md5_vfuncs {
+	struct mbox_md5_context *(*init)(void);
+	void (*more)(struct mbox_md5_context *ctx,
+		     struct message_header_line *hdr);
+	void (*finish)(struct mbox_md5_context *ctx,
+		       unsigned char result[STATIC_ARRAY 16]);
+};
+
+extern struct mbox_md5_vfuncs mbox_md5_apop3d;
+extern struct mbox_md5_vfuncs mbox_md5_all;
+
+#endif
diff --git a/src/lib-storage/index/mbox/mbox-save.c b/src/lib-storage/index/mbox/mbox-save.c
new file mode 100644
index 0000000..2fb3e19
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-save.c
@@ -0,0 +1,833 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "base64.h"
+#include "hostpid.h"
+#include "randgen.h"
+#include "istream.h"
+#include "ostream.h"
+#include "str.h"
+#include "write-full.h"
+#include "istream-header-filter.h"
+#include "istream-crlf.h"
+#include "istream-concat.h"
+#include "message-parser.h"
+#include "mail-user.h"
+#include "index-mail.h"
+#include "mbox-storage.h"
+#include "mbox-file.h"
+#include "mbox-from.h"
+#include "mbox-lock.h"
+#include "mbox-md5.h"
+#include "mbox-sync-private.h"
+
+#include <stddef.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <utime.h>
+
+#define MBOX_DELIVERY_ID_RAND_BYTES (64/8)
+
+struct mbox_save_context {
+	struct mail_save_context ctx;
+
+	struct mbox_mailbox *mbox;
+	struct mail_index_transaction *trans;
+	uoff_t append_offset, mail_offset;
+	time_t orig_atime;
+
+	string_t *headers;
+	size_t space_end_idx;
+	uint32_t seq, next_uid, uid_validity;
+
+	struct istream *input;
+	struct ostream *output;
+	uoff_t extra_hdr_offset, eoh_offset;
+	char last_char;
+
+	struct mbox_md5_context *mbox_md5_ctx;
+	char *x_delivery_id_header;
+
+	bool synced:1;
+	bool failed:1;
+	bool finished:1;
+};
+
+#define MBOX_SAVECTX(s)		container_of(s, struct mbox_save_context, ctx)
+
+static void ostream_error(struct mbox_save_context *ctx, const char *func)
+{
+	mbox_ostream_set_syscall_error(ctx->mbox, ctx->output, func);
+	ctx->failed = TRUE;
+}
+
+static void write_stream_error(struct mbox_save_context *ctx)
+{
+	ostream_error(ctx, "write()");
+}
+
+static void lseek_stream_error(struct mbox_save_context *ctx)
+{
+	ostream_error(ctx, "o_stream_seek()");
+}
+
+static int mbox_seek_to_end(struct mbox_save_context *ctx, uoff_t *offset)
+{
+	struct stat st;
+	char ch;
+	int fd;
+
+	if (ctx->mbox->mbox_writeonly) {
+		*offset = 0;
+		return 0;
+	}
+
+	fd = ctx->mbox->mbox_fd;
+	if (fstat(fd, &st) < 0) {
+		mbox_set_syscall_error(ctx->mbox, "fstat()");
+		return -1;
+	}
+
+	ctx->orig_atime = st.st_atime;
+
+	*offset = (uoff_t)st.st_size;
+	if (st.st_size == 0)
+		return 0;
+
+	if (lseek(fd, st.st_size-1, SEEK_SET) < 0) {
+                mbox_set_syscall_error(ctx->mbox, "lseek()");
+		return -1;
+	}
+
+	if (read(fd, &ch, 1) != 1) {
+		mbox_set_syscall_error(ctx->mbox, "read()");
+		return -1;
+	}
+
+	if (ch != '\n') {
+		if (write_full(fd, "\n", 1) < 0) {
+			mbox_set_syscall_error(ctx->mbox, "write()");
+			return -1;
+		}
+		*offset += 1;
+	}
+
+	return 0;
+}
+
+static int mbox_append_lf(struct mbox_save_context *ctx)
+{
+	if (o_stream_send(ctx->output, "\n", 1) < 0) {
+		write_stream_error(ctx);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int write_from_line(struct mbox_save_context *ctx, time_t received_date,
+			   const char *from_envelope)
+{
+	int ret;
+
+	T_BEGIN {
+		const char *line;
+
+		if (from_envelope == NULL) {
+			struct mail_storage *storage =
+				&ctx->mbox->storage->storage;
+
+			from_envelope =
+				strchr(storage->user->username, '@') != NULL ?
+				storage->user->username :
+				t_strconcat(storage->user->username,
+					    "@", my_hostdomain(), NULL);
+		} else if (*from_envelope == '\0') {
+			/* can't write empty envelope */
+			from_envelope = "MAILER-DAEMON";
+		}
+
+		/* save in local timezone, no matter what it was given with */
+		line = mbox_from_create(from_envelope, received_date);
+
+		if ((ret = o_stream_send_str(ctx->output, line)) < 0)
+			write_stream_error(ctx);
+	} T_END;
+	return ret;
+}
+
+static int mbox_write_content_length(struct mbox_save_context *ctx)
+{
+	uoff_t end_offset;
+	const char *str;
+	size_t len;
+
+	i_assert(ctx->eoh_offset != UOFF_T_MAX);
+
+	if (ctx->mbox->mbox_writeonly) {
+		/* we can't seek, don't set Content-Length */
+		return 0;
+	}
+
+	end_offset = ctx->output->offset;
+
+	/* write Content-Length headers */
+	str = t_strdup_printf("\nContent-Length: %s",
+			      dec2str(end_offset - ctx->eoh_offset));
+	len = strlen(str);
+
+	/* flush manually here so that we don't confuse seek() errors with
+	   buffer flushing errors */
+	if (o_stream_flush(ctx->output) < 0) {
+		write_stream_error(ctx);
+		return -1;
+	}
+	if (o_stream_seek(ctx->output, ctx->extra_hdr_offset +
+			  ctx->space_end_idx - len) < 0) {
+		lseek_stream_error(ctx);
+		return -1;
+	}
+
+	if (o_stream_send(ctx->output, str, len) < 0 ||
+	    o_stream_flush(ctx->output) < 0) {
+		write_stream_error(ctx);
+		return -1;
+	}
+
+	if (o_stream_seek(ctx->output, end_offset) < 0) {
+		lseek_stream_error(ctx);
+		return -1;
+	}
+	return 0;
+}
+
+static void mbox_save_init_sync(struct mailbox_transaction_context *t)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(t->box);
+	struct mbox_save_context *ctx = MBOX_SAVECTX(t->save_ctx);
+	const struct mail_index_header *hdr;
+	struct mail_index_view *view;
+
+	/* open a new view to get the header. this is required if we just
+	   synced the mailbox so we can get updated next_uid. */
+	mail_index_refresh(mbox->box.index);
+	view = mail_index_view_open(mbox->box.index);
+	hdr = mail_index_get_header(view);
+
+	ctx->next_uid = hdr->next_uid;
+	ctx->uid_validity = hdr->uid_validity;
+	ctx->synced = TRUE;
+
+	mail_index_view_close(&view);
+}
+
+static void status_flags_append(string_t *str, enum mail_flags flags,
+				const struct mbox_flag_type *flags_list)
+{
+	int i;
+
+	flags ^= MBOX_NONRECENT_KLUDGE;
+	for (i = 0; flags_list[i].chr != 0; i++) {
+		if ((flags & flags_list[i].flag) != 0)
+			str_append_c(str, flags_list[i].chr);
+	}
+}
+
+static void mbox_save_append_flag_headers(string_t *str, enum mail_flags flags)
+{
+	/* write the Status: header always. It always gets added soon anyway. */
+	str_append(str, "Status: ");
+	status_flags_append(str, flags, mbox_status_flags);
+	str_append_c(str, '\n');
+
+	if ((flags & XSTATUS_FLAGS_MASK) != 0) {
+		str_append(str, "X-Status: ");
+		status_flags_append(str, flags, mbox_xstatus_flags);
+		str_append_c(str, '\n');
+	}
+}
+
+static void
+mbox_save_append_keyword_headers(struct mbox_save_context *ctx,
+				 struct mail_keywords *keywords)
+{
+	unsigned char space[MBOX_HEADER_PADDING+1 +
+			    sizeof("Content-Length: \n")-1 + MAX_INT_STRLEN];
+	const ARRAY_TYPE(keywords) *keyword_names_list;
+	const char *const *keyword_names;
+	unsigned int i, count, keyword_names_count;
+
+	keyword_names_list = mail_index_get_keywords(ctx->mbox->box.index);
+	keyword_names = array_get(keyword_names_list, &keyword_names_count);
+
+	str_append(ctx->headers, "X-Keywords:");
+	count = keywords == NULL ? 0 : keywords->count;
+	for (i = 0; i < count; i++) {
+		i_assert(keywords->idx[i] < keyword_names_count);
+
+		str_append_c(ctx->headers, ' ');
+		str_append(ctx->headers, keyword_names[keywords->idx[i]]);
+	}
+
+	memset(space, ' ', sizeof(space));
+	str_append_data(ctx->headers, space, sizeof(space));
+	ctx->space_end_idx = str_len(ctx->headers);
+	str_append_c(ctx->headers, '\n');
+}
+
+static int
+mbox_save_init_file(struct mbox_save_context *ctx,
+		    struct mbox_transaction_context *t)
+{
+	struct mailbox_transaction_context *_t = &t->t;
+	struct mbox_mailbox *mbox = ctx->mbox;
+	struct mail_storage *storage = &mbox->storage->storage;
+	int ret;
+
+	if (mbox_is_backend_readonly(ctx->mbox)) {
+		mail_storage_set_error(storage, MAIL_ERROR_PERM,
+				       "Read-only mbox");
+		return -1;
+	}
+
+	if (ctx->append_offset == UOFF_T_MAX) {
+		/* first appended mail in this transaction */
+		if (t->write_lock_id == 0) {
+			if (mbox_lock(mbox, F_WRLCK, &t->write_lock_id) <= 0)
+				return -1;
+		}
+
+		if (mbox->mbox_fd == -1) {
+			if (mbox_file_open(mbox) < 0)
+				return -1;
+		}
+
+		/* update mbox_sync_dirty state */
+		ret = mbox_sync_has_changed(mbox, TRUE);
+		if (ret < 0)
+			return -1;
+	}
+
+	if (!ctx->synced) {
+		/* we'll need to assign UID for the mail immediately. */
+		if (mbox_sync(mbox, 0) < 0)
+			return -1;
+		mbox_save_init_sync(_t);
+	}
+
+	/* the syncing above could have changed the append offset */
+	if (ctx->append_offset == UOFF_T_MAX) {
+		if (mbox_seek_to_end(ctx, &ctx->append_offset) < 0)
+			return -1;
+
+		i_assert(mbox->mbox_fd != -1);
+		ctx->output = o_stream_create_fd_file(mbox->mbox_fd,
+						      ctx->append_offset,
+						      FALSE);
+		o_stream_cork(ctx->output);
+	}
+	return 0;
+}
+
+static void
+save_header_callback(struct header_filter_istream *input ATTR_UNUSED,
+		     struct message_header_line *hdr,
+		     bool *matched, struct mbox_save_context *ctx)
+{
+	if (hdr != NULL) {
+		if (str_begins(hdr->name, "From ")) {
+			/* we can't allow From_-lines in headers. there's no
+			   legitimate reason for allowing them in any case,
+			   so just drop them. */
+			*matched = TRUE;
+			return;
+		}
+
+		if (!*matched && ctx->mbox_md5_ctx != NULL)
+			ctx->mbox->md5_v.more(ctx->mbox_md5_ctx, hdr);
+	}
+}
+
+static void mbox_save_x_delivery_id(struct mbox_save_context *ctx)
+{
+	unsigned char md5_result[MD5_RESULTLEN];
+	buffer_t *buf;
+	string_t *str;
+	void *randbuf;
+
+	buf = t_buffer_create(256);
+	buffer_append(buf, &ioloop_time, sizeof(ioloop_time));
+	buffer_append(buf, &ioloop_timeval.tv_usec,
+		      sizeof(ioloop_timeval.tv_usec));
+
+	randbuf = buffer_append_space_unsafe(buf, MBOX_DELIVERY_ID_RAND_BYTES);
+	random_fill(randbuf, MBOX_DELIVERY_ID_RAND_BYTES);
+
+	md5_get_digest(buf->data, buf->used, md5_result);
+
+	str = t_str_new(128);
+	str_append(str, "X-Delivery-ID: ");
+	base64_encode(md5_result, sizeof(md5_result), str);
+	str_append_c(str, '\n');
+
+	ctx->x_delivery_id_header = i_strdup(str_c(str));
+}
+
+static struct istream *
+mbox_save_get_input_stream(struct mbox_save_context *ctx, struct istream *input)
+{
+	struct istream *filter, *ret, *cache_input, *streams[3];
+
+	/* filter out unwanted headers and keep track of headers' MD5 sum */
+	filter = i_stream_create_header_filter(input, HEADER_FILTER_EXCLUDE |
+					       HEADER_FILTER_NO_CR |
+					       HEADER_FILTER_ADD_MISSING_EOH |
+					       HEADER_FILTER_END_BODY_WITH_LF,
+					       mbox_save_drop_headers,
+					       mbox_save_drop_headers_count,
+					       save_header_callback, ctx);
+
+	if ((ctx->mbox->storage->storage.flags &
+	     MAIL_STORAGE_FLAG_KEEP_HEADER_MD5) != 0) {
+		/* we're using MD5 sums to generate POP3 UIDLs.
+		   clients don't like it much if there are duplicates,
+		   so make sure that there can't be any by appending
+		   our own X-Delivery-ID header. */
+		const char *hdr;
+
+		T_BEGIN {
+			mbox_save_x_delivery_id(ctx);
+		} T_END;
+		hdr = ctx->x_delivery_id_header;
+
+		streams[0] = i_stream_create_from_data(hdr, strlen(hdr));
+		streams[1] = filter;
+		streams[2] = NULL;
+		ret = i_stream_create_concat(streams);
+		i_stream_unref(&filter);
+		filter = ret;
+	}
+
+	/* convert linefeeds to wanted format */
+	ret = ctx->mbox->storage->storage.set->mail_save_crlf ?
+		i_stream_create_crlf(filter) : i_stream_create_lf(filter);
+	i_stream_unref(&filter);
+
+	/* caching creates a tee stream */
+	cache_input = index_mail_cache_parse_init(ctx->ctx.dest_mail, ret);
+	i_stream_unref(&ret);
+	ret = cache_input;
+	return ret;
+}
+
+struct mail_save_context *
+mbox_save_alloc(struct mailbox_transaction_context *t)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(t->box);
+	struct mbox_save_context *ctx;
+
+	i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
+
+	if (t->save_ctx == NULL) {
+		ctx = i_new(struct mbox_save_context, 1);
+		ctx->ctx.transaction = t;
+		ctx->mbox = mbox;
+		ctx->trans = t->itrans;
+		ctx->append_offset = UOFF_T_MAX;
+		ctx->headers = str_new(default_pool, 512);
+		ctx->mail_offset = UOFF_T_MAX;
+		t->save_ctx = &ctx->ctx;
+	}
+	return t->save_ctx;
+}
+
+int mbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
+{
+	struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx);
+	struct mail_save_data *mdata = &_ctx->data;
+	struct mbox_transaction_context *t = MBOX_TRANSCTX(_ctx->transaction);
+	enum mail_flags save_flags;
+	uint64_t offset;
+
+	/* FIXME: we could write timezone_offset to From-line.. */
+	if (mdata->received_date == (time_t)-1)
+		mdata->received_date = ioloop_time;
+
+	ctx->failed = FALSE;
+	ctx->seq = 0;
+
+	if (mbox_save_init_file(ctx, t) < 0) {
+		ctx->failed = TRUE;
+		return -1;
+	}
+
+	save_flags = mdata->flags;
+	if (mdata->uid == 0)
+		save_flags |= MAIL_RECENT;
+	str_truncate(ctx->headers, 0);
+	if (ctx->synced) {
+		if (ctx->mbox->mbox_save_md5)
+			ctx->mbox_md5_ctx = ctx->mbox->md5_v.init();
+		if (ctx->next_uid < mdata->uid) {
+			/* we can use the wanted UID */
+			ctx->next_uid = mdata->uid;
+		}
+		if (ctx->output->offset == 0) {
+			/* writing the first mail. Insert X-IMAPbase as well. */
+			str_printfa(ctx->headers, "X-IMAPbase: %u %010u\n",
+				    ctx->uid_validity, ctx->next_uid);
+		}
+		str_printfa(ctx->headers, "X-UID: %u\n", ctx->next_uid);
+
+		mail_index_append(ctx->trans, ctx->next_uid, &ctx->seq);
+		mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
+					save_flags & ENUM_NEGATE(MAIL_RECENT));
+		if (mdata->keywords != NULL) {
+			mail_index_update_keywords(ctx->trans, ctx->seq,
+						   MODIFY_REPLACE,
+						   mdata->keywords);
+		}
+		if (mdata->min_modseq != 0) {
+			mail_index_update_modseq(ctx->trans, ctx->seq,
+						 mdata->min_modseq);
+		}
+
+		offset = ctx->output->offset == 0 ? 0 :
+			ctx->output->offset - 1;
+		mail_index_update_ext(ctx->trans, ctx->seq,
+				      ctx->mbox->mbox_ext_idx, &offset, NULL);
+		ctx->next_uid++;
+
+		/* parse and cache the mail headers as we read it */
+		mail_set_seq_saving(_ctx->dest_mail, ctx->seq);
+	}
+	mbox_save_append_flag_headers(ctx->headers, save_flags);
+	mbox_save_append_keyword_headers(ctx, mdata->keywords);
+	str_append_c(ctx->headers, '\n');
+
+	i_assert(ctx->mbox->mbox_lock_type == F_WRLCK);
+
+	ctx->mail_offset = ctx->output->offset;
+	ctx->eoh_offset = UOFF_T_MAX;
+	ctx->last_char = '\n';
+
+	if (write_from_line(ctx, mdata->received_date, mdata->from_envelope) < 0)
+		ctx->failed = TRUE;
+	else
+		ctx->input = mbox_save_get_input_stream(ctx, input);
+	return ctx->failed ? -1 : 0;
+}
+
+static int mbox_save_body_input(struct mbox_save_context *ctx)
+{
+	const unsigned char *data;
+	size_t size;
+
+	data = i_stream_get_data(ctx->input, &size);
+	if (size > 0) {
+		if (o_stream_send(ctx->output, data, size) < 0) {
+			write_stream_error(ctx);
+			return -1;
+		}
+		ctx->last_char = data[size-1];
+		i_stream_skip(ctx->input, size);
+	}
+	return 0;
+}
+
+static int mbox_save_body(struct mbox_save_context *ctx)
+{
+	ssize_t ret;
+
+	while ((ret = i_stream_read(ctx->input)) != -1) {
+		if (mbox_save_body_input(ctx) < 0)
+			return -1;
+		/* i_stream_read() may have returned 0 at EOF
+		   because of this parser */
+		index_mail_cache_parse_continue(ctx->ctx.dest_mail);
+		if (ret == 0)
+			return 0;
+	}
+
+	i_assert(ctx->last_char == '\n');
+	return 0;
+}
+
+static int mbox_save_finish_headers(struct mbox_save_context *ctx)
+{
+	i_assert(ctx->eoh_offset == UOFF_T_MAX);
+
+	/* append our own headers and ending empty line */
+	ctx->extra_hdr_offset = ctx->output->offset;
+	if (o_stream_send(ctx->output, str_data(ctx->headers),
+			  str_len(ctx->headers)) < 0) {
+		write_stream_error(ctx);
+		return -1;
+	}
+	ctx->eoh_offset = ctx->output->offset;
+	return 0;
+}
+
+int mbox_save_continue(struct mail_save_context *_ctx)
+{
+	struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx);
+	const unsigned char *data;
+	size_t i, size;
+	ssize_t ret;
+
+	if (ctx->failed)
+		return -1;
+
+	if (ctx->eoh_offset != UOFF_T_MAX) {
+		/* writing body */
+		return mbox_save_body(ctx);
+	}
+
+	while ((ret = i_stream_read_more(ctx->input, &data, &size)) > 0) {
+		for (i = 0; i < size; i++) {
+			if (data[i] == '\n' &&
+			    ((i == 0 && ctx->last_char == '\n') ||
+			     (i > 0 && data[i-1] == '\n'))) {
+				/* end of headers. we don't need to worry about
+				   CRs because they're dropped */
+				break;
+			}
+		}
+		if (i != size) {
+			/* found end of headers. write the rest of them
+			   (not including the finishing empty line) */
+			if (o_stream_send(ctx->output, data, i) < 0) {
+				write_stream_error(ctx);
+				return -1;
+			}
+			ctx->last_char = '\n';
+			i_stream_skip(ctx->input, i + 1);
+			break;
+		}
+
+		if (o_stream_send(ctx->output, data, size) < 0) {
+			write_stream_error(ctx);
+			return -1;
+		}
+		i_assert(size > 0);
+		ctx->last_char = data[size-1];
+		i_stream_skip(ctx->input, size);
+		index_mail_cache_parse_continue(ctx->ctx.dest_mail);
+	}
+	if (ret == 0)
+		return 0;
+	if (ctx->input->stream_errno != 0) {
+		i_error("read(%s) failed: %s", i_stream_get_name(ctx->input),
+			i_stream_get_error(ctx->input));
+		ctx->failed = TRUE;
+		return -1;
+	}
+
+	i_assert(ctx->last_char == '\n');
+
+	if (ctx->mbox_md5_ctx != NULL) {
+		unsigned char hdr_md5_sum[16];
+
+		if (ctx->x_delivery_id_header != NULL) {
+			struct message_header_line hdr;
+
+			i_zero(&hdr);
+			hdr.name = ctx->x_delivery_id_header;
+			hdr.name_len = sizeof("X-Delivery-ID")-1;
+			hdr.middle = (const unsigned char *)hdr.name +
+				hdr.name_len;
+			hdr.middle_len = 2;
+			hdr.value = hdr.full_value =
+				hdr.middle + hdr.middle_len;
+			hdr.value_len = strlen((const char *)hdr.value);
+			ctx->mbox->md5_v.more(ctx->mbox_md5_ctx, &hdr);
+		}
+		ctx->mbox->md5_v.finish(ctx->mbox_md5_ctx, hdr_md5_sum);
+		mail_index_update_ext(ctx->trans, ctx->seq,
+				      ctx->mbox->md5hdr_ext_idx,
+				      hdr_md5_sum, NULL);
+	}
+
+	if (mbox_save_finish_headers(ctx) < 0)
+		return -1;
+
+	/* write body */
+	if (mbox_save_body_input(ctx) < 0)
+		return -1;
+	return ctx->input->eof ? 0 : mbox_save_body(ctx);
+}
+
+int mbox_save_finish(struct mail_save_context *_ctx)
+{
+	struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx);
+
+	if (!ctx->failed && ctx->eoh_offset == UOFF_T_MAX)
+		(void)mbox_save_finish_headers(ctx);
+
+	if (ctx->output != NULL) {
+		/* make sure everything is written */
+		if (o_stream_flush(ctx->output) < 0)
+			write_stream_error(ctx);
+	}
+
+	ctx->finished = TRUE;
+	if (!ctx->failed) {
+		i_assert(ctx->output != NULL);
+		T_BEGIN {
+			if (mbox_write_content_length(ctx) < 0 ||
+			    mbox_append_lf(ctx) < 0)
+				ctx->failed = TRUE;
+		} T_END;
+	}
+
+	index_mail_cache_parse_deinit(ctx->ctx.dest_mail,
+				      ctx->ctx.data.received_date,
+				      !ctx->failed);
+	if (ctx->input != NULL)
+		i_stream_destroy(&ctx->input);
+
+	if (ctx->failed && ctx->mail_offset != UOFF_T_MAX) {
+		/* saving this mail failed - truncate back to beginning of it */
+		i_assert(ctx->output != NULL);
+		(void)o_stream_flush(ctx->output);
+		if (ftruncate(ctx->mbox->mbox_fd, (off_t)ctx->mail_offset) < 0)
+			mbox_set_syscall_error(ctx->mbox, "ftruncate()");
+		(void)o_stream_seek(ctx->output, ctx->mail_offset);
+		ctx->mail_offset = UOFF_T_MAX;
+	}
+
+	if (ctx->seq != 0 && ctx->failed) {
+		index_storage_save_abort_last(&ctx->ctx, ctx->seq);
+	}
+	index_save_context_free(_ctx);
+	return ctx->failed ? -1 : 0;
+}
+
+void mbox_save_cancel(struct mail_save_context *_ctx)
+{
+	struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx);
+
+	ctx->failed = TRUE;
+	(void)mbox_save_finish(_ctx);
+}
+
+static void mbox_transaction_save_deinit(struct mbox_save_context *ctx)
+{
+	o_stream_destroy(&ctx->output);
+	str_free(&ctx->headers);
+}
+
+static void mbox_save_truncate(struct mbox_save_context *ctx)
+{
+	if (ctx->append_offset == UOFF_T_MAX || ctx->mbox->mbox_fd == -1)
+		return;
+
+	i_assert(ctx->mbox->mbox_lock_type == F_WRLCK);
+
+	/* failed, truncate file back to original size. output stream needs to
+	   be flushed before truncating so unref() won't write anything. */
+	if (ctx->output != NULL)
+		(void)o_stream_flush(ctx->output);
+
+	if (ftruncate(ctx->mbox->mbox_fd, (off_t)ctx->append_offset) < 0)
+		mbox_set_syscall_error(ctx->mbox, "ftruncate()");
+}
+
+int mbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
+{
+	struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx);
+	struct mailbox_transaction_context *_t = _ctx->transaction;
+	struct mbox_mailbox *mbox = ctx->mbox;
+	struct stat st;
+	int ret = 0;
+
+	i_assert(ctx->finished);
+	i_assert(mbox->mbox_fd != -1);
+
+	if (fstat(mbox->mbox_fd, &st) < 0) {
+		mbox_set_syscall_error(mbox, "fstat()");
+		ret = -1;
+	}
+
+	if (ctx->synced) {
+		_t->changes->uid_validity = ctx->uid_validity;
+		mail_index_append_finish_uids(ctx->trans, 0,
+					      &_t->changes->saved_uids);
+
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, next_uid),
+			&ctx->next_uid, sizeof(ctx->next_uid), FALSE);
+
+		if (ret == 0) {
+			mbox->mbox_hdr.sync_mtime = st.st_mtime;
+			mbox->mbox_hdr.sync_size = st.st_size;
+			mail_index_update_header_ext(ctx->trans,
+						     mbox->mbox_ext_idx,
+						     0, &mbox->mbox_hdr,
+						     sizeof(mbox->mbox_hdr));
+		}
+	}
+
+	if (ret == 0 && ctx->orig_atime != st.st_atime) {
+		/* try to set atime back to its original value.
+		   (it'll fail with EPERM for shared mailboxes where we aren't
+		   the file's owner) */
+		struct utimbuf buf;
+
+		buf.modtime = st.st_mtime;
+		buf.actime = ctx->orig_atime;
+		if (utime(mailbox_get_path(&mbox->box), &buf) < 0 &&
+		    errno != EPERM)
+			mbox_set_syscall_error(mbox, "utime()");
+	}
+
+	if (ctx->output != NULL) {
+		/* flush the final LF */
+		if (o_stream_flush(ctx->output) < 0)
+			write_stream_error(ctx);
+	}
+	if (mbox->mbox_fd != -1 && !mbox->mbox_writeonly &&
+	    mbox->storage->storage.set->parsed_fsync_mode != FSYNC_MODE_NEVER) {
+		if (fdatasync(mbox->mbox_fd) < 0) {
+			mbox_set_syscall_error(mbox, "fdatasync()");
+			mbox_save_truncate(ctx);
+			ret = -1;
+		}
+	}
+
+	mbox_transaction_save_deinit(ctx);
+	if (ret < 0)
+		i_free(ctx);
+	return ret;
+}
+
+void mbox_transaction_save_commit_post(struct mail_save_context *_ctx,
+				       struct mail_index_transaction_commit_result *result ATTR_UNUSED)
+{
+	struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx);
+
+	i_assert(ctx->mbox->mbox_lock_type == F_WRLCK);
+
+	if (ctx->synced) {
+		/* after saving mails with UIDs we need to update
+		   the last-uid */
+		(void)mbox_sync(ctx->mbox, MBOX_SYNC_HEADER |
+				MBOX_SYNC_REWRITE);
+	}
+	i_free(ctx);
+}
+
+void mbox_transaction_save_rollback(struct mail_save_context *_ctx)
+{
+	struct mbox_save_context *ctx = MBOX_SAVECTX(_ctx);
+
+	if (!ctx->finished)
+		mbox_save_cancel(&ctx->ctx);
+
+	mbox_save_truncate(ctx);
+	mbox_transaction_save_deinit(ctx);
+	i_free(ctx);
+}
diff --git a/src/lib-storage/index/mbox/mbox-settings.c b/src/lib-storage/index/mbox/mbox-settings.c
new file mode 100644
index 0000000..1df2452
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-settings.c
@@ -0,0 +1,55 @@
+/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "settings-parser.h"
+#include "mail-storage-settings.h"
+#include "mbox-settings.h"
+
+#include <stddef.h>
+
+#undef DEF
+#define DEF(type, name) \
+	SETTING_DEFINE_STRUCT_##type(#name, name, struct mbox_settings)
+
+static const struct setting_define mbox_setting_defines[] = {
+	DEF(STR, mbox_read_locks),
+	DEF(STR, mbox_write_locks),
+	DEF(TIME, mbox_lock_timeout),
+	DEF(TIME, mbox_dotlock_change_timeout),
+	DEF(SIZE, mbox_min_index_size),
+	DEF(BOOL, mbox_dirty_syncs),
+	DEF(BOOL, mbox_very_dirty_syncs),
+	DEF(BOOL, mbox_lazy_writes),
+	DEF(ENUM, mbox_md5),
+
+	SETTING_DEFINE_LIST_END
+};
+
+static const struct mbox_settings mbox_default_settings = {
+	.mbox_read_locks = "fcntl",
+	.mbox_write_locks = "dotlock fcntl",
+	.mbox_lock_timeout = 5*60,
+	.mbox_dotlock_change_timeout = 2*60,
+	.mbox_min_index_size = 0,
+	.mbox_dirty_syncs = TRUE,
+	.mbox_very_dirty_syncs = FALSE,
+	.mbox_lazy_writes = TRUE,
+	.mbox_md5 = "apop3d:all"
+};
+
+static const struct setting_parser_info mbox_setting_parser_info = {
+	.module_name = "mbox",
+	.defines = mbox_setting_defines,
+	.defaults = &mbox_default_settings,
+
+	.type_offset = SIZE_MAX,
+	.struct_size = sizeof(struct mbox_settings),
+
+	.parent_offset = SIZE_MAX,
+	.parent = &mail_user_setting_parser_info
+};
+
+const struct setting_parser_info *mbox_get_setting_parser_info(void)
+{
+	return &mbox_setting_parser_info;
+}
diff --git a/src/lib-storage/index/mbox/mbox-settings.h b/src/lib-storage/index/mbox/mbox-settings.h
new file mode 100644
index 0000000..eb99d82
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-settings.h
@@ -0,0 +1,18 @@
+#ifndef MBOX_SETTINGS_H
+#define MBOX_SETTINGS_H
+
+struct mbox_settings {
+	const char *mbox_read_locks;
+	const char *mbox_write_locks;
+	unsigned int mbox_lock_timeout;
+	unsigned int mbox_dotlock_change_timeout;
+	uoff_t mbox_min_index_size;
+	bool mbox_dirty_syncs;
+	bool mbox_very_dirty_syncs;
+	bool mbox_lazy_writes;
+	const char *mbox_md5;
+};
+
+const struct setting_parser_info *mbox_get_setting_parser_info(void);
+
+#endif
diff --git a/src/lib-storage/index/mbox/mbox-storage.c b/src/lib-storage/index/mbox/mbox-storage.c
new file mode 100644
index 0000000..4b2bb35
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-storage.c
@@ -0,0 +1,911 @@
+/* Copyright (c) 2002-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "restrict-access.h"
+#include "master-service.h"
+#include "mailbox-list-private.h"
+#include "mbox-storage.h"
+#include "mbox-lock.h"
+#include "mbox-file.h"
+#include "mbox-sync-private.h"
+#include "istream-raw-mbox.h"
+#include "mail-copy.h"
+#include "index-mail.h"
+
+#include <sys/stat.h>
+
+/* How often to touch the dotlock file when using KEEP_LOCKED flag */
+#define MBOX_LOCK_TOUCH_MSECS (10*1000)
+
+/* Assume that if atime < mtime, there are new mails. If it's good enough for
+   UW-IMAP, it's good enough for us. */
+#define STAT_GET_MARKED(st) \
+	((st).st_size == 0 ? MAILBOX_UNMARKED : \
+	 (st).st_atime < (st).st_mtime ? MAILBOX_MARKED : MAILBOX_UNMARKED)
+
+#define MBOX_LIST_CONTEXT(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, mbox_mailbox_list_module)
+
+struct mbox_mailbox_list {
+	union mailbox_list_module_context module_ctx;
+	const struct mbox_settings *set;
+};
+
+/* NOTE: must be sorted for istream-header-filter. Note that it's not such
+   a good idea to change this list, as the messages will then change from
+   client's point of view. So if you do it, change all mailboxes' UIDVALIDITY
+   so all caches are reset. */
+const char *mbox_hide_headers[] = {
+	"Content-Length",
+	"Status",
+	"X-IMAP",
+	"X-IMAPbase",
+	"X-Keywords",
+	"X-Status",
+	"X-UID"
+};
+unsigned int mbox_hide_headers_count = N_ELEMENTS(mbox_hide_headers);
+
+/* A bit ugly duplification of the above list. It's safe to modify this list
+   without bad side effects, just keep the list sorted. */
+const char *mbox_save_drop_headers[] = {
+	"Content-Length",
+	"Status",
+	"X-Delivery-ID",
+	"X-IMAP",
+	"X-IMAPbase",
+	"X-Keywords",
+	"X-Status",
+	"X-UID"
+};
+unsigned int mbox_save_drop_headers_count = N_ELEMENTS(mbox_save_drop_headers);
+
+extern struct mail_storage mbox_storage;
+extern struct mailbox mbox_mailbox;
+
+static struct event_category event_category_mbox = {
+	.name = "mbox",
+	.parent = &event_category_storage,
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(mbox_mailbox_list_module,
+				  &mailbox_list_module_register);
+
+static void
+mbox_set_syscall_error_str(struct mbox_mailbox *mbox, const char *function,
+			   const char *error)
+{
+	i_assert(function != NULL);
+
+	if (ENOQUOTA(errno)) {
+		mail_storage_set_error(&mbox->storage->storage,
+			MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA);
+	} else {
+		const char *toobig_error = errno != EFBIG ? "" :
+			" (process was started with ulimit -f limit)";
+		mailbox_set_critical(&mbox->box, "%s failed with mbox: %s%s",
+				     function, error, toobig_error);
+	}
+}
+
+void mbox_set_syscall_error(struct mbox_mailbox *mbox, const char *function)
+{
+	mbox_set_syscall_error_str(mbox, function, strerror(errno));
+}
+
+void mbox_istream_set_syscall_error(struct mbox_mailbox *mbox,
+				    struct istream *input,
+				    const char *function)
+{
+	errno = input->stream_errno;
+	mbox_set_syscall_error_str(mbox, function, i_stream_get_error(input));
+}
+
+void mbox_ostream_set_syscall_error(struct mbox_mailbox *mbox,
+				    struct ostream *output,
+				    const char *function)
+{
+	errno = output->stream_errno;
+	mbox_set_syscall_error_str(mbox, function, o_stream_get_error(output));
+}
+
+static int
+mbox_list_get_path(struct mailbox_list *list, const char *name,
+		   enum mailbox_list_path_type type, const char **path_r)
+{
+	struct mbox_mailbox_list *mlist = MBOX_LIST_CONTEXT(list);
+	const char *path, *p;
+	int ret;
+
+	*path_r = NULL;
+
+	ret = mlist->module_ctx.super.get_path(list, name, type, &path);
+	if (ret <= 0)
+		return ret;
+
+	switch (type) {
+	case MAILBOX_LIST_PATH_TYPE_CONTROL:
+	case MAILBOX_LIST_PATH_TYPE_INDEX:
+	case MAILBOX_LIST_PATH_TYPE_INDEX_CACHE:
+	case MAILBOX_LIST_PATH_TYPE_LIST_INDEX:
+		if (name == NULL && type == MAILBOX_LIST_PATH_TYPE_CONTROL &&
+		    list->set.control_dir != NULL) {
+			/* kind of a kludge for backwards compatibility:
+			   the subscriptions file is in the root control_dir
+			   without .imap/ suffix */
+			*path_r = path;
+			return 1;
+		}
+		if (name == NULL) {
+			*path_r = t_strconcat(path, "/"MBOX_INDEX_DIR_NAME, NULL);
+			return 1;
+		}
+
+		p = strrchr(path, '/');
+		if (p == NULL)
+			return 0;
+
+		*path_r = t_strconcat(t_strdup_until(path, p),
+				      "/"MBOX_INDEX_DIR_NAME"/", p+1, NULL);
+		break;
+	case MAILBOX_LIST_PATH_TYPE_DIR:
+	case MAILBOX_LIST_PATH_TYPE_ALT_DIR:
+	case MAILBOX_LIST_PATH_TYPE_MAILBOX:
+	case MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX:
+	case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE:
+		*path_r = path;
+		break;
+	}
+	return 1;
+}
+
+static struct mail_storage *mbox_storage_alloc(void)
+{
+	struct mbox_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mbox storage", 512+256);
+	storage = p_new(pool, struct mbox_storage, 1);
+	storage->storage = mbox_storage;
+	storage->storage.pool = pool;
+	return &storage->storage;
+}
+
+static int
+mbox_storage_create(struct mail_storage *_storage, struct mail_namespace *ns,
+		    const char **error_r)
+{
+	struct mbox_storage *storage = MBOX_STORAGE(_storage);
+	struct stat st;
+	const char *dir;
+
+	if (master_service_get_client_limit(master_service) > 1) {
+		/* we can't handle locking related problems. */
+		*error_r = "mbox requires client_limit=1 for service";
+		return -1;
+	}
+
+	storage->set = mail_namespace_get_driver_settings(ns, _storage);
+
+	if (mailbox_list_get_root_path(ns->list, MAILBOX_LIST_PATH_TYPE_INDEX, &dir)) {
+		_storage->temp_path_prefix = p_strconcat(_storage->pool, dir,
+			"/", mailbox_list_get_temp_prefix(ns->list), NULL);
+	}
+	if (stat(ns->list->set.root_dir, &st) == 0 && !S_ISDIR(st.st_mode)) {
+		*error_r = t_strdup_printf(
+			"mbox root directory can't be a file: %s "
+			"(http://wiki2.dovecot.org/MailLocation/Mbox)",
+			ns->list->set.root_dir);
+		return -1;
+	}
+	return 0;
+}
+
+static void mbox_storage_get_list_settings(const struct mail_namespace *ns,
+					   struct mailbox_list_settings *set)
+{
+	if (set->layout == NULL)
+		set->layout = MAILBOX_LIST_NAME_FS;
+	if (set->subscription_fname == NULL)
+		set->subscription_fname = MBOX_SUBSCRIPTION_FILE_NAME;
+
+	if (set->inbox_path == NULL &&
+	    strcasecmp(set->layout, MAILBOX_LIST_NAME_FS) == 0) {
+		set->inbox_path = t_strconcat(set->root_dir, "/inbox", NULL);
+		e_debug(ns->user->event, "mbox: INBOX defaulted to %s", set->inbox_path);
+	}
+}
+
+static bool mbox_is_file(const char *path, const char *name, bool debug)
+{
+	struct stat st;
+
+	if (stat(path, &st) < 0) {
+		if (debug) {
+			i_debug("mbox autodetect: %s: stat(%s) failed: %m",
+				name, path);
+		}
+		return FALSE;
+	}
+	if (S_ISDIR(st.st_mode)) {
+		if (debug) {
+			i_debug("mbox autodetect: %s: is a directory (%s)",
+			       name, path);
+		}
+		return FALSE;
+	}
+	if (access(path, R_OK|W_OK) < 0) {
+		if (debug) {
+			i_debug("mbox autodetect: %s: no R/W access (%s)",
+			       name, path);
+		}
+		return FALSE;
+	}
+
+	if (debug)
+		i_debug("mbox autodetect: %s: yes (%s)", name, path);
+	return TRUE;
+}
+
+static bool mbox_is_dir(const char *path, const char *name, bool debug)
+{
+	struct stat st;
+
+	if (stat(path, &st) < 0) {
+		if (debug) {
+			i_debug("mbox autodetect: %s: stat(%s) failed: %m",
+			       name, path);
+		}
+		return FALSE;
+	}
+	if (!S_ISDIR(st.st_mode)) {
+		if (debug) {
+			i_debug("mbox autodetect: %s: is not a directory (%s)",
+			       name, path);
+		}
+		return FALSE;
+	}
+	if (access(path, R_OK|W_OK|X_OK) < 0) {
+		if (debug) {
+			i_debug("mbox autodetect: %s: no R/W/X access (%s)",
+			       name, path);
+		}
+		return FALSE;
+	}
+
+	if (debug)
+		i_debug("mbox autodetect: %s: yes (%s)", name, path);
+	return TRUE;
+}
+
+static bool mbox_storage_is_root_dir(const char *dir, bool debug)
+{
+	if (mbox_is_dir(t_strconcat(dir, "/"MBOX_INDEX_DIR_NAME, NULL),
+			"has "MBOX_INDEX_DIR_NAME"/", debug))
+		return TRUE;
+	if (mbox_is_file(t_strconcat(dir, "/inbox", NULL), "has inbox", debug))
+		return TRUE;
+	if (mbox_is_file(t_strconcat(dir, "/mbox", NULL), "has mbox", debug))
+		return TRUE;
+	return FALSE;
+}
+
+static const char *mbox_storage_find_root_dir(const struct mail_namespace *ns)
+{
+	bool debug = ns->mail_set->mail_debug;
+	const char *home, *path;
+
+	if (ns->owner == NULL ||
+	    mail_user_get_home(ns->owner, &home) <= 0) {
+		if (debug)
+			i_debug("maildir: Home directory not set");
+		home = "";
+	}
+
+	path = t_strconcat(home, "/mail", NULL);
+	if (mbox_storage_is_root_dir(path, debug))
+		return path;
+
+	path = t_strconcat(home, "/Mail", NULL);
+	if (mbox_storage_is_root_dir(path, debug))
+		return path;
+	return NULL;
+}
+
+static const char *
+mbox_storage_find_inbox_file(const char *user, bool debug)
+{
+	const char *path;
+
+	path = t_strconcat("/var/mail/", user, NULL);
+	if (access(path, R_OK|W_OK) == 0) {
+		if (debug)
+			i_debug("mbox: INBOX exists (%s)", path);
+		return path;
+	}
+	if (debug)
+		i_debug("mbox: INBOX: access(%s, rw) failed: %m", path);
+
+	path = t_strconcat("/var/spool/mail/", user, NULL);
+	if (access(path, R_OK|W_OK) == 0) {
+		if (debug)
+			i_debug("mbox: INBOX exists (%s)", path);
+		return path;
+	}
+	if (debug)
+		i_debug("mbox: INBOX: access(%s, rw) failed: %m", path);
+	return NULL;
+}
+
+static bool mbox_storage_autodetect(const struct mail_namespace *ns,
+				    struct mailbox_list_settings *set)
+{
+	bool debug = ns->mail_set->mail_debug;
+	const char *root_dir, *inbox_path;
+
+	root_dir = set->root_dir;
+	inbox_path = set->inbox_path;
+
+	if (root_dir != NULL) {
+		if (inbox_path == NULL &&
+		    mbox_is_file(root_dir, "INBOX file", debug)) {
+			/* using location=<INBOX> */
+			inbox_path = root_dir;
+			root_dir = NULL;
+		} else if (!mbox_storage_is_root_dir(root_dir, debug))
+			return FALSE;
+	}
+	if (root_dir == NULL) {
+		root_dir = mbox_storage_find_root_dir(ns);
+		if (root_dir == NULL) {
+			if (debug)
+				i_debug("mbox: couldn't find root dir");
+			return FALSE;
+		}
+	}
+	if (inbox_path == NULL) {
+		inbox_path = mbox_storage_find_inbox_file(ns->user->username,
+							  debug);
+	}
+	set->root_dir = root_dir;
+	set->inbox_path = inbox_path;
+
+	mbox_storage_get_list_settings(ns, set);
+	return TRUE;
+}
+
+static bool want_memory_indexes(struct mbox_storage *storage, const char *path)
+{
+	struct stat st;
+
+	if (storage->set->mbox_min_index_size == 0)
+		return FALSE;
+
+	if (stat(path, &st) < 0) {
+		if (errno == ENOENT)
+			st.st_size = 0;
+		else {
+			mail_storage_set_critical(&storage->storage,
+						  "stat(%s) failed: %m", path);
+			return FALSE;
+		}
+	}
+	return (uoff_t)st.st_size < storage->set->mbox_min_index_size;
+}
+
+static struct mailbox *
+mbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
+		   const char *vname, enum mailbox_flags flags)
+{
+	struct mbox_mailbox *mbox;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mbox mailbox", 1024*3);
+	mbox = p_new(pool, struct mbox_mailbox, 1);
+	mbox->box = mbox_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.mail_vfuncs = &mbox_mail_vfuncs;
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
+
+	mbox->storage = MBOX_STORAGE(storage);
+	mbox->mbox_fd = -1;
+	mbox->mbox_lock_type = F_UNLCK;
+	mbox->mbox_list_index_ext_id = (uint32_t)-1;
+
+	if (strcmp(mbox->storage->set->mbox_md5, "apop3d") == 0)
+		mbox->md5_v = mbox_md5_apop3d;
+	else if (strcmp(mbox->storage->set->mbox_md5, "all") == 0)
+		mbox->md5_v = mbox_md5_all;
+	else {
+		i_fatal("Invalid mbox_md5 setting: %s",
+			mbox->storage->set->mbox_md5);
+	}
+
+	if ((storage->flags & MAIL_STORAGE_FLAG_KEEP_HEADER_MD5) != 0)
+		mbox->mbox_save_md5 = TRUE;
+	return &mbox->box;
+}
+
+static void mbox_lock_touch_timeout(struct mbox_mailbox *mbox)
+{
+	mbox_dotlock_touch(mbox);
+}
+
+static int
+mbox_mailbox_open_finish(struct mbox_mailbox *mbox, bool move_to_memory)
+{
+	if (index_storage_mailbox_open(&mbox->box, move_to_memory) < 0)
+		return -1;
+
+	mbox->mbox_ext_idx =
+		mail_index_ext_register(mbox->box.index, "mbox",
+					sizeof(mbox->mbox_hdr),
+					sizeof(uint64_t), sizeof(uint64_t));
+	mbox->md5hdr_ext_idx =
+		mail_index_ext_register(mbox->box.index, "header-md5",
+					0, 16, 1);
+	return 0;
+}
+
+static int mbox_mailbox_open_existing(struct mbox_mailbox *mbox)
+{
+	struct mailbox *box = &mbox->box;
+	const char *rootdir, *box_path = mailbox_get_path(box);
+	bool move_to_memory;
+
+	move_to_memory = want_memory_indexes(mbox->storage, box_path);
+
+	if (box->inbox_any || strcmp(box->name, "INBOX") == 0) {
+		/* if INBOX isn't under the root directory, it's probably in
+		   /var/mail and we want to allow privileged dotlocking */
+		rootdir = mailbox_list_get_root_forced(box->list,
+						       MAILBOX_LIST_PATH_TYPE_DIR);
+		if (!str_begins(box_path, rootdir))
+			mbox->mbox_privileged_locking = TRUE;
+	}
+	if ((box->flags & MAILBOX_FLAG_KEEP_LOCKED) != 0) {
+		if (mbox_lock(mbox, F_WRLCK, &mbox->mbox_global_lock_id) <= 0)
+			return -1;
+
+		if (mbox->mbox_dotlock != NULL) {
+			mbox->keep_lock_to =
+				timeout_add(MBOX_LOCK_TOUCH_MSECS,
+					    mbox_lock_touch_timeout, mbox);
+		}
+	}
+	return mbox_mailbox_open_finish(mbox, move_to_memory);
+}
+
+static bool mbox_storage_is_readonly(struct mailbox *box)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+
+	if (index_storage_is_readonly(box))
+		return TRUE;
+
+	if (mbox_is_backend_readonly(mbox)) {
+		/* return read-only only if there are no private flags
+		   (that are stored in index files) */
+		if (mailbox_get_private_flags_mask(box) == 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static int mbox_mailbox_open(struct mailbox *box)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	struct stat st;
+	int ret;
+
+	if (box->input != NULL) {
+		i_stream_ref(box->input);
+		mbox->mbox_file_stream = box->input;
+		mbox->backend_readonly = TRUE;
+		mbox->backend_readonly_set = TRUE;
+		mbox->no_mbox_file = TRUE;
+		return mbox_mailbox_open_finish(mbox, FALSE);
+	}
+
+	ret = stat(mailbox_get_path(box), &st);
+	if (ret == 0) {
+		if (!S_ISDIR(st.st_mode))
+			return mbox_mailbox_open_existing(mbox);
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+				       "Mailbox isn't selectable");
+		return -1;
+	} else if (ENOTFOUND(errno)) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+			T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname));
+		return -1;
+	} else if (mail_storage_set_error_from_errno(box->storage)) {
+		return -1;
+	} else {
+		mailbox_set_critical(box,
+			"stat(%s) failed: %m", mailbox_get_path(box));
+		return -1;
+	}
+}
+
+static int
+mbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	int ret = 0;
+
+	if (!box->opened) {
+		if (mailbox_open(box) < 0)
+			return -1;
+	}
+
+	if (update->uid_validity != 0 || update->min_next_uid != 0 ||
+	    !guid_128_is_empty(update->mailbox_guid)) {
+		mbox->sync_hdr_update = update;
+		ret = mbox_sync(mbox, MBOX_SYNC_HEADER | MBOX_SYNC_FORCE_SYNC |
+				MBOX_SYNC_REWRITE);
+		mbox->sync_hdr_update = NULL;
+	}
+	if (ret == 0)
+		ret = index_storage_mailbox_update(box, update);
+	return ret;
+}
+
+static int create_inbox(struct mailbox *box)
+{
+	const char *inbox_path;
+	int fd;
+
+	inbox_path = mailbox_get_path(box);
+
+	fd = open(inbox_path, O_RDWR | O_CREAT | O_EXCL, 0660);
+	if (fd == -1 && errno == EACCES) {
+		/* try again with increased privileges */
+		(void)restrict_access_use_priv_gid();
+		fd = open(inbox_path, O_RDWR | O_CREAT | O_EXCL, 0660);
+		restrict_access_drop_priv_gid();
+	}
+	if (fd != -1) {
+		i_close_fd(&fd);
+		return 0;
+	} else if (errno == EACCES) {
+		mailbox_set_critical(box, "%s",
+			mail_error_create_eacces_msg("open", inbox_path));
+		return -1;
+	} else if (errno == EEXIST) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS,
+				       "Mailbox already exists");
+		return -1;
+	} else {
+		mailbox_set_critical(box,
+			"open(%s, O_CREAT) failed: %m", inbox_path);
+		return -1;
+	}
+}
+
+static int
+mbox_mailbox_create(struct mailbox *box, const struct mailbox_update *update,
+		    bool directory)
+{
+	int fd, ret;
+
+	if ((ret = index_storage_mailbox_create(box, directory)) <= 0)
+		return ret;
+
+	if (box->inbox_any) {
+		if (create_inbox(box) < 0)
+			return -1;
+	} else {
+		/* create the mbox file */
+		ret = mailbox_create_fd(box, mailbox_get_path(box),
+					O_RDWR | O_CREAT | O_EXCL, &fd);
+		if (ret < 0)
+			return -1;
+		if (ret == 0) {
+			mail_storage_set_error(box->storage, MAIL_ERROR_EXISTS,
+					       "Mailbox already exists");
+			return -1;
+		}
+		i_close_fd(&fd);
+	}
+	return update == NULL ? 0 : mbox_mailbox_update(box, update);
+}
+
+static void mbox_mailbox_close(struct mailbox *box)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	const struct mail_index_header *hdr;
+	enum mbox_sync_flags sync_flags = 0;
+
+	if (mbox->mbox_stream != NULL &&
+	    istream_raw_mbox_is_corrupted(mbox->mbox_stream)) {
+		/* clear the corruption by forcing a full resync */
+		sync_flags |= MBOX_SYNC_UNDIRTY | MBOX_SYNC_FORCE_SYNC;
+	}
+
+	if (box->view != NULL) {
+		hdr = mail_index_get_header(box->view);
+		if ((hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0 &&
+		    !mbox_is_backend_readonly(mbox)) {
+			/* we've done changes to mbox which haven't been
+			   written yet. do it now. */
+			sync_flags |= MBOX_SYNC_REWRITE;
+		}
+	}
+	if (sync_flags != 0 && !mbox->invalid_mbox_file)
+		(void)mbox_sync(mbox, sync_flags);
+
+	if (mbox->mbox_global_lock_id != 0)
+		mbox_unlock(mbox, mbox->mbox_global_lock_id);
+	timeout_remove(&mbox->keep_lock_to);
+
+        mbox_file_close(mbox);
+	i_stream_destroy(&mbox->mbox_file_stream);
+
+	index_storage_mailbox_close(box);
+}
+
+static int
+mbox_mailbox_get_guid(struct mbox_mailbox *mbox, guid_128_t guid_r)
+{
+	const char *errstr;
+
+	if (mail_index_is_in_memory(mbox->box.index)) {
+		errstr = "Mailbox GUIDs are not permanent without index files";
+		if (mbox->storage->set->mbox_min_index_size != 0) {
+			errstr = t_strconcat(errstr,
+				" (mbox_min_index_size is non-zero)", NULL);
+		}
+		mail_storage_set_error(mbox->box.storage,
+				       MAIL_ERROR_NOTPOSSIBLE, errstr);
+		return -1;
+	}
+	if (mbox_sync_header_refresh(mbox) < 0)
+		return -1;
+
+	if (!guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
+		/* we have the GUID */
+	} else if (mbox_file_open(mbox) < 0)
+		return -1;
+	else if (mbox->backend_readonly) {
+		mail_storage_set_error(mbox->box.storage, MAIL_ERROR_PERM,
+			"Can't set mailbox GUID to a read-only mailbox");
+		return -1;
+	} else {
+		/* create another mailbox and sync  */
+		struct mailbox *box2;
+		struct mbox_mailbox *mbox2;
+		int ret;
+
+		i_assert(mbox->mbox_lock_type == F_UNLCK);
+		box2 = mailbox_alloc(mbox->box.list, mbox->box.vname, 0);
+		ret = mailbox_sync(box2, 0);
+		mbox2 = MBOX_MAILBOX(box2);
+		memcpy(guid_r, mbox2->mbox_hdr.mailbox_guid, GUID_128_SIZE);
+		mailbox_free(&box2);
+		return ret;
+	}
+	memcpy(guid_r, mbox->mbox_hdr.mailbox_guid, GUID_128_SIZE);
+	return 0;
+}
+
+static int
+mbox_mailbox_get_metadata(struct mailbox *box,
+			  enum mailbox_metadata_items items,
+			  struct mailbox_metadata *metadata_r)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+
+	if (index_mailbox_get_metadata(box, items, metadata_r) < 0)
+		return -1;
+	if ((items & MAILBOX_METADATA_GUID) != 0) {
+		if (mbox_mailbox_get_guid(mbox, metadata_r->guid) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static void mbox_notify_changes(struct mailbox *box)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+
+	if (box->notify_callback == NULL)
+		mailbox_watch_remove_all(box);
+	else if (!mbox->no_mbox_file)
+		mailbox_watch_add(box, mailbox_get_path(box));
+}
+
+static bool
+mbox_is_internal_name(struct mailbox_list *list ATTR_UNUSED,
+		      const char *name)
+{
+	size_t len;
+
+	/* don't allow *.lock files/dirs */
+	len = strlen(name);
+	if (len > 5 && strcmp(name+len-5, ".lock") == 0)
+		return TRUE;
+
+	return strcmp(name, MBOX_INDEX_DIR_NAME) == 0;
+}
+
+static void mbox_storage_add_list(struct mail_storage *storage,
+				  struct mailbox_list *list)
+{
+	struct mbox_mailbox_list *mlist;
+
+	mlist = p_new(list->pool, struct mbox_mailbox_list, 1);
+	mlist->module_ctx.super = list->v;
+	mlist->set = mail_namespace_get_driver_settings(list->ns, storage);
+
+	if (*list->set.maildir_name == '\0') {
+		/* have to use .imap/ directories */
+		list->v.get_path = mbox_list_get_path;
+	}
+	list->v.is_internal_name = mbox_is_internal_name;
+
+	MODULE_CONTEXT_SET(list, mbox_mailbox_list_module, mlist);
+}
+
+static struct mailbox_transaction_context *
+mbox_transaction_begin(struct mailbox *box,
+		       enum mailbox_transaction_flags flags,
+		       const char *reason)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	struct mbox_transaction_context *mt;
+
+	if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0)
+		mbox->external_transactions++;
+
+	mt = i_new(struct mbox_transaction_context, 1);
+	index_transaction_init(&mt->t, box, flags, reason);
+	return &mt->t;
+}
+
+static void
+mbox_transaction_unlock(struct mailbox *box, unsigned int lock_id1,
+			unsigned int lock_id2)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+
+	if (lock_id1 != 0)
+		mbox_unlock(mbox, lock_id1);
+	if (lock_id2 != 0)
+		mbox_unlock(mbox, lock_id2);
+	if (mbox->mbox_global_lock_id == 0) {
+		i_assert(mbox->box.transaction_count > 0);
+		i_assert(mbox->box.transaction_count > 1 ||
+			 mbox->external_transactions > 0 ||
+			 mbox->mbox_lock_type == F_UNLCK);
+	} else {
+		/* mailbox opened with MAILBOX_FLAG_KEEP_LOCKED */
+		i_assert(mbox->mbox_lock_type == F_WRLCK);
+	}
+}
+
+static int
+mbox_transaction_commit(struct mailbox_transaction_context *t,
+			struct mail_transaction_commit_changes *changes_r)
+{
+	struct mbox_transaction_context *mt = MBOX_TRANSCTX(t);
+	struct mailbox *box = t->box;
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	unsigned int read_lock_id = mt->read_lock_id;
+	unsigned int write_lock_id = mt->write_lock_id;
+	int ret;
+
+	if ((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) {
+		i_assert(mbox->external_transactions > 0);
+		mbox->external_transactions--;
+	}
+
+	ret = index_transaction_commit(t, changes_r);
+	mbox_transaction_unlock(box, read_lock_id, write_lock_id);
+	return ret;
+}
+
+static void
+mbox_transaction_rollback(struct mailbox_transaction_context *t)
+{
+	struct mbox_transaction_context *mt = MBOX_TRANSCTX(t);
+	struct mailbox *box = t->box;
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	unsigned int read_lock_id = mt->read_lock_id;
+	unsigned int write_lock_id = mt->write_lock_id;
+
+	if ((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0) {
+		i_assert(mbox->external_transactions > 0);
+		mbox->external_transactions--;
+	}
+
+	index_transaction_rollback(t);
+	mbox_transaction_unlock(box, read_lock_id, write_lock_id);
+}
+
+bool mbox_is_backend_readonly(struct mbox_mailbox *mbox)
+{
+	if (!mbox->backend_readonly_set) {
+		mbox->backend_readonly_set = TRUE;
+		if (access(mailbox_get_path(&mbox->box), R_OK|W_OK) < 0 &&
+		    errno == EACCES)
+			mbox->backend_readonly = TRUE;
+	}
+	return mbox->backend_readonly;
+}
+
+struct mail_storage mbox_storage = {
+	.name = MBOX_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_MAILBOX_IS_FILE |
+		MAIL_STORAGE_CLASS_FLAG_OPEN_STREAMS |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS,
+	.event_category = &event_category_mbox,
+
+	.v = {
+                mbox_get_setting_parser_info,
+		mbox_storage_alloc,
+		mbox_storage_create,
+		index_storage_destroy,
+		mbox_storage_add_list,
+		mbox_storage_get_list_settings,
+		mbox_storage_autodetect,
+		mbox_mailbox_alloc,
+		NULL,
+		NULL,
+	}
+};
+
+struct mailbox mbox_mailbox = {
+	.v = {
+		mbox_storage_is_readonly,
+		index_storage_mailbox_enable,
+		index_storage_mailbox_exists,
+		mbox_mailbox_open,
+		mbox_mailbox_close,
+		index_storage_mailbox_free,
+		mbox_mailbox_create,
+		mbox_mailbox_update,
+		index_storage_mailbox_delete,
+		index_storage_mailbox_rename,
+		index_storage_get_status,
+		mbox_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		mbox_list_index_has_changed,
+		mbox_list_index_update_sync,
+		mbox_storage_sync_init,
+		index_mailbox_sync_next,
+		index_mailbox_sync_deinit,
+		NULL,
+		mbox_notify_changes,
+		mbox_transaction_begin,
+		mbox_transaction_commit,
+		mbox_transaction_rollback,
+		NULL,
+		index_mail_alloc,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		mbox_save_alloc,
+		mbox_save_begin,
+		mbox_save_continue,
+		mbox_save_finish,
+		mbox_save_cancel,
+		mail_storage_copy,
+		mbox_transaction_save_commit_pre,
+		mbox_transaction_save_commit_post,
+		mbox_transaction_save_rollback,
+		index_storage_is_inconsistent
+	}
+};
diff --git a/src/lib-storage/index/mbox/mbox-storage.h b/src/lib-storage/index/mbox/mbox-storage.h
new file mode 100644
index 0000000..e35a795
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-storage.h
@@ -0,0 +1,115 @@
+#ifndef MBOX_STORAGE_H
+#define MBOX_STORAGE_H
+
+#include "index-storage.h"
+#include "mbox-settings.h"
+#include "mbox-md5.h"
+
+/* Padding to leave in X-Keywords header when rewriting mbox */
+#define MBOX_HEADER_PADDING 50
+/* Don't write Content-Length header unless it's value is larger than this. */
+#define MBOX_MIN_CONTENT_LENGTH_SIZE 1024
+
+#define MBOX_STORAGE_NAME "mbox"
+#define MBOX_SUBSCRIPTION_FILE_NAME ".subscriptions"
+#define MBOX_INDEX_DIR_NAME ".imap"
+#define MBOX_UIDVALIDITY_FNAME "dovecot-uidvalidity"
+
+struct mbox_index_header {
+	uint64_t sync_size;
+	uint32_t sync_mtime;
+	uint8_t dirty_flag;
+	uint8_t unused[3];
+	guid_128_t mailbox_guid;
+};
+
+struct mbox_list_index_record {
+	uint32_t mtime;
+	uint32_t size;
+};
+
+struct mbox_storage {
+	struct mail_storage storage;
+
+	const struct mbox_settings *set;
+	enum mbox_lock_type *read_locks;
+	enum mbox_lock_type *write_locks;
+	bool lock_settings_initialized:1;
+};
+
+struct mbox_mailbox {
+	struct mailbox box;
+	struct mbox_storage *storage;
+
+	int mbox_fd;
+	struct istream *mbox_stream, *mbox_file_stream;
+	int mbox_lock_type;
+	dev_t mbox_dev;
+	ino_t mbox_ino;
+	unsigned int mbox_excl_locks, mbox_shared_locks;
+	struct dotlock *mbox_dotlock;
+	unsigned int mbox_lock_id, mbox_global_lock_id;
+	struct timeout *keep_lock_to;
+	bool mbox_writeonly;
+	unsigned int external_transactions;
+
+	uint32_t mbox_ext_idx, md5hdr_ext_idx, mbox_list_index_ext_id;
+	struct mbox_index_header mbox_hdr;
+	const struct mailbox_update *sync_hdr_update;
+
+	struct mbox_md5_vfuncs md5_v;
+
+	bool no_mbox_file:1;
+	bool invalid_mbox_file:1;
+	bool mbox_broken_offsets:1;
+	bool mbox_save_md5:1;
+	bool mbox_dotlocked:1;
+	bool mbox_used_privileges:1;
+	bool mbox_privileged_locking:1;
+	bool syncing:1;
+	bool backend_readonly:1;
+	bool backend_readonly_set:1;
+};
+
+struct mbox_transaction_context {
+	struct mailbox_transaction_context t;
+	union mail_index_transaction_module_context module_ctx;
+
+	unsigned int read_lock_id;
+	unsigned int write_lock_id;
+};
+
+#define MBOX_STORAGE(s)		container_of(s, struct mbox_storage, storage)
+#define MBOX_MAILBOX(s)		container_of(s, struct mbox_mailbox, box)
+#define MBOX_TRANSCTX(s)	container_of(s, struct mbox_transaction_context, t)
+
+extern struct mail_vfuncs mbox_mail_vfuncs;
+extern const char *mbox_hide_headers[], *mbox_save_drop_headers[];
+extern unsigned int mbox_hide_headers_count, mbox_save_drop_headers_count;
+
+void mbox_set_syscall_error(struct mbox_mailbox *mbox, const char *function);
+void mbox_istream_set_syscall_error(struct mbox_mailbox *mbox,
+				    struct istream *input,
+				    const char *function);
+void mbox_ostream_set_syscall_error(struct mbox_mailbox *mbox,
+				    struct ostream *output,
+				    const char *function);
+
+struct mailbox_sync_context *
+mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+
+struct mail_save_context *
+mbox_save_alloc(struct mailbox_transaction_context *_t);
+int mbox_save_begin(struct mail_save_context *ctx, struct istream *input);
+int mbox_save_continue(struct mail_save_context *ctx);
+int mbox_save_finish(struct mail_save_context *ctx);
+void mbox_save_cancel(struct mail_save_context *ctx);
+
+int mbox_transaction_save_commit_pre(struct mail_save_context *ctx);
+void mbox_transaction_save_commit_post(struct mail_save_context *ctx,
+				       struct mail_index_transaction_commit_result *result);
+void mbox_transaction_save_rollback(struct mail_save_context *ctx);
+
+bool mbox_is_backend_readonly(struct mbox_mailbox *mbox);
+
+#endif
diff --git a/src/lib-storage/index/mbox/mbox-sync-list-index.c b/src/lib-storage/index/mbox/mbox-sync-list-index.c
new file mode 100644
index 0000000..934b7ea
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-sync-list-index.c
@@ -0,0 +1,109 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mbox-storage.h"
+#include "mbox-sync-private.h"
+
+static unsigned int
+mbox_list_get_ext_id(struct mbox_mailbox *mbox,
+		     struct mail_index_view *view)
+{
+	if (mbox->mbox_list_index_ext_id == (uint32_t)-1) {
+		mbox->mbox_list_index_ext_id =
+			mail_index_ext_register(mail_index_view_get_index(view),
+				"mbox", 0,
+				sizeof(struct mbox_list_index_record),
+				sizeof(uint32_t));
+	}
+	return mbox->mbox_list_index_ext_id;
+}
+
+int mbox_list_index_has_changed(struct mailbox *box,
+				struct mail_index_view *list_view,
+				uint32_t seq, bool quick, const char **reason_r)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	const struct mbox_list_index_record *rec;
+	const void *data;
+	const char *path;
+	struct stat st;
+	uint32_t ext_id;
+	bool expunged;
+	int ret;
+
+	ret = index_storage_list_index_has_changed(box, list_view, seq,
+						   quick, reason_r);
+	if (ret != 0 || box->storage->set->mailbox_list_index_very_dirty_syncs)
+		return ret;
+
+	ext_id = mbox_list_get_ext_id(mbox, list_view);
+	mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
+	rec = data;
+
+	if (rec == NULL) {
+		*reason_r = "mbox record is missing";
+		return 1;
+	} else if (expunged) {
+		*reason_r = "mbox record is expunged";
+		return 1;
+	} else if (rec->mtime == 0) {
+		/* not synced */
+		*reason_r = "mbox record mtime=0";
+		return 1;
+	}
+
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path);
+	if (ret < 0)
+		return ret;
+	i_assert(ret > 0);
+
+	if (stat(path, &st) < 0) {
+		mailbox_set_critical(box, "stat(%s) failed: %m", path);
+		return -1;
+	}
+	if ((time_t)rec->mtime != st.st_mtime) {
+		*reason_r = t_strdup_printf(
+			"mbox record mtime changed %u != %"PRIdTIME_T,
+			rec->mtime, st.st_mtime);
+		return 1;
+	}
+	uint32_t new_size = (uint32_t)(st.st_size & 0xffffffffU);
+	if (rec->size != new_size) {
+		*reason_r = t_strdup_printf("mbox record size changed %u != %u",
+					    rec->size, new_size);
+		return 1;
+	}
+	return 0;
+}
+
+void mbox_list_index_update_sync(struct mailbox *box,
+				 struct mail_index_transaction *trans,
+				 uint32_t seq)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	struct mail_index_view *list_view;
+	const struct mbox_index_header *mhdr = &mbox->mbox_hdr;
+	const struct mbox_list_index_record *old_rec;
+	struct mbox_list_index_record new_rec;
+	const void *data;
+	uint32_t ext_id;
+	bool expunged;
+
+	index_storage_list_index_update_sync(box, trans, seq);
+
+	/* get the current record */
+	list_view = mail_index_transaction_get_view(trans);
+	ext_id = mbox_list_get_ext_id(mbox, list_view);
+	mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
+	if (expunged)
+		return;
+	old_rec = data;
+
+	i_zero(&new_rec);
+	new_rec.mtime = mhdr->sync_mtime;
+	new_rec.size = mhdr->sync_size & 0xffffffffU;
+
+	if (old_rec == NULL ||
+	    memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0)
+		mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL);
+}
diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c
new file mode 100644
index 0000000..bbc2da2
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-sync-parse.c
@@ -0,0 +1,616 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+/* MD5 header summing logic was pretty much copy&pasted from popa3d by
+   Solar Designer */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "buffer.h"
+#include "istream.h"
+#include "str.h"
+#include "write-full.h"
+#include "message-parser.h"
+#include "mail-index.h"
+#include "mbox-storage.h"
+#include "mbox-md5.h"
+#include "mbox-sync-private.h"
+
+
+#define IS_LWSP_LF(c) (IS_LWSP(c) || (c) == '\n')
+
+struct mbox_sync_header_func {
+	const char *header;
+	bool (*func)(struct mbox_sync_mail_context *ctx,
+		     struct message_header_line *hdr);
+};
+
+struct mbox_flag_type mbox_status_flags[] = {
+	{ 'R', MAIL_SEEN },
+	{ 'O', MBOX_NONRECENT_KLUDGE },
+	{ 0, 0 }
+};
+
+struct mbox_flag_type mbox_xstatus_flags[] = {
+	{ 'A', MAIL_ANSWERED },
+	{ 'F', MAIL_FLAGGED },
+	{ 'T', MAIL_DRAFT },
+	{ 'D', MAIL_DELETED },
+	{ 0, 0 }
+};
+
+static void parse_trailing_whitespace(struct mbox_sync_mail_context *ctx,
+				      struct message_header_line *hdr)
+{
+	size_t i, space = 0;
+
+	/* the value may contain newlines. we can't count whitespace before
+	   and after it as a single contiguous whitespace block, as that may
+	   get us into situation where removing whitespace goes eg.
+	   " \n \n" -> " \n\n" which would then be treated as end of headers.
+
+	   that could probably be avoided by being careful, but as newlines
+	   should never be there (we don't generate them), it's not worth the
+	   trouble. */
+
+	for (i = hdr->full_value_len; i > 0; i--) {
+		if (!IS_LWSP(hdr->full_value[i-1]))
+			break;
+		space++;
+	}
+
+	if ((ssize_t)space > ctx->mail.space) {
+		i_assert(space != 0);
+		ctx->mail.offset = ctx->hdr_offset + str_len(ctx->header) + i;
+		ctx->mail.space = space;
+	}
+}
+
+static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr)
+{
+	int i;
+
+	for (i = 0; flags[i].chr != 0; i++) {
+		if (flags[i].chr == chr)
+			return flags[i].flag;
+	}
+
+	return 0;
+}
+
+static bool parse_status_flags(struct mbox_sync_mail_context *ctx,
+			       struct message_header_line *hdr,
+			       struct mbox_flag_type *flags_list)
+{
+	enum mail_flags flag;
+	size_t i;
+	bool duplicates = FALSE;
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+	for (i = 0; i < hdr->full_value_len; i++) {
+		flag = mbox_flag_find(flags_list, hdr->full_value[i]);
+		if ((ctx->mail.flags & flag) != 0)
+			duplicates = TRUE;
+		else
+			ctx->mail.flags |= flag;
+	}
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+	return duplicates;
+}
+
+static bool parse_status(struct mbox_sync_mail_context *ctx,
+			 struct message_header_line *hdr)
+{
+	if (parse_status_flags(ctx, hdr, mbox_status_flags))
+		ctx->mail.status_broken = TRUE;
+	ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
+	return TRUE;
+}
+
+static bool parse_x_status(struct mbox_sync_mail_context *ctx,
+			   struct message_header_line *hdr)
+{
+	if (parse_status_flags(ctx, hdr, mbox_xstatus_flags))
+		ctx->mail.xstatus_broken = TRUE;
+	ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header);
+	return TRUE;
+}
+
+static void
+parse_imap_keywords_list(struct mbox_sync_mail_context *ctx,
+                         struct message_header_line *hdr, size_t pos)
+{
+	struct mailbox *box = &ctx->sync_ctx->mbox->box;
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	const char *keyword, *error;
+	size_t keyword_start;
+	unsigned int idx, count;
+
+	count = 0;
+	while (pos < hdr->full_value_len) {
+		if (IS_LWSP_LF(hdr->full_value[pos])) {
+                        pos++;
+			continue;
+		}
+
+		/* read the keyword */
+		keyword_start = pos;
+		for (; pos < hdr->full_value_len; pos++) {
+			if (IS_LWSP_LF(hdr->full_value[pos]))
+				break;
+		}
+
+		/* add it to index's keyword list if it's not there already */
+		keyword = t_strndup(hdr->full_value + keyword_start,
+				    pos - keyword_start);
+		if (mailbox_keyword_is_valid(&ctx->sync_ctx->mbox->box,
+					     keyword, &error)) {
+			mail_index_keyword_lookup_or_create(box->index,
+							    keyword, &idx);
+		}
+		count++;
+	}
+
+	if (count != array_count(ibox->keyword_names)) {
+		/* need to update this list */
+		ctx->imapbase_rewrite = TRUE;
+		ctx->need_rewrite = TRUE;
+	}
+}
+
+static bool parse_x_imap_base(struct mbox_sync_mail_context *ctx,
+			      struct message_header_line *hdr)
+{
+	size_t i, j, uid_last_pos;
+	uint32_t uid_validity, uid_last;
+
+	if (ctx->seq != 1 || ctx->seen_imapbase ||
+	    ctx->sync_ctx->renumber_uids) {
+		/* Valid only in first message */
+		return FALSE;
+	}
+
+	/* <uid-validity> 10x<uid-last> */
+	for (i = 0, uid_validity = 0; i < hdr->full_value_len; i++) {
+		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') {
+			if (hdr->full_value[i] != ' ')
+				return FALSE;
+			break;
+		}
+		uid_validity = uid_validity * 10 + (hdr->full_value[i] - '0');
+	}
+
+	if (uid_validity == 0) {
+		/* broken */
+		return FALSE;
+	}
+
+	for (; i < hdr->full_value_len; i++) {
+		if (!IS_LWSP_LF(hdr->full_value[i]))
+			break;
+	}
+	uid_last_pos = i;
+
+	for (uid_last = 0, j = 0; i < hdr->full_value_len; i++, j++) {
+		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') {
+			if (!IS_LWSP_LF(hdr->full_value[i]))
+				return FALSE;
+			break;
+		}
+		uid_last = uid_last * 10 + (hdr->full_value[i] - '0');
+	}
+
+	if (j != 10 ||
+	    hdr->full_value_offset != ctx->hdr_offset + str_len(ctx->header)) {
+		/* uid-last field must be exactly 10 characters to make
+		   rewriting it easier. also don't try to do this if some
+		   headers have been removed */
+		ctx->imapbase_rewrite = TRUE;
+		ctx->need_rewrite = TRUE;
+	} else {
+		ctx->last_uid_value_start_pos = uid_last_pos;
+		ctx->sync_ctx->base_uid_last_offset =
+			hdr->full_value_offset + uid_last_pos;
+	}
+
+	if (ctx->sync_ctx->base_uid_validity == 0) {
+		/* first time parsing this (ie. we're not rewriting).
+		   save the values. */
+		ctx->sync_ctx->base_uid_validity = uid_validity;
+		ctx->sync_ctx->base_uid_last = uid_last;
+
+		if (ctx->sync_ctx->next_uid-1 <= uid_last) {
+			/* new messages have been added since our last sync.
+			   just update our internal next_uid. */
+			ctx->sync_ctx->next_uid = uid_last+1;
+		} else {
+			/* we need to rewrite the next-uid */
+			ctx->need_rewrite = TRUE;
+		}
+		i_assert(ctx->sync_ctx->next_uid > ctx->sync_ctx->prev_msg_uid);
+	}
+
+	ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
+	ctx->seen_imapbase = TRUE;
+
+	T_BEGIN {
+		parse_imap_keywords_list(ctx, hdr, i);
+	} T_END;
+	parse_trailing_whitespace(ctx, hdr);
+	return TRUE;
+}
+
+static bool parse_x_imap(struct mbox_sync_mail_context *ctx,
+			 struct message_header_line *hdr)
+{
+	if (!parse_x_imap_base(ctx, hdr))
+		return FALSE;
+
+	/* this is the c-client style "FOLDER INTERNAL DATA" message.
+	   skip it. */
+	ctx->mail.pseudo = TRUE;
+	return TRUE;
+}
+
+static bool parse_x_keywords_real(struct mbox_sync_mail_context *ctx,
+				  struct message_header_line *hdr)
+{
+	struct mailbox *box = &ctx->sync_ctx->mbox->box;
+	ARRAY_TYPE(keyword_indexes) keyword_list;
+	const unsigned int *list;
+	string_t *keyword;
+	size_t keyword_start;
+	unsigned int i, idx, count;
+	size_t pos;
+
+	if (array_is_created(&ctx->mail.keywords))
+		return FALSE; /* duplicate header, delete */
+
+	/* read keyword indexes to temporary array first */
+	keyword = t_str_new(128);
+	t_array_init(&keyword_list, 16);
+
+	for (pos = 0; pos < hdr->full_value_len; ) {
+		if (IS_LWSP_LF(hdr->full_value[pos])) {
+                        pos++;
+			continue;
+		}
+
+		/* read the keyword string */
+		keyword_start = pos;
+		for (; pos < hdr->full_value_len; pos++) {
+			if (IS_LWSP_LF(hdr->full_value[pos]))
+				break;
+		}
+
+		str_truncate(keyword, 0);
+		str_append_data(keyword, hdr->full_value + keyword_start,
+				pos - keyword_start);
+		if (!mail_index_keyword_lookup(box->index, str_c(keyword),
+					       &idx)) {
+			/* keyword wasn't found. that means the sent mail
+			   originally contained X-Keywords header. Delete it. */
+			return FALSE;
+		}
+
+		/* check that the keyword isn't already added there.
+		   we don't want duplicates. */
+		list = array_get(&keyword_list, &count);
+		for (i = 0; i < count; i++) {
+			if (list[i] == idx)
+				break;
+		}
+
+		if (i == count)
+			array_push_back(&keyword_list, &idx);
+	}
+
+	/* once we know how many keywords there are, we can allocate the array
+	   from mail_keyword_pool without wasting memory. */
+	if (array_count(&keyword_list) > 0) {
+		p_array_init(&ctx->mail.keywords,
+			     ctx->sync_ctx->mail_keyword_pool,
+			     array_count(&keyword_list));
+		array_append_array(&ctx->mail.keywords, &keyword_list);
+	}
+
+	ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
+	parse_trailing_whitespace(ctx, hdr);
+	return TRUE;
+}
+
+static bool parse_x_keywords(struct mbox_sync_mail_context *ctx,
+			     struct message_header_line *hdr)
+{
+	bool ret;
+
+	T_BEGIN {
+		ret = parse_x_keywords_real(ctx, hdr);
+	} T_END;
+	return ret;
+}
+
+static bool parse_x_uid(struct mbox_sync_mail_context *ctx,
+			struct message_header_line *hdr)
+{
+	uint32_t value = 0;
+	size_t i;
+
+	if (ctx->mail.uid != 0) {
+		/* duplicate */
+		return FALSE;
+	}
+
+	for (i = 0; i < hdr->full_value_len; i++) {
+		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9')
+			break;
+		value = value*10 + (hdr->full_value[i] - '0');
+	}
+
+	for (; i < hdr->full_value_len; i++) {
+		if (!IS_LWSP_LF(hdr->full_value[i])) {
+			/* broken value */
+			return FALSE;
+		}
+	}
+
+	if (ctx->sync_ctx == NULL) {
+		/* we're in mbox_sync_parse_match_mail().
+		   don't do any extra checks. */
+		ctx->mail.uid = value;
+		return TRUE;
+	}
+
+	if (ctx->seq == 1 && !ctx->seen_imapbase) {
+		/* Don't bother allowing X-UID before X-IMAPbase
+		   header. c-client doesn't allow it either, and this
+		   way the UID doesn't have to be reset if X-IMAPbase
+		   header isn't what we expect it to be. */
+		return FALSE;
+	}
+
+	if (value == ctx->sync_ctx->next_uid) {
+		/* X-UID is the next expected one. allow it because
+		   we'd just use this UID anyway. X-IMAPbase header
+		   still needs to be updated for this. */
+		ctx->sync_ctx->next_uid++;
+	} else if (value > ctx->sync_ctx->next_uid) {
+		/* UID is larger than expected. Don't allow it because
+		   incoming mails can contain untrusted X-UID fields,
+		   causing possibly DoS if the UIDs get large enough. */
+		ctx->mail.uid_broken = TRUE;
+		return FALSE;
+	}
+
+	if (value <= ctx->sync_ctx->prev_msg_uid) {
+		/* broken - UIDs must be growing */
+		ctx->mail.uid_broken = TRUE;
+		return FALSE;
+	}
+
+	ctx->mail.uid = value;
+	/* if we had multiple X-UID headers, we could have
+	   uid_broken=TRUE here. */
+	ctx->mail.uid_broken = FALSE;
+
+	if (ctx->sync_ctx->dest_first_mail && ctx->seq != 1) {
+		/* if we're expunging the first mail, delete this header since
+		   otherwise X-IMAPbase header would be added after this, which
+		   we don't like */
+		return FALSE;
+	}
+
+	ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
+	ctx->parsed_uid = value;
+	parse_trailing_whitespace(ctx, hdr);
+	return TRUE;
+}
+
+static bool parse_content_length(struct mbox_sync_mail_context *ctx,
+				 struct message_header_line *hdr)
+{
+	uoff_t value = 0;
+	size_t i;
+
+	if (ctx->content_length != UOFF_T_MAX) {
+		/* duplicate */
+		return FALSE;
+	}
+
+	for (i = 0; i < hdr->full_value_len; i++) {
+		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9')
+			break;
+		value = value*10 + (hdr->full_value[i] - '0');
+	}
+
+	for (; i < hdr->full_value_len; i++) {
+		if (!IS_LWSP_LF(hdr->full_value[i])) {
+			/* broken value */
+			return FALSE;
+		}
+	}
+
+	ctx->content_length = value;
+	return TRUE;
+}
+
+static struct mbox_sync_header_func header_funcs[] = {
+	{ "Content-Length", parse_content_length },
+	{ "Status", parse_status },
+	{ "X-IMAP", parse_x_imap },
+	{ "X-IMAPbase", parse_x_imap_base },
+	{ "X-Keywords", parse_x_keywords },
+	{ "X-Status", parse_x_status },
+	{ "X-UID", parse_x_uid }
+};
+
+static int mbox_sync_bsearch_header_func_cmp(const void *p1, const void *p2)
+{
+	const char *key = p1;
+	const struct mbox_sync_header_func *func = p2;
+
+	return strcasecmp(key, func->header);
+}
+
+int mbox_sync_parse_next_mail(struct istream *input,
+			      struct mbox_sync_mail_context *ctx)
+{
+	struct mbox_sync_context *sync_ctx = ctx->sync_ctx;
+	struct message_header_parser_ctx *hdr_ctx;
+	struct message_header_line *hdr;
+	struct mbox_sync_header_func *func;
+	struct mbox_md5_context *mbox_md5_ctx;
+	size_t line_start_pos;
+	int i, ret;
+
+	ctx->hdr_offset = ctx->mail.offset;
+	ctx->mail.flags = MAIL_RECENT; /* default to having recent flag */
+
+        ctx->header_first_change = SIZE_MAX;
+	ctx->header_last_change = 0;
+
+	for (i = 0; i < MBOX_HDR_COUNT; i++)
+		ctx->hdr_pos[i] = SIZE_MAX;
+
+	ctx->content_length = UOFF_T_MAX;
+	str_truncate(ctx->header, 0);
+
+        mbox_md5_ctx = ctx->sync_ctx->mbox->md5_v.init();
+
+        line_start_pos = 0;
+	hdr_ctx = message_parse_header_init(input, NULL, 0);
+	while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) {
+		if (hdr->eoh) {
+			ctx->have_eoh = TRUE;
+			break;
+		}
+
+		if (!hdr->continued) {
+			line_start_pos = str_len(ctx->header);
+			str_append(ctx->header, hdr->name);
+			str_append_data(ctx->header, hdr->middle, hdr->middle_len);
+		}
+
+		func = bsearch(hdr->name, header_funcs,
+			       N_ELEMENTS(header_funcs), sizeof(*header_funcs),
+			       mbox_sync_bsearch_header_func_cmp);
+
+		if (func != NULL) {
+			if (hdr->continues) {
+				hdr->use_full_value = TRUE;
+				continue;
+			}
+
+			if (!func->func(ctx, hdr)) {
+				/* this header is broken, remove it */
+				ctx->need_rewrite = TRUE;
+				str_truncate(ctx->header, line_start_pos);
+				if (ctx->header_first_change == SIZE_MAX) {
+					ctx->header_first_change =
+						line_start_pos;
+				}
+				continue;
+			}
+			buffer_append(ctx->header, hdr->full_value,
+				      hdr->full_value_len);
+		} else {
+			ctx->sync_ctx->mbox->md5_v.more(mbox_md5_ctx, hdr);
+			buffer_append(ctx->header, hdr->value,
+				      hdr->value_len);
+		}
+		if (!hdr->no_newline) {
+			if (hdr->crlf_newline)
+				str_append_c(ctx->header, '\r');
+			str_append_c(ctx->header, '\n');
+		}
+	}
+	i_assert(ret != 0);
+	message_parse_header_deinit(&hdr_ctx);
+
+	ctx->sync_ctx->mbox->md5_v.finish(mbox_md5_ctx, ctx->hdr_md5_sum);
+
+	if ((ctx->seq == 1 && !ctx->seen_imapbase) ||
+	    (ctx->seq > 1 && sync_ctx->dest_first_mail)) {
+		/* missing X-IMAPbase */
+		ctx->need_rewrite = TRUE;
+		if (sync_ctx->base_uid_validity == 0) {
+			/* figure out a new UIDVALIDITY for us. */
+			sync_ctx->base_uid_validity =
+				sync_ctx->hdr->uid_validity != 0 &&
+				!sync_ctx->renumber_uids ?
+				sync_ctx->hdr->uid_validity :
+				I_MAX((uint32_t)ioloop_time, 1);
+		}
+	}
+
+	ctx->body_offset = input->v_offset;
+	if (input->stream_errno != 0) {
+		mbox_sync_set_critical(ctx->sync_ctx, "read(%s) failed: %s",
+			i_stream_get_name(input), i_stream_get_error(input));
+		return -1;
+	}
+	return 0;
+}
+
+bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox,
+				struct mail_index_view *view, uint32_t seq)
+{
+        struct mbox_sync_mail_context ctx;
+	struct message_header_parser_ctx *hdr_ctx;
+	struct message_header_line *hdr;
+	struct header_func *func;
+	struct mbox_md5_context *mbox_md5_ctx;
+	const void *data;
+	bool expunged;
+	uint32_t uid;
+	int ret;
+
+	/* we only wish to be sure that this mail actually is what we expect
+	   it to be. If there's X-UID header and it matches our UID, we use it.
+	   Otherwise it could mean that the X-UID header is invalid and it's
+	   just not yet been rewritten. In that case use MD5 sum, if it
+	   exists. */
+
+	mail_index_lookup_uid(view, seq, &uid);
+	i_zero(&ctx);
+        mbox_md5_ctx = mbox->md5_v.init();
+
+	hdr_ctx = message_parse_header_init(mbox->mbox_stream, NULL, 0);
+	while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) {
+		if (hdr->eoh)
+			break;
+
+		func = bsearch(hdr->name, header_funcs,
+			       N_ELEMENTS(header_funcs), sizeof(*header_funcs),
+			       mbox_sync_bsearch_header_func_cmp);
+		if (func != NULL) {
+			if (strcasecmp(hdr->name, "X-UID") == 0) {
+				if (hdr->continues) {
+					hdr->use_full_value = TRUE;
+					continue;
+				}
+				(void)parse_x_uid(&ctx, hdr);
+
+				if (ctx.mail.uid == uid)
+					break;
+			}
+		} else {
+			mbox->md5_v.more(mbox_md5_ctx, hdr);
+		}
+	}
+	i_assert(ret != 0);
+	message_parse_header_deinit(&hdr_ctx);
+
+	mbox->md5_v.finish(mbox_md5_ctx, ctx.hdr_md5_sum);
+
+	if (ctx.mail.uid == uid)
+		return TRUE;
+
+	/* match by MD5 sum */
+	mbox->mbox_save_md5 = TRUE;
+
+	mail_index_lookup_ext(view, seq, mbox->md5hdr_ext_idx,
+			      &data, &expunged);
+	return data == NULL ? 0 :
+		memcmp(data, ctx.hdr_md5_sum, 16) == 0;
+}
diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h
new file mode 100644
index 0000000..7585e4d
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-sync-private.h
@@ -0,0 +1,192 @@
+#ifndef MBOX_SYNC_PRIVATE_H
+#define MBOX_SYNC_PRIVATE_H
+
+#include "md5.h"
+#include "mail-index.h"
+
+#include <sys/stat.h>
+
+enum mbox_sync_flags {
+	MBOX_SYNC_HEADER	= 0x02,
+	MBOX_SYNC_LOCK_READING	= 0x04,
+	MBOX_SYNC_UNDIRTY	= 0x08,
+	MBOX_SYNC_REWRITE	= 0x10,
+	MBOX_SYNC_FORCE_SYNC	= 0x20,
+	MBOX_SYNC_READONLY	= 0x40
+};
+
+struct mbox_flag_type {
+	char chr;
+	enum mail_flags flag;
+};
+
+enum header_position {
+	MBOX_HDR_STATUS,
+	MBOX_HDR_X_IMAPBASE,
+	MBOX_HDR_X_KEYWORDS,
+	MBOX_HDR_X_STATUS,
+	MBOX_HDR_X_UID,
+
+        MBOX_HDR_COUNT
+};
+
+/* kludgy. swap MAIL_RECENT with MBOX_NONRECENT_KLUDGE when writing Status
+   header, because 'O' flag means non-recent but internally we want to use
+   recent flag. */
+#define MBOX_NONRECENT_KLUDGE MAIL_RECENT
+
+#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT_KLUDGE)
+#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED)
+extern struct mbox_flag_type mbox_status_flags[];
+extern struct mbox_flag_type mbox_xstatus_flags[];
+
+struct mbox_sync_mail {
+	/* uid=0 can mean that this mail describes an expunged area or that
+	   this is a pseudo message */
+	uint32_t uid;
+	uint32_t idx_seq;
+
+	ARRAY_TYPE(keyword_indexes) keywords;
+	uint8_t flags;
+
+	bool uid_broken:1;
+	bool expunged:1;
+	bool pseudo:1;
+	bool status_broken:1;
+	bool xstatus_broken:1;
+
+	uoff_t from_offset;
+	uoff_t body_size;
+
+	/* following variables have a bit overloaded functionality:
+
+	   a) space <= 0 : offset points to beginning of headers. space is the
+	      amount of space missing that is required to be able to rewrite
+	      the headers
+	   b) space > 0 : offset points to beginning of whitespace that can
+	      be removed. space is the amount of data that can be removed from
+	      there. note that the message may contain more whitespace
+	      elsewhere. */
+	uoff_t offset;
+	off_t space;
+};
+
+struct mbox_sync_mail_context {
+	struct mbox_sync_context *sync_ctx;
+	struct mbox_sync_mail mail;
+
+	uint32_t seq;
+	uoff_t hdr_offset, body_offset;
+
+	size_t header_first_change, header_last_change;
+	string_t *header;
+
+	unsigned char hdr_md5_sum[16];
+
+	uoff_t content_length;
+
+	size_t hdr_pos[MBOX_HDR_COUNT];
+	uint32_t parsed_uid, last_uid_updated_value;
+	unsigned int last_uid_value_start_pos;
+
+	bool have_eoh:1;
+	bool need_rewrite:1;
+	bool seen_imapbase:1;
+	bool updated:1;
+	bool recent:1;
+	bool dirty:1;
+	bool imapbase_rewrite:1;
+	bool imapbase_updated:1;
+};
+
+struct mbox_sync_context {
+	struct mbox_mailbox *mbox;
+        enum mbox_sync_flags flags;
+	struct istream *input, *file_input;
+	int write_fd;
+
+	time_t orig_mtime, orig_atime;
+	uoff_t orig_size;
+	struct stat last_stat;
+
+	struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *t;
+
+	struct mail_index_header reset_hdr;
+	const struct mail_index_header *hdr;
+
+	string_t *header, *from_line;
+
+	/* header state: */
+	uint32_t base_uid_validity, base_uid_last;
+	uoff_t base_uid_last_offset;
+
+	/* mail state: */
+	ARRAY(struct mbox_sync_mail) mails;
+	struct index_sync_changes_context *sync_changes;
+
+	/* per-mail pool */
+	pool_t mail_keyword_pool;
+	/* used for mails[].keywords */
+	pool_t saved_keywords_pool;
+
+	uint32_t prev_msg_uid, next_uid, idx_next_uid;
+	uint32_t seq, idx_seq, need_space_seq;
+	uint32_t last_nonrecent_uid;
+	off_t expunged_space, space_diff;
+
+	bool dest_first_mail:1;
+	bool first_mail_crlf_expunged:1;
+
+	/* global flags: */
+	bool keep_recent:1;
+	bool readonly:1;
+	bool delay_writes:1;
+	bool renumber_uids:1;
+	bool moved_offsets:1;
+	bool ext_modified:1;
+	bool index_reset:1;
+	bool errors:1;
+};
+
+int mbox_sync_header_refresh(struct mbox_mailbox *mbox);
+int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags);
+int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty);
+void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,
+			    const char *fmt, ...) ATTR_FORMAT(2, 3);
+
+int mbox_sync_parse_next_mail(struct istream *input,
+			      struct mbox_sync_mail_context *ctx);
+bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox,
+				struct mail_index_view *view, uint32_t seq);
+
+void mbox_sync_update_header(struct mbox_sync_mail_context *ctx);
+void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
+				  const struct mbox_sync_mail *mail);
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff);
+int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx,
+                      struct mbox_sync_mail_context *mail_ctx,
+		      uoff_t end_offset, off_t move_diff, uoff_t extra_space,
+		      uint32_t first_seq, uint32_t last_seq);
+
+int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset);
+void mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx);
+void mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty);
+int mbox_move(struct mbox_sync_context *sync_ctx,
+	      uoff_t dest, uoff_t source, uoff_t size);
+void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx,
+			   size_t pos, size_t need, size_t have);
+void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
+				 size_t size);
+int mbox_sync_get_guid(struct mbox_mailbox *mbox);
+
+int mbox_list_index_has_changed(struct mailbox *box,
+				struct mail_index_view *list_view,
+				uint32_t seq, bool quick,
+				const char **reason_r);
+void mbox_list_index_update_sync(struct mailbox *box,
+				 struct mail_index_transaction *trans,
+				 uint32_t seq);
+
+#endif
diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c
new file mode 100644
index 0000000..eeb4878
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c
@@ -0,0 +1,615 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "istream.h"
+#include "ostream.h"
+#include "str.h"
+#include "write-full.h"
+#include "message-parser.h"
+#include "mbox-storage.h"
+#include "mbox-sync-private.h"
+#include "istream-raw-mbox.h"
+
+int mbox_move(struct mbox_sync_context *sync_ctx,
+	      uoff_t dest, uoff_t source, uoff_t size)
+{
+	struct mbox_mailbox *mbox = sync_ctx->mbox;
+	struct istream *input;
+	struct ostream *output;
+	int ret;
+
+	i_assert(source > 0 || (dest != 1 && dest != 2));
+	i_assert(size < OFF_T_MAX);
+
+	if (size == 0 || source == dest)
+		return 0;
+
+	i_stream_sync(sync_ctx->input);
+
+	output = o_stream_create_fd_file(sync_ctx->write_fd, UOFF_T_MAX, FALSE);
+	i_stream_seek(sync_ctx->file_input, source);
+	if (o_stream_seek(output, dest) < 0) {
+		mbox_ostream_set_syscall_error(sync_ctx->mbox, output,
+					       "o_stream_seek()");
+		o_stream_unref(&output);
+		return -1;
+	}
+
+	/* we're moving data within a file. it really shouldn't be failing at
+	   this point or we're corrupted. */
+	input = i_stream_create_limit(sync_ctx->file_input, size);
+	o_stream_nsend_istream(output, input);
+	if (input->stream_errno != 0) {
+		mailbox_set_critical(&mbox->box,
+			"read() failed with mbox: %s",
+			i_stream_get_error(input));
+		ret = -1;
+	} else if (output->stream_errno != 0) {
+		mailbox_set_critical(&mbox->box,
+			"write() failed with mbox: %s",
+			o_stream_get_error(output));
+		ret = -1;
+	} else if (input->v_offset != size) {
+		mbox_sync_set_critical(sync_ctx,
+			"mbox_move(%"PRIuUOFF_T", %"PRIuUOFF_T", %"PRIuUOFF_T
+			") moved only %"PRIuUOFF_T" bytes",
+			dest, source, size, input->v_offset);
+		ret = -1;
+	} else {
+		ret = 0;
+	}
+	i_stream_unref(&input);
+
+	mbox_sync_file_updated(sync_ctx, FALSE);
+	o_stream_destroy(&output);
+	return ret;
+}
+
+static int mbox_fill_space(struct mbox_sync_context *sync_ctx,
+			   uoff_t offset, uoff_t size)
+{
+	unsigned char space[1024];
+
+	memset(space, ' ', sizeof(space));
+	while (size > sizeof(space)) {
+		if (pwrite_full(sync_ctx->write_fd, space,
+				sizeof(space), offset) < 0) {
+			mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
+			return -1;
+		}
+		size -= sizeof(space);
+	}
+
+	if (pwrite_full(sync_ctx->write_fd, space, size, offset) < 0) {
+		mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
+		return -1;
+	}
+	mbox_sync_file_updated(sync_ctx, TRUE);
+	return 0;
+}
+
+void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
+				 size_t size)
+{
+	size_t data_size, pos, start_pos;
+	const unsigned char *data;
+	void *p;
+
+	i_assert(size < SSIZE_T_MAX);
+
+	if (ctx->mail.pseudo)
+		start_pos = ctx->hdr_pos[MBOX_HDR_X_IMAPBASE];
+	else if (ctx->mail.space > 0) {
+		/* update the header using the existing offset.
+		   otherwise we might chose wrong header and just decrease
+		   the available space */
+		start_pos = ctx->mail.offset - ctx->hdr_offset;
+	} else {
+		/* Append at the end of X-Keywords header,
+		   or X-UID if it doesn't exist */
+		start_pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != SIZE_MAX ?
+			ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] :
+			ctx->hdr_pos[MBOX_HDR_X_UID];
+	}
+
+	data = str_data(ctx->header);
+	data_size = str_len(ctx->header);
+	i_assert(start_pos < data_size);
+
+	for (pos = start_pos; pos < data_size; pos++) {
+		if (data[pos] == '\n') {
+			/* possibly continues in next line */
+			if (pos+1 == data_size || !IS_LWSP(data[pos+1]))
+				break;
+			start_pos = pos+1;
+		} else if (!IS_LWSP(data[pos]) && data[pos] != '\r') {
+			start_pos = pos+1;
+		}
+	}
+
+	/* pos points to end of header now, and start_pos to beginning
+	   of whitespace. */
+	mbox_sync_move_buffer(ctx, pos, size, 0);
+
+	p = buffer_get_space_unsafe(ctx->header, pos, size);
+	memset(p, ' ', size);
+
+	if (ctx->header_first_change > pos)
+		ctx->header_first_change = pos;
+	ctx->header_last_change = SIZE_MAX;
+
+	ctx->mail.space = (pos - start_pos) + size;
+	ctx->mail.offset = ctx->hdr_offset;
+	if (ctx->mail.space > 0)
+		ctx->mail.offset += start_pos;
+}
+
+static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx,
+					  size_t start_pos, size_t *size)
+{
+	const unsigned char *data;
+	size_t data_size, pos, last_line_pos;
+
+	/* find the end of the LWSP */
+	data = str_data(ctx->header);
+	data_size = str_len(ctx->header);
+
+	for (pos = last_line_pos = start_pos; pos < data_size; pos++) {
+		if (data[pos] == '\n') {
+			/* possibly continues in next line */
+			if (pos+1 == data_size || !IS_LWSP(data[pos+1])) {
+				data_size = pos;
+				break;
+			}
+                        last_line_pos = pos+1;
+		} else if (!IS_LWSP(data[pos]) && data[pos] != '\r') {
+			start_pos = last_line_pos = pos+1;
+		}
+	}
+
+	if (start_pos == data_size)
+		return;
+
+	/* and remove what we can */
+	if (ctx->header_first_change > start_pos)
+		ctx->header_first_change = start_pos;
+	ctx->header_last_change = SIZE_MAX;
+
+	if (data_size - start_pos <= *size) {
+		/* remove it all */
+		mbox_sync_move_buffer(ctx, start_pos, 0, data_size - start_pos);
+		*size -= data_size - start_pos;
+		return;
+	}
+
+	/* we have more space than needed. since we're removing from
+	   the beginning of header instead of end, we don't have to
+	   worry about multiline-headers. */
+	mbox_sync_move_buffer(ctx, start_pos, 0, *size);
+	if (last_line_pos <= start_pos + *size)
+		last_line_pos = start_pos;
+	else
+		last_line_pos -= *size;
+	data_size -= *size;
+
+	*size = 0;
+
+	if (ctx->mail.space < (off_t)(data_size - last_line_pos)) {
+		ctx->mail.space = data_size - last_line_pos;
+		ctx->mail.offset = ctx->hdr_offset;
+		if (ctx->mail.space > 0)
+			ctx->mail.offset += last_line_pos;
+	}
+}
+
+static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx,
+					   size_t size)
+{
+	static enum header_position space_positions[] = {
+                MBOX_HDR_X_UID,
+                MBOX_HDR_X_KEYWORDS,
+                MBOX_HDR_X_IMAPBASE
+	};
+        enum header_position pos;
+	int i;
+
+	ctx->mail.space = 0;
+	ctx->mail.offset = ctx->hdr_offset;
+
+	for (i = 0; i < 3 && size > 0; i++) {
+		pos = space_positions[i];
+		if (ctx->hdr_pos[pos] != SIZE_MAX) {
+			mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos],
+						      &size);
+		}
+	}
+
+	/* FIXME: see if we could remove X-Keywords header completely */
+}
+
+static void mbox_sync_first_mail_written(struct mbox_sync_mail_context *ctx,
+					 uoff_t hdr_offset)
+{
+	/* we wrote the first mail. update last-uid offset so we can find
+	   it later */
+	i_assert(ctx->last_uid_value_start_pos != 0);
+	i_assert(ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] != SIZE_MAX);
+
+	ctx->sync_ctx->base_uid_last_offset = hdr_offset +
+		ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] +
+		ctx->last_uid_value_start_pos;
+
+	if (ctx->imapbase_updated) {
+		/* update so a) we don't try to update it later needlessly,
+		   b) if we do actually update it, we see the correct value */
+		ctx->sync_ctx->base_uid_last = ctx->last_uid_updated_value;
+	}
+}
+
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff)
+{
+        struct mbox_sync_context *sync_ctx = ctx->sync_ctx;
+	size_t old_hdr_size, new_hdr_size;
+
+	i_assert(sync_ctx->mbox->mbox_lock_type == F_WRLCK);
+
+	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
+	new_hdr_size = str_len(ctx->header);
+
+	if (new_hdr_size <= old_hdr_size) {
+		/* add space. note that we must call add_space() even if we're
+		   not adding anything so mail.offset gets fixed. */
+		mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
+	} else if (new_hdr_size > old_hdr_size) {
+		/* try removing the space where we can */
+		mbox_sync_headers_remove_space(ctx,
+					       new_hdr_size - old_hdr_size);
+		new_hdr_size = str_len(ctx->header);
+
+		if (new_hdr_size <= old_hdr_size) {
+			/* good, we removed enough. */
+			i_assert(new_hdr_size == old_hdr_size);
+		} else if (move_diff < 0 &&
+			   new_hdr_size - old_hdr_size <= (uoff_t)-move_diff) {
+			/* moving backwards - we can use the extra space from
+			   it, just update expunged_space accordingly */
+			i_assert(ctx->mail.space == 0);
+			i_assert(sync_ctx->expunged_space >=
+				 (off_t)(new_hdr_size - old_hdr_size));
+			sync_ctx->expunged_space -= new_hdr_size - old_hdr_size;
+		} else {
+			/* couldn't get enough space */
+			i_assert(ctx->mail.space == 0);
+			ctx->mail.space =
+				-(ssize_t)(new_hdr_size - old_hdr_size);
+			return 0;
+		}
+	}
+
+	i_assert(ctx->mail.space >= 0);
+
+	if (ctx->header_first_change == SIZE_MAX && move_diff == 0) {
+		/* no changes actually. we get here if index sync record told
+		   us to do something that was already there */
+		return 1;
+	}
+
+	if (move_diff != 0) {
+		/* forget about partial write optimizations */
+		ctx->header_first_change = 0;
+		ctx->header_last_change = 0;
+	}
+
+	if (ctx->header_last_change != SIZE_MAX &&
+	    ctx->header_last_change != 0)
+		str_truncate(ctx->header, ctx->header_last_change);
+
+	if (pwrite_full(sync_ctx->write_fd,
+			str_data(ctx->header) + ctx->header_first_change,
+			str_len(ctx->header) - ctx->header_first_change,
+			(off_t)ctx->hdr_offset + (off_t)ctx->header_first_change +
+			move_diff) < 0) {
+		mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
+		return -1;
+	}
+
+	if (sync_ctx->dest_first_mail &&
+	    (ctx->imapbase_updated || ctx->sync_ctx->base_uid_last != 0)) {
+		/* the position might have moved as a result of moving
+		   whitespace */
+		mbox_sync_first_mail_written(ctx, (off_t)ctx->hdr_offset + move_diff);
+	}
+
+	mbox_sync_file_updated(sync_ctx, FALSE);
+	return 1;
+}
+
+static int mbox_sync_read_next(struct mbox_sync_context *sync_ctx,
+			       struct mbox_sync_mail_context *mail_ctx,
+			       struct mbox_sync_mail *mails,
+			       uint32_t seq, uint32_t idx,
+			       uoff_t expunged_space)
+{
+	unsigned int first_mail_expunge_extra;
+	uint32_t orig_next_uid;
+
+	i_zero(mail_ctx);
+	mail_ctx->sync_ctx = sync_ctx;
+	mail_ctx->seq = seq;
+	mail_ctx->header = sync_ctx->header;
+
+	if (istream_raw_mbox_get_header_offset(sync_ctx->input,
+					       &mail_ctx->mail.offset) < 0) {
+		mbox_sync_set_critical(sync_ctx,
+			"Couldn't get header offset for seq=%u", seq);
+		return -1;
+	}
+	mail_ctx->mail.body_size = mails[idx].body_size;
+
+	orig_next_uid = sync_ctx->next_uid;
+	if (mails[idx].uid != 0) {
+		/* This will force the UID to be the one that we originally
+		   assigned to it, regardless of whether it's broken or not in
+		   the file. */
+		sync_ctx->next_uid = mails[idx].uid;
+		sync_ctx->prev_msg_uid = mails[idx].uid - 1;
+	} else {
+		/* Pseudo mail shouldn't have X-UID header at all */
+		i_assert(mails[idx].pseudo);
+		sync_ctx->prev_msg_uid = 0;
+	}
+
+	first_mail_expunge_extra = 1 +
+		(sync_ctx->first_mail_crlf_expunged ? 1 : 0);
+	if (mails[idx].from_offset +
+	    first_mail_expunge_extra - expunged_space != 0) {
+		sync_ctx->dest_first_mail = mails[idx].from_offset == 0;
+	} else {
+		/* we need to skip over the initial \n (it's already counted in
+		   expunged_space) */
+		sync_ctx->dest_first_mail = TRUE;
+		mails[idx].from_offset += first_mail_expunge_extra;
+	}
+
+	if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0)
+		return -1;
+	i_assert(mail_ctx->mail.pseudo == mails[idx].pseudo);
+
+	/* set next_uid back before updating the headers. this is important
+	   if we're updating the first message to make X-IMAP[base] header
+	   have the correct value. */
+	sync_ctx->next_uid = orig_next_uid;
+
+	if (mails[idx].space != 0) {
+		if (mails[idx].space < 0) {
+			/* remove all possible spacing before updating */
+			mbox_sync_headers_remove_space(mail_ctx, SIZE_MAX);
+		}
+		mbox_sync_update_header_from(mail_ctx, &mails[idx]);
+	} else {
+		/* updating might just try to add headers and mess up our
+		   calculations completely. so only add the EOH here. */
+		if (mail_ctx->have_eoh)
+			str_append_c(mail_ctx->header, '\n');
+	}
+	return 0;
+}
+
+static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx,
+                                   struct mbox_sync_mail_context *mail_ctx,
+				   struct mbox_sync_mail *mails,
+				   uint32_t seq, uint32_t idx, uint32_t padding,
+				   off_t move_diff, uoff_t expunged_space,
+				   uoff_t end_offset, bool first_nonexpunged)
+{
+	struct mbox_sync_mail_context new_mail_ctx;
+	uoff_t offset, dest_offset;
+	size_t need_space;
+
+	if (mail_ctx == NULL) {
+		if (mbox_sync_seek(sync_ctx, mails[idx].from_offset) < 0)
+			return -1;
+
+		if (mbox_sync_read_next(sync_ctx, &new_mail_ctx, mails, seq, idx,
+					expunged_space) < 0)
+			return -1;
+		mail_ctx = &new_mail_ctx;
+	} else {
+		i_assert(seq == mail_ctx->seq);
+		if (mail_ctx->mail.space < 0)
+			mail_ctx->mail.space = 0;
+		i_stream_seek(sync_ctx->input, mail_ctx->body_offset);
+	}
+
+	if (mail_ctx->mail.space <= 0) {
+		need_space = str_len(mail_ctx->header) - mail_ctx->mail.space -
+			(mail_ctx->body_offset - mail_ctx->hdr_offset);
+		if (need_space != (uoff_t)-mails[idx].space) {
+			/* this check works only if we're doing the first
+			   write, or if the file size was changed externally */
+			mbox_sync_file_update_ext_modified(sync_ctx);
+
+			mbox_sync_set_critical(sync_ctx,
+				"seq=%u uid=%u uid_broken=%d "
+				"originally needed %"PRIuUOFF_T
+				" bytes, now needs %zu bytes",
+				seq, mails[idx].uid, mails[idx].uid_broken ? 1 : 0,
+				(uoff_t)-mails[idx].space, need_space);
+			return -1;
+		}
+	}
+
+	if (first_nonexpunged && expunged_space > 0) {
+		/* move From-line (after parsing headers so we don't
+		   overwrite them) */
+		i_assert(mails[idx].from_offset >= expunged_space);
+		if (mbox_move(sync_ctx, mails[idx].from_offset - expunged_space,
+			      mails[idx].from_offset,
+			      mails[idx].offset - mails[idx].from_offset) < 0)
+			return -1;
+	}
+
+	if (mails[idx].space == 0) {
+		/* don't touch spacing */
+	} else if (padding < (uoff_t)mail_ctx->mail.space) {
+		mbox_sync_headers_remove_space(mail_ctx, mail_ctx->mail.space -
+					       padding);
+	} else {
+		mbox_sync_headers_add_space(mail_ctx, padding -
+					    mail_ctx->mail.space);
+	}
+
+	/* move the body of this message and headers of next message forward,
+	   then write the headers */
+	offset = sync_ctx->input->v_offset;
+	dest_offset = offset + move_diff;
+	i_assert(offset <= end_offset);
+	if (mbox_move(sync_ctx, dest_offset, offset, end_offset - offset) < 0)
+		return -1;
+
+	/* the header may actually be moved backwards if there was expunged
+	   space which we wanted to remove */
+	i_assert(dest_offset >= str_len(mail_ctx->header));
+	dest_offset -= str_len(mail_ctx->header);
+	i_assert(dest_offset >= mails[idx].from_offset - expunged_space);
+	if (pwrite_full(sync_ctx->write_fd, str_data(mail_ctx->header),
+			str_len(mail_ctx->header), dest_offset) < 0) {
+		mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
+		return -1;
+	}
+	mbox_sync_file_updated(sync_ctx, TRUE);
+
+	if (sync_ctx->dest_first_mail) {
+		mbox_sync_first_mail_written(mail_ctx, dest_offset);
+		sync_ctx->dest_first_mail = FALSE;
+	}
+
+	mails[idx].offset = dest_offset +
+		(mail_ctx->mail.offset - mail_ctx->hdr_offset);
+	mails[idx].space = mail_ctx->mail.space;
+	return 0;
+}
+
+int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx,
+		      struct mbox_sync_mail_context *mail_ctx,
+		      uoff_t end_offset, off_t move_diff, uoff_t extra_space,
+		      uint32_t first_seq, uint32_t last_seq)
+{
+	struct mbox_sync_mail *mails;
+	uoff_t offset, dest_offset, next_end_offset, next_move_diff;
+	uoff_t start_offset, expunged_space;
+	uint32_t idx, first_nonexpunged_idx, padding_per_mail;
+	uint32_t orig_prev_msg_uid;
+	unsigned int count;
+	int ret = 0;
+
+	i_assert(extra_space < OFF_T_MAX);
+	i_assert(sync_ctx->mbox->mbox_lock_type == F_WRLCK);
+
+	mails = array_get_modifiable(&sync_ctx->mails, &count);
+	i_assert(count == last_seq - first_seq + 1);
+
+	/* if there's expunges in mails[], we would get more correct balancing
+	   by counting only them here. however, that might make us overwrite
+	   data which hasn't yet been copied backwards. to avoid too much
+	   complexity, we just leave all the rest of the extra space to first
+	   mail */
+	idx = last_seq - first_seq + 1;
+	padding_per_mail = extra_space / idx;
+
+	/* after expunge the next mail must have been missing space, or we
+	   would have moved it backwards already */
+	expunged_space = 0;
+	start_offset = mails[0].from_offset;
+	for (first_nonexpunged_idx = 0;; first_nonexpunged_idx++) {
+		i_assert(first_nonexpunged_idx != idx);
+		if (!mails[first_nonexpunged_idx].expunged)
+			break;
+                expunged_space += mails[first_nonexpunged_idx].space;
+	}
+	i_assert(mails[first_nonexpunged_idx].space < 0);
+
+	orig_prev_msg_uid = sync_ctx->prev_msg_uid;
+
+	/* start moving backwards. */
+	while (idx > first_nonexpunged_idx) {
+		idx--;
+		if (idx == first_nonexpunged_idx) {
+			/* give the rest of the extra space to first mail.
+			   we might also have to move the mail backwards to
+			   fill the expunged space */
+			padding_per_mail = move_diff + (off_t)expunged_space +
+				(off_t)mails[idx].space;
+		}
+
+		next_end_offset = mails[idx].offset;
+
+		if (mails[idx].space <= 0 && !mails[idx].expunged) {
+			/* give space to this mail. end_offset is left to
+			   contain this message's From-line (ie. below we
+			   move only headers + body). */
+			bool first_nonexpunged = idx == first_nonexpunged_idx;
+
+			next_move_diff = -mails[idx].space;
+			if (mbox_sync_read_and_move(sync_ctx, mail_ctx, mails,
+						    first_seq + idx, idx,
+						    padding_per_mail,
+						    move_diff, expunged_space,
+						    end_offset,
+						    first_nonexpunged) < 0) {
+				ret = -1;
+				break;
+			}
+			move_diff -= next_move_diff + mails[idx].space;
+		} else {
+			/* this mail provides more space. just move it forward
+			   from the extra space offset and set end_offset to
+			   point to beginning of extra space. that way the
+			   header will be moved along with previous mail's
+			   body.
+
+			   if this is expunged mail, we're moving following
+			   mail's From-line and maybe headers. */
+			offset = mails[idx].offset + mails[idx].space;
+			dest_offset = offset + move_diff;
+			i_assert(offset <= end_offset);
+			if (mbox_move(sync_ctx, dest_offset, offset,
+				      end_offset - offset) < 0) {
+				ret = -1;
+				break;
+			}
+
+			move_diff += mails[idx].space;
+			if (!mails[idx].expunged) {
+				move_diff -= padding_per_mail;
+				mails[idx].space = padding_per_mail;
+
+				if (mbox_fill_space(sync_ctx, move_diff +
+						    mails[idx].offset,
+						    padding_per_mail) < 0) {
+					ret = -1;
+					break;
+				}
+			}
+			mails[idx].offset += move_diff;
+		}
+		mail_ctx = NULL;
+
+		i_assert(move_diff >= 0 || idx == first_nonexpunged_idx);
+		i_assert(next_end_offset <= end_offset);
+
+		end_offset = next_end_offset;
+		mails[idx].from_offset += move_diff;
+	}
+
+	if (ret == 0) {
+		i_assert(mails[idx].from_offset == start_offset);
+		i_assert(move_diff + (off_t)expunged_space >= 0);
+	}
+
+	mbox_sync_file_updated(sync_ctx, FALSE);
+	sync_ctx->prev_msg_uid = orig_prev_msg_uid;
+	return ret;
+}
diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c
new file mode 100644
index 0000000..8442013
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-sync-update.c
@@ -0,0 +1,466 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "str.h"
+#include "message-parser.h"
+#include "index-storage.h"
+#include "index-sync-changes.h"
+#include "mbox-storage.h"
+#include "mbox-sync-private.h"
+
+/* Line length when to wrap X-IMAP, X-IMAPbase and X-Keywords headers to next
+   line. Keep this pretty long, as after we wrap we lose compatibility with
+   UW-IMAP */
+#define KEYWORD_WRAP_LINE_LENGTH 1024
+
+static void status_flags_append(struct mbox_sync_mail_context *ctx,
+				const struct mbox_flag_type *flags_list)
+{
+	int i;
+
+	for (i = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			str_append_c(ctx->header, flags_list[i].chr);
+	}
+}
+
+void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx,
+			   size_t pos, size_t need, size_t have)
+{
+	ssize_t diff = (ssize_t)need - (ssize_t)have;
+	int i;
+
+	i_assert(have < SSIZE_T_MAX);
+
+	if (diff == 0) {
+		if (ctx->header_last_change < pos + have ||
+		    ctx->header_last_change == SIZE_MAX)
+			ctx->header_last_change = pos + have;
+	} else {
+		/* FIXME: if (diff < ctx->space && pos < ctx->offset) then
+		   move the data only up to space offset and give/take the
+		   space from there. update header_last_change accordingly.
+		   (except pos and offset can't be compared directly) */
+		ctx->header_last_change = SIZE_MAX;
+		for (i = 0; i < MBOX_HDR_COUNT; i++) {
+			if (ctx->hdr_pos[i] > pos &&
+			    ctx->hdr_pos[i] != SIZE_MAX)
+				ctx->hdr_pos[i] = (ssize_t)ctx->hdr_pos[i] + diff;
+		}
+
+		if (ctx->mail.space > 0) {
+			i_assert(ctx->mail.offset + ctx->mail.space <=
+				 ctx->hdr_offset + pos ||
+				 ctx->mail.offset > ctx->hdr_offset + pos + have);
+			if (ctx->mail.offset > ctx->hdr_offset + pos) {
+				/* free space offset moves */
+				ctx->mail.offset = (ssize_t)ctx->mail.offset + diff;
+			}
+		}
+
+		if (diff < 0)
+			str_delete(ctx->header, pos, -diff);
+		else {
+			ctx->header_last_change = SIZE_MAX;
+			buffer_copy(ctx->header, pos + diff,
+				    ctx->header, pos, SIZE_MAX);
+		}
+	}
+}
+
+static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos,
+				 const struct mbox_flag_type *flags_list)
+{
+	unsigned char *data;
+	size_t size;
+	int i, need, have;
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+
+	if (ctx->header_first_change > pos)
+		ctx->header_first_change = pos;
+
+	/* how many bytes do we need? */
+	for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			need++;
+	}
+
+	/* how many bytes do we have now? */
+	data = buffer_get_modifiable_data(ctx->header, &size);
+	for (have = 0; pos < size; pos++) {
+		if (data[pos] == '\n' || data[pos] == '\r')
+			break;
+
+		/* see if this is unknown flag for us */
+		for (i = 0; flags_list[i].chr != 0; i++) {
+			if (flags_list[i].chr == (char)data[pos])
+				break;
+		}
+
+		if (flags_list[i].chr != 0)
+			have++;
+		else {
+			/* save this one */
+			data[pos-have] = data[pos];
+		}
+	}
+	pos -= have;
+        mbox_sync_move_buffer(ctx, pos, need, have);
+
+	/* @UNSAFE */
+	data = buffer_get_space_unsafe(ctx->header, pos, need);
+	for (i = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			*data++ = flags_list[i].chr;
+	}
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+}
+
+static void
+keywords_append(struct mbox_sync_context *sync_ctx, string_t *dest,
+		const ARRAY_TYPE(keyword_indexes) *keyword_indexes_arr)
+{
+	struct index_mailbox_context *ibox =
+		INDEX_STORAGE_CONTEXT(&sync_ctx->mbox->box);
+	const char *const *keyword_names;
+	const unsigned int *keyword_indexes;
+	unsigned int i, idx_count, keywords_count;
+	size_t last_break;
+
+	keyword_names = array_get(ibox->keyword_names,
+				  &keywords_count);
+	keyword_indexes = array_get(keyword_indexes_arr, &idx_count);
+
+	for (i = 0, last_break = str_len(dest); i < idx_count; i++) {
+		i_assert(keyword_indexes[i] < keywords_count);
+
+		/* wrap the line whenever it gets too long */
+		if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH) {
+			if (i > 0)
+				str_append_c(dest, ' ');
+		} else {
+			str_append(dest, "\n\t");
+			last_break = str_len(dest);
+		}
+		str_append(dest, keyword_names[keyword_indexes[i]]);
+	}
+}
+
+static void
+keywords_append_all(struct mbox_sync_mail_context *ctx, string_t *dest,
+		    size_t startpos)
+{
+	struct index_mailbox_context *ibox =
+		INDEX_STORAGE_CONTEXT(&ctx->sync_ctx->mbox->box);
+	const char *const *names;
+	const unsigned char *p;
+	unsigned int i, count;
+	size_t last_break;
+
+	p = str_data(dest);
+	if (str_len(dest) - startpos < KEYWORD_WRAP_LINE_LENGTH)
+		last_break = startpos;
+	else {
+		/* set last_break to beginning of line */
+		for (last_break = str_len(dest); last_break > 0; last_break--) {
+			if (p[last_break-1] == '\n')
+				break;
+		}
+	}
+
+	names = array_get(ibox->keyword_names, &count);
+	for (i = 0; i < count; i++) {
+		/* wrap the line whenever it gets too long */
+		if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH)
+			str_append_c(dest, ' ');
+		else {
+			str_append(dest, "\n\t");
+			last_break = str_len(dest);
+		}
+		str_append(dest, names[i]);
+	}
+}
+
+static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx)
+{
+	size_t new_hdr_size, startpos;
+
+	new_hdr_size = str_len(ctx->header);
+	if (new_hdr_size > 0 &&
+	    str_data(ctx->header)[new_hdr_size-1] != '\n') {
+		/* broken header - doesn't end with \n. fix it. */
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->sync_ctx->dest_first_mail &&
+	    ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX) {
+		i_assert(ctx->sync_ctx->base_uid_validity != 0);
+
+		str_append(ctx->header, "X-IMAPbase: ");
+		ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
+		/* startpos must start from identical position as when
+		   updating */
+		startpos = str_len(ctx->header);
+		str_printfa(ctx->header, "%u ",
+			    ctx->sync_ctx->base_uid_validity);
+
+		ctx->last_uid_updated_value = ctx->sync_ctx->next_uid-1;
+		ctx->last_uid_value_start_pos = str_len(ctx->header) -
+			ctx->hdr_pos[MBOX_HDR_X_IMAPBASE];
+		ctx->imapbase_updated = TRUE;
+		str_printfa(ctx->header, "%010u", ctx->last_uid_updated_value);
+
+		keywords_append_all(ctx, ctx->header, startpos);
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX && !ctx->mail.pseudo) {
+		str_append(ctx->header, "X-UID: ");
+		ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
+		str_printfa(ctx->header, "%u\n", ctx->mail.uid);
+	}
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+
+	if (ctx->hdr_pos[MBOX_HDR_STATUS] == SIZE_MAX &&
+	    (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) {
+		str_append(ctx->header, "Status: ");
+		ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
+		status_flags_append(ctx, mbox_status_flags);
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == SIZE_MAX &&
+	    (ctx->mail.flags & XSTATUS_FLAGS_MASK) != 0) {
+		str_append(ctx->header, "X-Status: ");
+		ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header);
+		status_flags_append(ctx, mbox_xstatus_flags);
+		str_append_c(ctx->header, '\n');
+	}
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+
+	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX &&
+	    array_is_created(&ctx->mail.keywords) &&
+	    array_count(&ctx->mail.keywords) > 0) {
+		str_append(ctx->header, "X-Keywords: ");
+		ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
+		keywords_append(ctx->sync_ctx, ctx->header,
+				&ctx->mail.keywords);
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->content_length == UOFF_T_MAX &&
+	    ctx->mail.body_size >= MBOX_MIN_CONTENT_LENGTH_SIZE) {
+		str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n",
+			    ctx->mail.body_size);
+	}
+
+	if (str_len(ctx->header) != new_hdr_size) {
+		if (ctx->header_first_change == SIZE_MAX)
+			ctx->header_first_change = new_hdr_size;
+		ctx->header_last_change = SIZE_MAX;
+	}
+
+	if (ctx->have_eoh)
+		str_append_c(ctx->header, '\n');
+}
+
+static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx)
+{
+	if (ctx->hdr_pos[MBOX_HDR_STATUS] != SIZE_MAX) {
+		status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_STATUS],
+				     mbox_status_flags);
+	}
+}
+
+static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx)
+{
+	if (ctx->hdr_pos[MBOX_HDR_X_STATUS] != SIZE_MAX) {
+		status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_X_STATUS],
+				     mbox_xstatus_flags);
+	}
+}
+
+static void mbox_sync_update_line(struct mbox_sync_mail_context *ctx,
+				  size_t pos, string_t *new_line)
+{
+	const char *hdr, *p;
+	uoff_t file_pos;
+
+	if (ctx->header_first_change > pos)
+		ctx->header_first_change = pos;
+
+	/* set p = end of header, handle also wrapped headers */
+	hdr = p = str_c(ctx->header) + pos;
+	for (;;) {
+		p = strchr(p, '\n');
+		if (p == NULL) {
+			/* shouldn't really happen, but allow anyway.. */
+			p = hdr + strlen(hdr);
+			break;
+		}
+		if (p[1] != '\t' && p[1] != ' ')
+			break;
+		p += 2;
+	}
+
+	file_pos = pos + ctx->hdr_offset;
+	if (ctx->mail.space > 0 && ctx->mail.offset >= file_pos &&
+	    ctx->mail.offset < file_pos + (p - hdr)) {
+		/* extra space points to this line. remove it. */
+		ctx->mail.offset = ctx->hdr_offset;
+		ctx->mail.space = 0;
+	}
+
+	mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1);
+	buffer_copy(ctx->header, pos, new_line, 0, SIZE_MAX);
+}
+
+static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx)
+{
+	string_t *str;
+
+	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX)
+		return;
+
+	str = t_str_new(256);
+	if (array_is_created(&ctx->mail.keywords))
+		keywords_append(ctx->sync_ctx, str, &ctx->mail.keywords);
+	str_append_c(str, '\n');
+	mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_KEYWORDS], str);
+}
+
+static void mbox_sync_update_x_imap_base(struct mbox_sync_mail_context *ctx)
+{
+	struct mbox_sync_context *sync_ctx = ctx->sync_ctx;
+	string_t *str;
+
+	i_assert(sync_ctx->base_uid_validity != 0);
+
+	if (!sync_ctx->dest_first_mail ||
+	    ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX)
+		return;
+
+	if (!ctx->imapbase_rewrite) {
+		/* uid-last might need updating, but we'll do it later by
+		   writing it directly where needed. */
+		return;
+	}
+
+	/* a) keyword list changed, b) uid-last didn't use 10 digits */
+	str = t_str_new(200);
+	str_printfa(str, "%u ", sync_ctx->base_uid_validity);
+
+	ctx->last_uid_updated_value = sync_ctx->next_uid-1;
+	ctx->last_uid_value_start_pos = str_len(str);
+	ctx->imapbase_updated = TRUE;
+	str_printfa(str, "%010u", ctx->last_uid_updated_value);
+
+	keywords_append_all(ctx, str, 0);
+	str_append_c(str, '\n');
+
+        mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_IMAPBASE], str);
+}
+
+static void mbox_sync_update_x_uid(struct mbox_sync_mail_context *ctx)
+{
+	string_t *str;
+
+	if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX ||
+	    ctx->mail.uid == ctx->parsed_uid)
+		return;
+
+	str = t_str_new(64);
+	str_printfa(str, "%u\n", ctx->mail.uid);
+	mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_UID], str);
+}
+
+static void mbox_sync_update_header_real(struct mbox_sync_mail_context *ctx)
+{
+	i_assert(ctx->mail.uid != 0 || ctx->mail.pseudo);
+
+	if (!ctx->sync_ctx->keep_recent)
+		ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT);
+
+	mbox_sync_update_status(ctx);
+	mbox_sync_update_xstatus(ctx);
+	mbox_sync_update_xkeywords(ctx);
+
+	mbox_sync_update_x_imap_base(ctx);
+	mbox_sync_update_x_uid(ctx);
+
+	mbox_sync_add_missing_headers(ctx);
+	ctx->updated = TRUE;
+}
+
+void mbox_sync_update_header(struct mbox_sync_mail_context *ctx)
+{
+	T_BEGIN {
+		mbox_sync_update_header_real(ctx);
+	} T_END;
+}
+
+static void
+mbox_sync_update_header_from_real(struct mbox_sync_mail_context *ctx,
+				  const struct mbox_sync_mail *mail)
+{
+	if (mail->status_broken ||
+	    (ctx->mail.flags & STATUS_FLAGS_MASK) !=
+	    (mail->flags & STATUS_FLAGS_MASK) ||
+	    (ctx->mail.flags & MAIL_RECENT) != 0) {
+		ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(STATUS_FLAGS_MASK)) |
+			(mail->flags & STATUS_FLAGS_MASK);
+		if (!ctx->sync_ctx->keep_recent)
+                        ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT);
+		mbox_sync_update_status(ctx);
+	}
+	if (mail->xstatus_broken ||
+	    (ctx->mail.flags & XSTATUS_FLAGS_MASK) !=
+	    (mail->flags & XSTATUS_FLAGS_MASK)) {
+		ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(XSTATUS_FLAGS_MASK)) |
+			(mail->flags & XSTATUS_FLAGS_MASK);
+		mbox_sync_update_xstatus(ctx);
+	}
+	if (!array_is_created(&mail->keywords) ||
+	    array_count(&mail->keywords) == 0) {
+		/* no keywords for this mail */
+		if (array_is_created(&ctx->mail.keywords)) {
+			array_clear(&ctx->mail.keywords);
+			mbox_sync_update_xkeywords(ctx);
+		}
+	} else if (!array_is_created(&ctx->mail.keywords)) {
+		/* adding first keywords */
+		p_array_init(&ctx->mail.keywords,
+			     ctx->sync_ctx->mail_keyword_pool,
+			     array_count(&mail->keywords));
+		array_append_array(&ctx->mail.keywords,
+				   &mail->keywords);
+		mbox_sync_update_xkeywords(ctx);
+	} else if (!array_cmp(&ctx->mail.keywords, &mail->keywords)) {
+		/* keywords changed. */
+		array_clear(&ctx->mail.keywords);
+		array_append_array(&ctx->mail.keywords,
+				   &mail->keywords);
+		mbox_sync_update_xkeywords(ctx);
+	}
+
+	i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid);
+	ctx->mail.uid = mail->uid;
+
+	mbox_sync_update_x_imap_base(ctx);
+	mbox_sync_update_x_uid(ctx);
+	mbox_sync_add_missing_headers(ctx);
+}
+
+void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
+				  const struct mbox_sync_mail *mail)
+{
+	T_BEGIN {
+		mbox_sync_update_header_from_real(ctx, mail);
+	} T_END;
+}
diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c
new file mode 100644
index 0000000..0d2aa7f
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-sync.c
@@ -0,0 +1,2066 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+/*
+   Modifying mbox can be slow, so we try to do it all at once minimizing the
+   required disk I/O. We may need to:
+
+   - Update message flags in Status, X-Status and X-Keywords headers
+   - Write missing X-UID and X-IMAPbase headers
+   - Write missing or broken Content-Length header if there's space
+   - Expunge specified messages
+
+   Here's how we do it:
+
+   - Start reading the mails from the beginning
+   - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
+     of them, remember how much each message has and offset to beginning of the
+     padding
+   - If header needs to be rewritten and there's enough space, do it
+       - If we didn't have enough space, remember how much was missing
+   - Continue reading and counting the padding in each message. If available
+     padding is enough to rewrite all the previous messages needing it, do it
+   - When we encounter expunged message, treat all of it as padding and
+     rewrite previous messages if needed (and there's enough space).
+     Afterwards keep moving messages backwards to fill the expunged space.
+     Moving is done by rewriting each message's headers, with possibly adding
+     missing Content-Length header and padding. Message bodies are moved
+     without modifications.
+   - If we encounter end of file, grow the file and rewrite needed messages
+   - Rewriting is done by moving message body forward, rewriting message's
+     header and doing the same for previous message, until all of them are
+     rewritten.
+*/
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "buffer.h"
+#include "hostpid.h"
+#include "istream.h"
+#include "file-set-size.h"
+#include "str.h"
+#include "read-full.h"
+#include "write-full.h"
+#include "sleep.h"
+#include "message-date.h"
+#include "istream-raw-mbox.h"
+#include "mbox-storage.h"
+#include "index-sync-changes.h"
+#include "mailbox-uidvalidity.h"
+#include "mailbox-recent-flags.h"
+#include "mbox-from.h"
+#include "mbox-file.h"
+#include "mbox-lock.h"
+#include "mbox-sync-private.h"
+
+#include <stddef.h>
+#include <utime.h>
+#include <sys/stat.h>
+
+/* The text below was taken exactly as c-client wrote it to my mailbox,
+   so it's probably copyrighted by University of Washington. */
+#define PSEUDO_MESSAGE_BODY \
+"This text is part of the internal format of your mail folder, and is not\n" \
+"a real message.  It is created automatically by the mail system software.\n" \
+"If deleted, important folder data will be lost, and it will be re-created\n" \
+"with the data reset to initial values.\n"
+
+void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx,
+			    const char *fmt, ...)
+{
+	va_list va;
+
+	sync_ctx->errors = TRUE;
+	if (sync_ctx->ext_modified) {
+		mailbox_set_critical(&sync_ctx->mbox->box,
+			"mbox was modified while we were syncing, "
+			"check your locking settings");
+	}
+
+	va_start(va, fmt);
+	mailbox_set_critical(&sync_ctx->mbox->box,
+			     "Sync failed for mbox: %s",
+			     t_strdup_vprintf(fmt, va));
+	va_end(va);
+}
+
+int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
+{
+	if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
+		mbox_sync_set_critical(sync_ctx,
+			"Unexpectedly lost From-line at offset %"PRIuUOFF_T,
+			from_offset);
+		return -1;
+	}
+	return 0;
+}
+
+void mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx)
+{
+	struct stat st;
+
+	/* Do this even if ext_modified is already set. Expunging code relies
+	   on last_stat being updated. */
+	if (fstat(sync_ctx->write_fd, &st) < 0) {
+		mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
+		return;
+	}
+
+	if (st.st_size != sync_ctx->last_stat.st_size ||
+	    (sync_ctx->last_stat.st_mtime != 0 &&
+	     !CMP_ST_MTIME(&st, &sync_ctx->last_stat)))
+		sync_ctx->ext_modified = TRUE;
+
+	sync_ctx->last_stat = st;
+}
+
+void mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty)
+{
+	if (dirty) {
+		/* just mark the stat as dirty. */
+		sync_ctx->last_stat.st_mtime = 0;
+		return;
+	}
+	if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0)
+		mbox_set_syscall_error(sync_ctx->mbox, "fstat()");
+	i_stream_sync(sync_ctx->input);
+}
+
+static int
+mbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
+			 struct mbox_sync_mail_context *mail_ctx)
+{
+	uoff_t offset;
+
+	/* get EOF */
+	(void)istream_raw_mbox_get_header_offset(sync_ctx->input, &offset);
+	if (istream_raw_mbox_is_eof(sync_ctx->input))
+		return 0;
+
+	p_clear(sync_ctx->mail_keyword_pool);
+	i_zero(mail_ctx);
+	mail_ctx->sync_ctx = sync_ctx;
+	mail_ctx->seq = ++sync_ctx->seq;
+	mail_ctx->header = sync_ctx->header;
+
+	mail_ctx->mail.from_offset =
+		istream_raw_mbox_get_start_offset(sync_ctx->input);
+	if (istream_raw_mbox_get_header_offset(sync_ctx->input, &mail_ctx->mail.offset) < 0) {
+		mbox_sync_set_critical(sync_ctx,
+			"Couldn't get header offset for seq=%u", mail_ctx->seq);
+		return -1;
+	}
+
+	if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0)
+		return -1;
+	if (istream_raw_mbox_is_corrupted(sync_ctx->input))
+		return -1;
+
+	i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset ||
+		 sync_ctx->input->eof);
+
+	if (istream_raw_mbox_get_body_size(sync_ctx->input,
+					   mail_ctx->content_length,
+					   &mail_ctx->mail.body_size) < 0) {
+		mbox_sync_set_critical(sync_ctx,
+			"Couldn't get body size for seq=%u", mail_ctx->seq);
+		return -1;
+	}
+	i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
+
+	if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 &&
+	    !mail_ctx->mail.pseudo) {
+		if (!sync_ctx->keep_recent) {
+			/* need to add 'O' flag to Status-header */
+			mail_ctx->need_rewrite = TRUE;
+		}
+		mail_ctx->recent = TRUE;
+	}
+	return 1;
+}
+
+static void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
+				       uint32_t uid, bool *sync_expunge_r)
+{
+	guid_128_t expunged_guid_128;
+
+	if (uid == 0 || sync_ctx->index_reset) {
+		/* nothing for this or the future ones */
+		uid = (uint32_t)-1;
+	}
+
+	index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r,
+				expunged_guid_128);
+	if (sync_ctx->readonly) {
+		/* we can't expunge anything from read-only mboxes */
+		*sync_expunge_r = FALSE;
+	}
+}
+
+static bool
+mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
+			 uint32_t uid, const struct mail_index_record **rec_r)
+{
+        const struct mail_index_record *rec = NULL;
+	uint32_t messages_count;
+	bool ret = FALSE;
+
+	if (sync_ctx->index_reset) {
+		*rec_r = NULL;
+		return TRUE;
+	}
+
+	messages_count =
+		mail_index_view_get_messages_count(sync_ctx->sync_view);
+	while (sync_ctx->idx_seq <= messages_count) {
+		rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
+		if (uid <= rec->uid)
+			break;
+
+		/* externally expunged message, remove from index */
+		mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
+                sync_ctx->idx_seq++;
+		rec = NULL;
+	}
+
+	if (rec == NULL && uid < sync_ctx->idx_next_uid) {
+		/* this UID was already in index and it was expunged */
+		mbox_sync_set_critical(sync_ctx,
+			"Expunged message reappeared to mailbox "
+			"(UID %u < %u, seq=%u, idx_msgs=%u)",
+			uid, sync_ctx->idx_next_uid,
+			sync_ctx->seq, messages_count);
+		ret = FALSE; rec = NULL;
+	} else if (rec != NULL && rec->uid != uid) {
+		/* new UID in the middle of the mailbox - shouldn't happen */
+		mbox_sync_set_critical(sync_ctx,
+			"UID inserted in the middle of mailbox "
+			"(%u > %u, seq=%u, idx_msgs=%u)",
+			rec->uid, uid, sync_ctx->seq, messages_count);
+		ret = FALSE; rec = NULL;
+	} else {
+		ret = TRUE;
+	}
+
+	*rec_r = rec;
+	return ret;
+}
+
+static void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
+				     unsigned char hdr_md5_sum[],
+				     const struct mail_index_record **rec_r)
+{
+        const struct mail_index_record *rec = NULL;
+	uint32_t messages_count;
+	const void *data;
+
+	if (sync_ctx->index_reset) {
+		*rec_r = NULL;
+		return;
+	}
+
+	messages_count =
+		mail_index_view_get_messages_count(sync_ctx->sync_view);
+	while (sync_ctx->idx_seq <= messages_count) {
+		rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq);
+		mail_index_lookup_ext(sync_ctx->sync_view,
+				      sync_ctx->idx_seq,
+				      sync_ctx->mbox->md5hdr_ext_idx,
+				      &data, NULL);
+		if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0)
+			break;
+
+		/* externally expunged message, remove from index */
+		mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
+                sync_ctx->idx_seq++;
+		rec = NULL;
+	}
+
+	*rec_r = rec;
+}
+
+static void
+mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
+                             struct mbox_sync_mail *mail,
+			     bool nocheck)
+{
+	const void *data;
+	uint64_t offset;
+
+	if (!nocheck) {
+		/* see if from_offset needs updating */
+		mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
+				      sync_ctx->mbox->mbox_ext_idx,
+				      &data, NULL);
+		if (data != NULL &&
+		    *((const uint64_t *)data) == mail->from_offset)
+			return;
+	}
+
+	offset = mail->from_offset;
+	mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
+			      sync_ctx->mbox->mbox_ext_idx, &offset, NULL);
+}
+
+static void
+mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
+{
+	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	struct mail_index *index = sync_ctx->mbox->box.index;
+	struct mail_keywords *keywords;
+
+	keywords = !array_is_created(&mail_ctx->mail.keywords) ?
+		mail_index_keywords_create(index, NULL) :
+		mail_index_keywords_create_from_indexes(index,
+			&mail_ctx->mail.keywords);
+	mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
+				   MODIFY_REPLACE, keywords);
+	mail_index_keywords_unref(&keywords);
+}
+
+static void
+mbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx)
+{
+        struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	const void *ext_data;
+
+	mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq,
+			      sync_ctx->mbox->md5hdr_ext_idx, &ext_data, NULL);
+	if (ext_data == NULL ||
+	    memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) {
+		mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
+				      sync_ctx->mbox->md5hdr_ext_idx,
+				      mail_ctx->hdr_md5_sum, NULL);
+	}
+}
+
+static void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx,
+				      const struct mail_index_record *rec)
+{
+	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	ARRAY_TYPE(keyword_indexes) idx_keywords;
+	uint8_t idx_flags, mbox_flags;
+
+	/* default to undirtying the message. it gets added back if
+	   flags/keywords don't match what is in the index. */
+	mail_ctx->mail.flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY);
+
+	/* replace flags */
+	idx_flags = rec->flags & MAIL_FLAGS_NONRECENT;
+	mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT;
+	if (idx_flags != mbox_flags) {
+		mail_ctx->need_rewrite = TRUE;
+		mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) |
+			idx_flags | MAIL_INDEX_MAIL_FLAG_DIRTY;
+	}
+
+	/* replace keywords */
+	t_array_init(&idx_keywords, 32);
+	mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq,
+				   &idx_keywords);
+	if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) {
+		mail_ctx->need_rewrite = TRUE;
+		mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
+
+		if (!array_is_created(&mail_ctx->mail.keywords)) {
+			p_array_init(&mail_ctx->mail.keywords,
+				     sync_ctx->mail_keyword_pool,
+				     array_count(&idx_keywords));
+		}
+		array_clear(&mail_ctx->mail.keywords);
+		array_append_array(&mail_ctx->mail.keywords, &idx_keywords);
+	}
+}
+
+static void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx,
+				   const struct mail_index_record *rec)
+{
+	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	struct mailbox *box = &sync_ctx->mbox->box;
+	struct mbox_sync_mail *mail = &mail_ctx->mail;
+	enum mail_index_sync_type sync_type;
+	ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT;
+	uint8_t flags, orig_flags;
+
+	if (rec != NULL) {
+		if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+			/* flags and keywords are dirty. replace the current
+			   ones from the flags in index file. */
+			mbox_sync_get_dirty_flags(mail_ctx, rec);
+		}
+	}
+
+	flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT;
+	if (array_is_created(&mail->keywords)) {
+		t_array_init(&orig_keywords, 32);
+		array_append_array(&orig_keywords, &mail->keywords);
+	}
+
+	/* apply new changes */
+	index_sync_changes_apply(sync_ctx->sync_changes,
+				 sync_ctx->mail_keyword_pool,
+				 &flags, &mail->keywords, &sync_type);
+	if (flags != orig_flags ||
+	    !index_keyword_array_cmp(&mail->keywords, &orig_keywords)) {
+		mail_ctx->need_rewrite = TRUE;
+		mail->flags = flags | (mail->flags & MAIL_RECENT) |
+			MAIL_INDEX_MAIL_FLAG_DIRTY;
+	}
+	if (sync_type != 0) {
+		mailbox_sync_notify(box, mail_ctx->mail.uid,
+				   index_sync_type_convert(sync_type));
+	}
+}
+
+static void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
+				   const struct mail_index_record *rec)
+{
+	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	struct mbox_sync_mail *mail = &mail_ctx->mail;
+	ARRAY_TYPE(keyword_indexes) idx_keywords;
+	uint8_t mbox_flags;
+
+	mbox_flags = mail->flags & ENUM_NEGATE(MAIL_RECENT);
+	if (!sync_ctx->delay_writes) {
+		/* changes are written to the mbox file */
+		mbox_flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY);
+	} else if (mail_ctx->need_rewrite) {
+		/* make sure this message gets written later */
+		mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
+	}
+
+	if (rec == NULL) {
+		/* new message */
+		mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
+		mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
+					MODIFY_REPLACE, mbox_flags);
+		mbox_sync_update_index_keywords(mail_ctx);
+
+		if (sync_ctx->mbox->mbox_save_md5) {
+			mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
+				sync_ctx->mbox->md5hdr_ext_idx,
+				mail_ctx->hdr_md5_sum, NULL);
+		}
+	} else {
+		if ((rec->flags & MAIL_FLAGS_NONRECENT) !=
+		    (mbox_flags & MAIL_FLAGS_NONRECENT)) {
+			/* flags other than recent/dirty have changed */
+			mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
+						MODIFY_REPLACE, mbox_flags);
+		} else if (((rec->flags ^ mbox_flags) &
+			    MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+			/* only dirty flag state changed */
+			bool dirty;
+
+			dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
+			mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
+				dirty ? MODIFY_ADD : MODIFY_REMOVE,
+				(enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
+		}
+
+		/* see if keywords changed */
+		t_array_init(&idx_keywords, 32);
+		mail_index_lookup_keywords(sync_ctx->sync_view,
+					   sync_ctx->idx_seq, &idx_keywords);
+		if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords))
+			mbox_sync_update_index_keywords(mail_ctx);
+
+		/* see if we need to update md5 sum. */
+		if (sync_ctx->mbox->mbox_save_md5)
+			mbox_sync_update_md5_if_changed(mail_ctx);
+	}
+
+	if (!mail_ctx->recent) {
+		/* Mail has "Status: O" header. No messages before this
+		   can be recent. */
+		sync_ctx->last_nonrecent_uid = mail->uid;
+	}
+
+	/* update from_offsets, but not if we're going to rewrite this message.
+	   rewriting would just move it anyway. */
+	if (sync_ctx->need_space_seq == 0) {
+		bool nocheck = rec == NULL || sync_ctx->expunged_space > 0;
+		mbox_sync_update_from_offset(sync_ctx, mail, nocheck);
+	}
+}
+
+static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
+{
+	struct istream *input = ctx->sync_ctx->file_input;
+	const unsigned char *data;
+	size_t size, from_line_size;
+
+	buffer_set_used_size(ctx->sync_ctx->from_line, 0);
+	from_line_size = ctx->hdr_offset - ctx->mail.from_offset;
+
+	i_stream_seek(input, ctx->mail.from_offset);
+	for (;;) {
+		data = i_stream_get_data(input, &size);
+		if (size >= from_line_size)
+			size = from_line_size;
+
+		buffer_append(ctx->sync_ctx->from_line, data, size);
+		i_stream_skip(input, size);
+		from_line_size -= size;
+
+		if (from_line_size == 0)
+			break;
+
+		if (i_stream_read(input) < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx)
+{
+	unsigned char buf[10];
+	const char *str;
+	uint32_t uid_last;
+	unsigned int i;
+	int ret;
+
+	i_assert(sync_ctx->base_uid_last_offset != 0);
+
+	/* first check that the 10 bytes are there and they're exactly as
+	   expected. just an extra safety check to make sure we never write
+	   to wrong location in the mbox file. */
+	ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf),
+			 sync_ctx->base_uid_last_offset);
+	if (ret < 0) {
+		mbox_set_syscall_error(sync_ctx->mbox, "pread_full()");
+		return -1;
+	}
+	if (ret == 0) {
+		mbox_sync_set_critical(sync_ctx,
+			"X-IMAPbase uid-last offset unexpectedly outside mbox");
+		return -1;
+	}
+
+	for (i = 0, uid_last = 0; i < sizeof(buf); i++) {
+		if (buf[i] < '0' || buf[i] > '9') {
+			uid_last = (uint32_t)-1;
+			break;
+		}
+		uid_last = uid_last * 10 + (buf[i] - '0');
+	}
+
+	if (uid_last != sync_ctx->base_uid_last) {
+		mbox_sync_set_critical(sync_ctx,
+			"X-IMAPbase uid-last unexpectedly lost");
+		return -1;
+	}
+
+	/* and write it */
+	str = t_strdup_printf("%010u", sync_ctx->next_uid - 1);
+	if (pwrite_full(sync_ctx->write_fd, str, 10,
+			sync_ctx->base_uid_last_offset) < 0) {
+		mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()");
+		return -1;
+	}
+	mbox_sync_file_updated(sync_ctx, FALSE);
+
+	sync_ctx->base_uid_last = sync_ctx->next_uid - 1;
+	return 0;
+}
+
+static int
+mbox_write_from_line(struct mbox_sync_mail_context *ctx)
+{
+	string_t *str = ctx->sync_ctx->from_line;
+
+	if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str),
+			ctx->mail.from_offset) < 0) {
+		mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()");
+		return -1;
+	}
+
+	mbox_sync_file_updated(ctx->sync_ctx, FALSE);
+	return 0;
+}
+
+static void update_from_offsets(struct mbox_sync_context *sync_ctx)
+{
+	const struct mbox_sync_mail *mails;
+	unsigned int i, count;
+	uint32_t ext_idx;
+	uint64_t offset;
+
+	ext_idx = sync_ctx->mbox->mbox_ext_idx;
+
+	mails = array_get(&sync_ctx->mails, &count);
+	for (i = 0; i < count; i++) {
+		if (mails[i].idx_seq == 0 || mails[i].expunged)
+			continue;
+
+		sync_ctx->moved_offsets = TRUE;
+		offset = mails[i].from_offset;
+		mail_index_update_ext(sync_ctx->t, mails[i].idx_seq,
+				      ext_idx, &offset, NULL);
+	}
+}
+
+static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
+{
+	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	struct mailbox *box = &sync_ctx->mbox->box;
+
+	mailbox_sync_notify(box, mail_ctx->mail.uid,
+			   MAILBOX_SYNC_TYPE_EXPUNGE);
+	mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq);
+
+	mail_ctx->mail.expunged = TRUE;
+	mail_ctx->mail.offset = mail_ctx->mail.from_offset;
+	mail_ctx->mail.space =
+		mail_ctx->body_offset - mail_ctx->mail.from_offset +
+		mail_ctx->mail.body_size;
+	mail_ctx->mail.body_size = 0;
+	mail_ctx->mail.uid = 0;
+
+	if (sync_ctx->seq == 1) {
+		/* expunging first message, fix space to contain next
+		   message's \n header too since it will be removed. */
+		mail_ctx->mail.space++;
+		if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) {
+			mail_ctx->mail.space++;
+			sync_ctx->first_mail_crlf_expunged = TRUE;
+		}
+
+		/* uid-last offset is invalid now */
+                sync_ctx->base_uid_last_offset = 0;
+	}
+
+	sync_ctx->expunged_space += mail_ctx->mail.space;
+}
+
+static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
+{
+	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	uoff_t orig_from_offset, postlf_from_offset = UOFF_T_MAX;
+	off_t move_diff;
+	int ret;
+
+	if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
+		/* move the header backwards to fill expunged space */
+		move_diff = -sync_ctx->expunged_space;
+
+		orig_from_offset = mail_ctx->mail.from_offset;
+		if (sync_ctx->dest_first_mail) {
+			/* we're moving this mail to beginning of file.
+			   skip the initial \n (it's already counted in
+			   expunged_space) */
+			mail_ctx->mail.from_offset++;
+			if (sync_ctx->first_mail_crlf_expunged)
+				mail_ctx->mail.from_offset++;
+		}
+		postlf_from_offset = mail_ctx->mail.from_offset;
+
+		/* read the From-line before rewriting overwrites it */
+		if (mbox_read_from_line(mail_ctx) < 0)
+			return -1;
+		i_assert((off_t)mail_ctx->mail.from_offset + move_diff != 1 &&
+			 (off_t)mail_ctx->mail.from_offset + move_diff != 2);
+
+		mbox_sync_update_header(mail_ctx);
+		ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
+		if (ret < 0)
+			return -1;
+
+		if (ret > 0) {
+			/* rewrite successful, write From-line to
+			   new location */
+			i_assert((off_t)mail_ctx->mail.from_offset >=
+				 -move_diff);
+			mail_ctx->mail.from_offset = (off_t)mail_ctx->mail.from_offset + move_diff;
+			mail_ctx->mail.offset = (off_t)mail_ctx->mail.offset + move_diff;
+			if (mbox_write_from_line(mail_ctx) < 0)
+				return -1;
+		} else {
+			if (sync_ctx->dest_first_mail) {
+				/* didn't have enough space, move the offset
+				   back so seeking into it doesn't fail */
+				mail_ctx->mail.from_offset = orig_from_offset;
+			}
+		}
+	} else if (mail_ctx->need_rewrite) {
+		mbox_sync_update_header(mail_ctx);
+		if (sync_ctx->delay_writes && sync_ctx->need_space_seq == 0) {
+			/* mark it dirty and do it later. we can't do this
+			   if we're in the middle of rewriting acquiring more
+			   space. */
+			mail_ctx->dirty = TRUE;
+			return 0;
+		}
+
+		if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
+			return -1;
+	} else {
+		/* nothing to do */
+		return 0;
+	}
+
+	if (ret == 0 && sync_ctx->need_space_seq == 0) {
+		/* first mail with no space to write it */
+		sync_ctx->need_space_seq = sync_ctx->seq;
+		sync_ctx->space_diff = 0;
+
+		if (sync_ctx->expunged_space > 0) {
+			/* create dummy message to describe the expunged data */
+			struct mbox_sync_mail mail;
+
+			/* if this is going to be the first mail, increase the
+			   from_offset to point to the beginning of the
+			   From-line, because the previous [CR]LF is already
+			   covered by expunged_space. */
+			i_assert(postlf_from_offset != UOFF_T_MAX);
+			mail_ctx->mail.from_offset = postlf_from_offset;
+
+			i_zero(&mail);
+			mail.expunged = TRUE;
+			mail.offset = mail.from_offset =
+				mail_ctx->mail.from_offset -
+				sync_ctx->expunged_space;
+			mail.space = sync_ctx->expunged_space;
+
+                        sync_ctx->space_diff = sync_ctx->expunged_space;
+			sync_ctx->expunged_space = 0;
+			i_assert(sync_ctx->space_diff < -mail_ctx->mail.space);
+
+			sync_ctx->need_space_seq--;
+			array_push_back(&sync_ctx->mails, &mail);
+		}
+	}
+	return 0;
+}
+
+static int
+mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
+{
+	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	uoff_t end_offset, move_diff, extra_space, needed_space;
+	uint32_t last_seq;
+	ARRAY_TYPE(keyword_indexes) keywords_copy;
+
+	i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 ||
+		 mail_ctx->mail.offset == mail_ctx->hdr_offset);
+
+	if (array_is_created(&mail_ctx->mail.keywords)) {
+		/* mail's keywords are allocated from a pool that's cleared
+		   for each mail. we'll need to copy it to something more
+		   permanent. */
+		p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool,
+			     array_count(&mail_ctx->mail.keywords));
+		array_append_array(&keywords_copy, &mail_ctx->mail.keywords);
+		mail_ctx->mail.keywords = keywords_copy;
+	}
+	array_push_back(&sync_ctx->mails, &mail_ctx->mail);
+
+	sync_ctx->space_diff += mail_ctx->mail.space;
+	if (sync_ctx->space_diff < 0) {
+		if (sync_ctx->expunged_space > 0) {
+			i_assert(sync_ctx->expunged_space ==
+				 mail_ctx->mail.space);
+                        sync_ctx->expunged_space = 0;
+		}
+		return 0;
+	}
+
+	/* we have enough space now */
+	if (mail_ctx->mail.uid == 0) {
+		/* this message was expunged. fill more or less of the space.
+		   space_diff now consists of a negative "bytes needed" sum,
+		   plus the expunged space of this message. so it contains how
+		   many bytes of _extra_ space we have. */
+		i_assert(mail_ctx->mail.space >= sync_ctx->space_diff);
+		extra_space = MBOX_HEADER_PADDING *
+			(sync_ctx->seq - sync_ctx->need_space_seq + 1);
+		needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
+		if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
+			/* don't waste too much on padding */
+			move_diff = needed_space + extra_space;
+			sync_ctx->expunged_space =
+				mail_ctx->mail.space - move_diff;
+		} else {
+			move_diff = mail_ctx->mail.space;
+			extra_space = sync_ctx->space_diff;
+			sync_ctx->expunged_space = 0;
+		}
+		last_seq = sync_ctx->seq - 1;
+		array_pop_back(&sync_ctx->mails);
+		end_offset = mail_ctx->mail.from_offset;
+	} else {
+		/* this message gave enough space from headers. rewriting stops
+		   at the end of this message's headers. */
+		sync_ctx->expunged_space = 0;
+		last_seq = sync_ctx->seq;
+		end_offset = mail_ctx->body_offset;
+
+		move_diff = 0;
+		extra_space = sync_ctx->space_diff;
+	}
+
+	mbox_sync_file_update_ext_modified(sync_ctx);
+	if (mbox_sync_rewrite(sync_ctx,
+			      last_seq == sync_ctx->seq ? mail_ctx : NULL,
+			      end_offset, move_diff, extra_space,
+			      sync_ctx->need_space_seq, last_seq) < 0)
+		return -1;
+
+	update_from_offsets(sync_ctx);
+
+	/* mail_ctx may contain wrong data after rewrite, so make sure we
+	   don't try to access it */
+	i_zero(mail_ctx);
+
+	sync_ctx->need_space_seq = 0;
+	sync_ctx->space_diff = 0;
+	array_clear(&sync_ctx->mails);
+	p_clear(sync_ctx->saved_keywords_pool);
+	return 0;
+}
+
+static int
+mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
+{
+	struct mbox_mailbox *mbox = sync_ctx->mbox;
+	uoff_t old_offset, offset;
+	uint32_t uid;
+	int ret;
+        bool deleted;
+
+	if (seq == 0) {
+		if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) {
+			mbox->invalid_mbox_file = TRUE;
+			mail_storage_set_error(&mbox->storage->storage,
+				MAIL_ERROR_NOTPOSSIBLE,
+				"Mailbox isn't a valid mbox file");
+			return -1;
+		}
+		seq++;
+	} else {
+		old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
+
+		ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted);
+		if (ret < 0) {
+			if (deleted) {
+				mbox_sync_set_critical(sync_ctx,
+					"Message was expunged unexpectedly");
+			}
+			return -1;
+		}
+		if (ret == 0) {
+			if (istream_raw_mbox_seek(mbox->mbox_stream,
+						  old_offset) < 0) {
+				mbox_sync_set_critical(sync_ctx,
+					"Error seeking back to original "
+					"offset %s", dec2str(old_offset));
+				return -1;
+			}
+			return 0;
+		}
+	}
+
+	if (seq <= 1)
+		uid = 0;
+	else
+		mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid);
+
+	sync_ctx->prev_msg_uid = uid;
+
+        /* set to -1, since it's always increased later */
+	sync_ctx->seq = seq-1;
+	if (sync_ctx->seq == 0 &&
+	    istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
+		/* this mbox has pseudo mail which contains the X-IMAP header */
+		sync_ctx->seq++;
+	}
+
+        sync_ctx->idx_seq = seq;
+	sync_ctx->dest_first_mail = sync_ctx->seq == 0;
+	if (istream_raw_mbox_get_body_offset(sync_ctx->input, &offset) < 0) {
+		mbox_sync_set_critical(sync_ctx,
+			"Message body offset lookup failed");
+		return -1;
+	}
+	return 1;
+}
+
+static int
+mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
+{
+	struct mail_index_view *sync_view = sync_ctx->sync_view;
+	uint32_t seq1, seq2;
+	uoff_t size;
+	int ret;
+
+	i_assert(!sync_ctx->index_reset);
+
+	if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1,
+					 &seq1, &seq2)) {
+		/* doesn't exist anymore, seek to end of file */
+		ret = i_stream_get_size(sync_ctx->file_input, TRUE, &size);
+		if (ret < 0) {
+			mbox_istream_set_syscall_error(sync_ctx->mbox,
+				sync_ctx->file_input, "i_stream_get_size()");
+			return -1;
+		}
+		i_assert(ret != 0);
+
+		if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream,
+					  size) < 0) {
+			mbox_sync_set_critical(sync_ctx,
+				"Error seeking to end of mbox");
+			return -1;
+		}
+		sync_ctx->idx_seq =
+			mail_index_view_get_messages_count(sync_view) + 1;
+		return 1;
+	}
+
+	return mbox_sync_seek_to_seq(sync_ctx, seq1);
+}
+
+static int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx,
+				       uint32_t next_uid, bool *partial,
+				       bool *skipped_mails)
+{
+	uint32_t messages_count, uid;
+	int ret;
+
+	i_assert(!sync_ctx->index_reset);
+
+	/* delete sync records up to next message. so if there's still
+	   something left in array, it means the next message needs modifying */
+	index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid);
+	if (index_sync_changes_have(sync_ctx->sync_changes))
+		return 1;
+
+	if (sync_ctx->hdr->first_recent_uid <= next_uid &&
+	    !sync_ctx->keep_recent) {
+		/* we'll need to rewrite Status: O headers */
+		return 1;
+	}
+
+	uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes);
+
+	if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid &&
+	    (uid > sync_ctx->hdr->first_recent_uid || uid == 0) &&
+	    !sync_ctx->keep_recent) {
+		/* we'll need to rewrite Status: O headers */
+		uid = sync_ctx->hdr->first_recent_uid;
+	}
+
+	if (uid != 0) {
+		/* we can skip forward to next record which needs updating. */
+		if (uid != next_uid) {
+			*skipped_mails = TRUE;
+			next_uid = uid;
+		}
+		ret = mbox_sync_seek_to_uid(sync_ctx, next_uid);
+	} else {
+		/* if there's no sync records left, we can stop. except if
+		   this is a dirty sync, check if there are new messages. */
+		if (sync_ctx->mbox->mbox_hdr.dirty_flag == 0)
+			return 0;
+
+		messages_count =
+			mail_index_view_get_messages_count(sync_ctx->sync_view);
+		if (sync_ctx->seq + 1 != messages_count) {
+			ret = mbox_sync_seek_to_seq(sync_ctx, messages_count);
+			*skipped_mails = TRUE;
+		} else {
+			ret = 1;
+		}
+		*partial = FALSE;
+	}
+
+	if (ret == 0) {
+		/* seek failed because the offset is dirty. just ignore and
+		   continue from where we are now. */
+		*partial = FALSE;
+		ret = 1;
+	}
+	return ret;
+}
+
+static void mbox_sync_hdr_update(struct mbox_sync_context *sync_ctx,
+				 struct mbox_sync_mail_context *mail_ctx)
+{
+	const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update;
+
+	if (update->uid_validity != 0) {
+		sync_ctx->base_uid_validity = update->uid_validity;
+		mail_ctx->imapbase_rewrite = TRUE;
+		mail_ctx->need_rewrite = TRUE;
+	}
+	if (update->min_next_uid != 0 &&
+	    sync_ctx->base_uid_last+1 < update->min_next_uid) {
+		i_assert(sync_ctx->next_uid <= update->min_next_uid);
+		sync_ctx->base_uid_last = update->min_next_uid-1;
+		sync_ctx->next_uid = update->min_next_uid;
+		mail_ctx->imapbase_rewrite = TRUE;
+		mail_ctx->need_rewrite = TRUE;
+	}
+}
+
+static bool mbox_sync_imapbase(struct mbox_sync_context *sync_ctx,
+			       struct mbox_sync_mail_context *mail_ctx)
+{
+	if (sync_ctx->base_uid_validity != 0 &&
+	    sync_ctx->hdr->uid_validity != 0 &&
+	    sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
+		i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s",
+			  sync_ctx->hdr->uid_validity,
+			  sync_ctx->base_uid_validity,
+			  mailbox_get_path(&sync_ctx->mbox->box));
+		sync_ctx->index_reset = TRUE;
+		return TRUE;
+	}
+	if (sync_ctx->mbox->sync_hdr_update != NULL)
+		mbox_sync_hdr_update(sync_ctx, mail_ctx);
+	return FALSE;
+}
+
+static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
+                          struct mbox_sync_mail_context *mail_ctx,
+			  bool partial)
+{
+	const struct mail_index_record *rec;
+	uint32_t uid, messages_count;
+	uoff_t offset;
+	int ret;
+	bool expunged, skipped_mails, uids_broken;
+
+	messages_count =
+		mail_index_view_get_messages_count(sync_ctx->sync_view);
+
+	/* always start from first message so we can read X-IMAP or
+	   X-IMAPbase header */
+	ret = mbox_sync_seek_to_seq(sync_ctx, 0);
+	if (ret <= 0)
+		return ret;
+
+	if (sync_ctx->renumber_uids) {
+		/* expunge everything */
+		while (sync_ctx->idx_seq <= messages_count) {
+			mail_index_expunge(sync_ctx->t,
+					   sync_ctx->idx_seq++);
+		}
+	}
+
+	skipped_mails = uids_broken = FALSE;
+	while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
+		uid = mail_ctx->mail.uid;
+
+		if (mail_ctx->seq == 1) {
+			if (mbox_sync_imapbase(sync_ctx, mail_ctx)) {
+				sync_ctx->mbox->mbox_hdr.dirty_flag = 1;
+				return 0;
+			}
+		}
+
+		if (mail_ctx->mail.uid_broken && partial) {
+			/* UID ordering problems, resync everything to make
+			   sure we get everything right */
+			if (sync_ctx->mbox->mbox_hdr.dirty_flag != 0)
+				return 0;
+
+			mbox_sync_set_critical(sync_ctx,
+				"UIDs broken with partial sync");
+
+			sync_ctx->mbox->mbox_hdr.dirty_flag = 1;
+			return 0;
+		}
+		if (mail_ctx->mail.uid_broken)
+			uids_broken = TRUE;
+
+		if (mail_ctx->mail.pseudo)
+			uid = 0;
+
+		rec = NULL; ret = 1;
+		if (uid != 0) {
+			if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec))
+				ret = 0;
+		}
+
+		if (ret == 0) {
+			/* UID found but it's broken */
+			uid = 0;
+		} else if (uid == 0 &&
+			   !mail_ctx->mail.pseudo &&
+			   (sync_ctx->delay_writes ||
+			    sync_ctx->idx_seq <= messages_count)) {
+			/* If we can't use/store X-UID header, use MD5 sum.
+			   Also check for existing MD5 sums when we're actually
+			   able to write X-UIDs. */
+			sync_ctx->mbox->mbox_save_md5 = TRUE;
+
+			mbox_sync_find_index_md5(sync_ctx,
+						 mail_ctx->hdr_md5_sum, &rec);
+			if (rec != NULL)
+				uid = mail_ctx->mail.uid = rec->uid;
+		}
+
+		/* get all sync records related to this message. with pseudo
+		   message just get the first sync record so we can jump to
+		   it with partial seeking. */
+		mbox_sync_read_index_syncs(sync_ctx,
+					   mail_ctx->mail.pseudo ? 1 : uid,
+					   &expunged);
+
+		if (mail_ctx->mail.pseudo) {
+			/* if it was set, it was for the next message */
+			expunged = FALSE;
+		} else {
+			if (rec == NULL) {
+				/* message wasn't found from index. we have to
+				   read everything from now on, no skipping */
+				partial = FALSE;
+			}
+		}
+
+		if (uid == 0 && !mail_ctx->mail.pseudo) {
+			/* missing/broken X-UID. all the rest of the mails
+			   need new UIDs. */
+			while (sync_ctx->idx_seq <= messages_count) {
+				mail_index_expunge(sync_ctx->t,
+						   sync_ctx->idx_seq++);
+			}
+
+			if (sync_ctx->next_uid == (uint32_t)-1) {
+				/* oh no, we're out of UIDs. this shouldn't
+				   happen normally, so just try to get it fixed
+				   without crashing. */
+				mailbox_set_critical(&sync_ctx->mbox->box,
+					"Out of UIDs, renumbering them in mbox");
+				sync_ctx->renumber_uids = TRUE;
+				return 0;
+			}
+
+			mail_ctx->need_rewrite = TRUE;
+			mail_ctx->mail.uid = sync_ctx->next_uid++;
+		}
+		sync_ctx->prev_msg_uid = mail_ctx->mail.uid;
+
+		if (!mail_ctx->mail.pseudo)
+			mail_ctx->mail.idx_seq = sync_ctx->idx_seq;
+
+		if (!expunged) {
+			if (!mail_ctx->mail.pseudo) T_BEGIN {
+				mbox_sync_update_flags(mail_ctx, rec);
+			} T_END;
+			if (mbox_sync_handle_header(mail_ctx) < 0)
+				return -1;
+			sync_ctx->dest_first_mail = FALSE;
+		} else {
+			mbox_sync_handle_expunge(mail_ctx);
+		}
+
+		if (!mail_ctx->mail.pseudo) {
+			if (!expunged) T_BEGIN {
+				mbox_sync_update_index(mail_ctx, rec);
+			} T_END;
+			sync_ctx->idx_seq++;
+		}
+
+		if (istream_raw_mbox_next(sync_ctx->input,
+					  mail_ctx->mail.body_size) < 0)
+			return -1;
+		offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
+
+		if (sync_ctx->need_space_seq != 0) {
+			if (mbox_sync_handle_missing_space(mail_ctx) < 0)
+				return -1;
+			if (mbox_sync_seek(sync_ctx, offset) < 0)
+				return -1;
+		} else if (sync_ctx->expunged_space > 0) {
+			if (!expunged) {
+				/* move the body */
+				mbox_sync_file_update_ext_modified(sync_ctx);
+				if (mbox_move(sync_ctx,
+					      mail_ctx->body_offset -
+					      sync_ctx->expunged_space,
+					      mail_ctx->body_offset,
+					      mail_ctx->mail.body_size) < 0)
+					return -1;
+				if (mbox_sync_seek(sync_ctx, offset) < 0)
+					return -1;
+			}
+		} else if (partial) {
+			ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1,
+							  &partial,
+							  &skipped_mails);
+			if (ret <= 0)
+				break;
+		}
+	}
+	if (ret < 0)
+		return -1;
+
+	if (istream_raw_mbox_is_eof(sync_ctx->input)) {
+		/* rest of the messages in index don't exist -> expunge them */
+		while (sync_ctx->idx_seq <= messages_count)
+			mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
+	}
+
+	if (!skipped_mails)
+		sync_ctx->mbox->mbox_hdr.dirty_flag = 0;
+	sync_ctx->mbox->mbox_broken_offsets = FALSE;
+
+	if (uids_broken && sync_ctx->delay_writes) {
+		/* once we get around to writing the changes, we'll need to do
+		   a full sync to avoid the "UIDs broken in partial sync"
+		   error */
+		sync_ctx->mbox->mbox_hdr.dirty_flag = 1;
+	}
+	return 1;
+}
+
+static int mbox_write_pseudo(struct mbox_sync_context *sync_ctx, bool force)
+{
+	string_t *str;
+	unsigned int uid_validity;
+
+	i_assert(sync_ctx->write_fd != -1);
+
+	if (sync_ctx->mbox->sync_hdr_update != NULL) {
+		const struct mailbox_update *update =
+			sync_ctx->mbox->sync_hdr_update;
+		bool change = FALSE;
+
+		if (update->uid_validity != 0) {
+			sync_ctx->base_uid_validity = update->uid_validity;
+			change = TRUE;
+		}
+		if (update->min_next_uid != 0) {
+			sync_ctx->base_uid_last = update->min_next_uid-1;
+			change = TRUE;
+		}
+		if (!change && !force)
+			return 0;
+	}
+
+	uid_validity = sync_ctx->base_uid_validity != 0 ?
+		sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity;
+	i_assert(uid_validity != 0);
+
+	str = t_str_new(1024);
+	str_printfa(str, "%sDate: %s\n"
+		    "From: Mail System Internal Data <MAILER-DAEMON@%s>\n"
+		    "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA"
+		    "\nMessage-ID: <%s@%s>\n"
+		    "X-IMAP: %u %010u\n"
+		    "Status: RO\n"
+		    "\n"
+		    PSEUDO_MESSAGE_BODY
+		    "\n",
+                    mbox_from_create("MAILER_DAEMON", ioloop_time),
+		    message_date_create(ioloop_time),
+		    my_hostname, dec2str(ioloop_time), my_hostname,
+		    uid_validity, sync_ctx->next_uid-1);
+
+	if (pwrite_full(sync_ctx->write_fd,
+			str_data(str), str_len(str), 0) < 0) {
+		if (!ENOSPACE(errno)) {
+			mbox_set_syscall_error(sync_ctx->mbox,
+					       "pwrite_full()");
+			return -1;
+		}
+
+		/* out of disk space, truncate to empty */
+		if (ftruncate(sync_ctx->write_fd, 0) < 0)
+			mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
+	}
+
+	sync_ctx->base_uid_validity = uid_validity;
+	sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */
+	sync_ctx->base_uid_last = sync_ctx->next_uid-1;
+	return 0;
+}
+
+static int mbox_append_zero(struct mbox_sync_context *sync_ctx,
+			    uoff_t orig_file_size, uoff_t count)
+{
+	char block[IO_BLOCK_SIZE];
+	uoff_t offset = orig_file_size;
+	ssize_t ret = 0;
+
+	memset(block, 0, I_MIN(sizeof(block), count));
+	while (count > 0) {
+		ret = pwrite(sync_ctx->write_fd, block,
+			     I_MIN(sizeof(block), count), offset);
+		if (ret < 0)
+			break;
+		offset += ret;
+		count -= ret;
+	}
+
+	if (ret < 0) {
+		mbox_set_syscall_error(sync_ctx->mbox, "pwrite()");
+		if (ftruncate(sync_ctx->write_fd, orig_file_size) < 0)
+			mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
+		return -1;
+	}
+	return 0;
+}
+
+static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
+					struct mbox_sync_mail_context *mail_ctx)
+{
+	uoff_t file_size, offset, padding, trailer_size;
+	int ret;
+
+	if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
+		i_assert(sync_ctx->need_space_seq == 0);
+		i_assert(sync_ctx->expunged_space == 0);
+		return 0;
+	}
+
+	ret = i_stream_get_size(sync_ctx->file_input, TRUE, &file_size);
+	if (ret < 0) {
+		mbox_istream_set_syscall_error(sync_ctx->mbox,
+			sync_ctx->file_input, "i_stream_get_size()");
+		return -1;
+	}
+	if (ret == 0) {
+		/* Not a file - allow anyway */
+		return 0;
+	}
+
+	if (file_size < sync_ctx->file_input->v_offset) {
+		mbox_sync_set_critical(sync_ctx,
+			"file size unexpectedly shrank "
+			"(%"PRIuUOFF_T" vs %"PRIuUOFF_T")", file_size,
+			sync_ctx->file_input->v_offset);
+		return -1;
+	}
+	trailer_size = file_size - sync_ctx->file_input->v_offset;
+	i_assert(trailer_size <= 2);
+
+	if (sync_ctx->need_space_seq != 0) {
+		i_assert(sync_ctx->write_fd != -1);
+
+		i_assert(sync_ctx->space_diff < 0);
+		padding = MBOX_HEADER_PADDING *
+			(sync_ctx->seq - sync_ctx->need_space_seq + 1);
+		sync_ctx->space_diff -= padding;
+
+		i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff);
+		sync_ctx->space_diff += sync_ctx->expunged_space;
+		sync_ctx->expunged_space = 0;
+
+		if (mail_ctx->have_eoh && !mail_ctx->updated)
+			str_append_c(mail_ctx->header, '\n');
+
+		i_assert(sync_ctx->space_diff < 0);
+
+		if (mbox_append_zero(sync_ctx, file_size,
+				     -sync_ctx->space_diff) < 0)
+			return -1;
+		mbox_sync_file_updated(sync_ctx, FALSE);
+
+		if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size,
+				      -sync_ctx->space_diff, padding,
+				      sync_ctx->need_space_seq,
+				      sync_ctx->seq) < 0)
+			return -1;
+
+		update_from_offsets(sync_ctx);
+
+		sync_ctx->need_space_seq = 0;
+		array_clear(&sync_ctx->mails);
+		p_clear(sync_ctx->saved_keywords_pool);
+	}
+
+	if (sync_ctx->expunged_space > 0) {
+		i_assert(sync_ctx->write_fd != -1);
+
+		mbox_sync_file_update_ext_modified(sync_ctx);
+
+		/* copy trailer, then truncate the file */
+		file_size = sync_ctx->last_stat.st_size;
+		if (file_size == (uoff_t)sync_ctx->expunged_space) {
+			/* everything deleted, the trailer_size still contains
+			   the \n trailer though */
+			trailer_size = 0;
+		} else if (sync_ctx->expunged_space == (off_t)file_size + 1 ||
+			   sync_ctx->expunged_space == (off_t)file_size + 2) {
+			/* everything deleted and we didn't have a proper
+			   trailer. */
+			trailer_size = 0;
+			sync_ctx->expunged_space = file_size;
+		}
+
+		i_assert(file_size >= sync_ctx->expunged_space + trailer_size);
+		offset = file_size - sync_ctx->expunged_space - trailer_size;
+		i_assert(offset == 0 || offset > 31);
+
+		if (mbox_move(sync_ctx, offset,
+			      offset + sync_ctx->expunged_space,
+			      trailer_size) < 0)
+			return -1;
+		if (ftruncate(sync_ctx->write_fd,
+			      offset + trailer_size) < 0) {
+			mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()");
+			return -1;
+		}
+
+		if (offset == 0) {
+			if (mbox_write_pseudo(sync_ctx, TRUE) < 0)
+				return -1;
+		}
+
+                sync_ctx->expunged_space = 0;
+		mbox_sync_file_updated(sync_ctx, FALSE);
+	} else {
+		if (file_size == 0 && sync_ctx->mbox->sync_hdr_update != NULL) {
+			if (mbox_write_pseudo(sync_ctx, FALSE) < 0)
+				return -1;
+		}
+	}
+	return 0;
+}
+
+static void
+mbox_sync_index_update_ext_header(struct mbox_mailbox *mbox,
+				  struct mail_index_transaction *trans)
+{
+	const struct mailbox_update *update = mbox->sync_hdr_update;
+	const void *data;
+	size_t data_size;
+
+	if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) {
+		memcpy(mbox->mbox_hdr.mailbox_guid, update->mailbox_guid,
+		       sizeof(mbox->mbox_hdr.mailbox_guid));
+	} else if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
+		guid_128_generate(mbox->mbox_hdr.mailbox_guid);
+	}
+
+	mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx,
+				  &data, &data_size);
+	if (data_size != sizeof(mbox->mbox_hdr) ||
+	    memcmp(data, &mbox->mbox_hdr, data_size) != 0) {
+		if (data_size != sizeof(mbox->mbox_hdr)) {
+			/* upgrading from v1.x */
+			mail_index_ext_resize(trans, mbox->mbox_ext_idx,
+					      sizeof(mbox->mbox_hdr),
+					      sizeof(uint64_t),
+					      sizeof(uint64_t));
+		}
+		mail_index_update_header_ext(trans, mbox->mbox_ext_idx,
+					     0, &mbox->mbox_hdr,
+					     sizeof(mbox->mbox_hdr));
+	}
+}
+
+static uint32_t mbox_get_uidvalidity_next(struct mailbox_list *list)
+{
+	const char *path;
+
+	path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL);
+	path = t_strconcat(path, "/"MBOX_UIDVALIDITY_FNAME, NULL);
+	return mailbox_uidvalidity_next(list, path);
+}
+
+static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
+{
+	struct mail_index_view *view;
+	const struct stat *st;
+	uint32_t first_recent_uid, seq, seq2;
+
+	if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) {
+		mbox_istream_set_syscall_error(sync_ctx->mbox,
+			sync_ctx->file_input, "i_stream_stat()");
+		return -1;
+	}
+
+	if (sync_ctx->moved_offsets &&
+	    ((uint64_t)st->st_size == sync_ctx->mbox->mbox_hdr.sync_size ||
+	     (uint64_t)st->st_size == sync_ctx->orig_size)) {
+		/* We moved messages inside the mbox file without changing
+		   the file's size. If mtime doesn't change, another process
+		   not using the same index file as us can't know that the file
+		   was changed. So make sure the mtime changes. This should
+		   happen rarely enough that the sleeping doesn't become a
+		   performance problem.
+
+		   Note that to do this perfectly safe we should do this wait
+		   whenever mails are moved or expunged, regardless of whether
+		   the file's size changed. That however could become a
+		   performance problem and the consequences of being wrong are
+		   quite minimal (an extra logged error message). */
+		while (sync_ctx->orig_mtime == st->st_mtime) {
+			i_sleep_msecs(500);
+			if (utime(mailbox_get_path(&sync_ctx->mbox->box), NULL) < 0) {
+				mbox_set_syscall_error(sync_ctx->mbox,
+						       "utime()");
+				return -1;
+			}
+
+			if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) {
+				mbox_istream_set_syscall_error(sync_ctx->mbox,
+					sync_ctx->file_input, "i_stream_stat()");
+				return -1;
+			}
+		}
+	}
+
+	sync_ctx->mbox->mbox_hdr.sync_mtime = st->st_mtime;
+	sync_ctx->mbox->mbox_hdr.sync_size = st->st_size;
+	mbox_sync_index_update_ext_header(sync_ctx->mbox, sync_ctx->t);
+
+	/* only reason not to have UID validity at this point is if the file
+	   is entirely empty. In that case just make up a new one if needed. */
+	i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0);
+
+	if (sync_ctx->base_uid_validity == 0) {
+		sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ?
+			sync_ctx->hdr->uid_validity :
+			mbox_get_uidvalidity_next(sync_ctx->mbox->box.list);
+	}
+	if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
+		mail_index_update_header(sync_ctx->t,
+			offsetof(struct mail_index_header, uid_validity),
+			&sync_ctx->base_uid_validity,
+			sizeof(sync_ctx->base_uid_validity), TRUE);
+	}
+
+	if (istream_raw_mbox_is_eof(sync_ctx->input) &&
+	    sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
+		i_assert(sync_ctx->next_uid != 0);
+		mail_index_update_header(sync_ctx->t,
+			offsetof(struct mail_index_header, next_uid),
+			&sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
+	}
+
+	if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) {
+		/* other sessions have already marked more messages as
+		   recent. */
+		sync_ctx->last_nonrecent_uid =
+			sync_ctx->hdr->first_recent_uid - 1;
+	}
+
+	/* mark recent messages */
+	view = mail_index_transaction_open_updated_view(sync_ctx->t);
+	if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1,
+					(uint32_t)-1, &seq, &seq2)) {
+		mailbox_recent_flags_set_seqs(&sync_ctx->mbox->box,
+					      view, seq, seq2);
+	}
+	mail_index_view_close(&view);
+
+	first_recent_uid = !sync_ctx->keep_recent ?
+		sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1;
+	if (sync_ctx->hdr->first_recent_uid < first_recent_uid) {
+		mail_index_update_header(sync_ctx->t,
+			offsetof(struct mail_index_header, first_recent_uid),
+			&first_recent_uid, sizeof(first_recent_uid), FALSE);
+	}
+	return 0;
+}
+
+static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
+{
+	sync_ctx->base_uid_validity = 0;
+	sync_ctx->base_uid_last = 0;
+	sync_ctx->base_uid_last_offset = 0;
+
+	array_clear(&sync_ctx->mails);
+	p_clear(sync_ctx->saved_keywords_pool);
+
+	index_sync_changes_reset(sync_ctx->sync_changes);
+        mail_index_sync_reset(sync_ctx->index_sync_ctx);
+	mail_index_transaction_reset(sync_ctx->t);
+
+	if (sync_ctx->index_reset) {
+		mail_index_reset(sync_ctx->t);
+		sync_ctx->reset_hdr.next_uid = 1;
+		sync_ctx->hdr = &sync_ctx->reset_hdr;
+		mailbox_recent_flags_reset(&sync_ctx->mbox->box);
+	}
+
+	sync_ctx->prev_msg_uid = 0;
+	sync_ctx->next_uid = sync_ctx->hdr->next_uid;
+	sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
+	sync_ctx->seq = 0;
+	sync_ctx->idx_seq = 1;
+	sync_ctx->need_space_seq = 0;
+	sync_ctx->expunged_space = 0;
+	sync_ctx->space_diff = 0;
+
+	sync_ctx->dest_first_mail = TRUE;
+	sync_ctx->ext_modified = FALSE;
+	sync_ctx->errors = FALSE;
+}
+
+static int mbox_sync_do(struct mbox_sync_context *sync_ctx,
+			enum mbox_sync_flags flags)
+{
+	struct mbox_index_header *mbox_hdr = &sync_ctx->mbox->mbox_hdr;
+	struct mbox_sync_mail_context mail_ctx;
+	const struct stat *st;
+	unsigned int i;
+	bool partial;
+	int ret;
+
+	if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) {
+		mbox_istream_set_syscall_error(sync_ctx->mbox,
+			sync_ctx->file_input, "i_stream_stat()");
+		return -1;
+	}
+	sync_ctx->last_stat = *st;
+	sync_ctx->orig_size = st->st_size;
+	sync_ctx->orig_atime = st->st_atime;
+	sync_ctx->orig_mtime = st->st_mtime;
+
+	if ((flags & MBOX_SYNC_FORCE_SYNC) != 0) {
+		/* forcing a full sync. assume file has changed. */
+		partial = FALSE;
+		mbox_hdr->dirty_flag = 1;
+	} else if ((uint32_t)st->st_mtime == mbox_hdr->sync_mtime &&
+		   (uint64_t)st->st_size == mbox_hdr->sync_size) {
+		/* file is fully synced */
+		if (mbox_hdr->dirty_flag != 0 && (flags & MBOX_SYNC_UNDIRTY) != 0)
+			partial = FALSE;
+		else
+			partial = TRUE;
+	} else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
+		   (uint64_t)st->st_size == mbox_hdr->sync_size) {
+		/* we want to do full syncing. always do this if
+		   file size hasn't changed but timestamp has. it most
+		   likely means that someone had modified some header
+		   and we probably want to know about it */
+		partial = FALSE;
+		sync_ctx->mbox->mbox_hdr.dirty_flag = 1;
+	} else {
+		/* see if we can delay syncing the whole file.
+		   normally we only notice expunges and appends
+		   in partial syncing. */
+		partial = TRUE;
+		sync_ctx->mbox->mbox_hdr.dirty_flag = 1;
+	}
+
+	mbox_sync_restart(sync_ctx);
+	for (i = 0;;) {
+		ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial);
+		if (ret > 0 && !sync_ctx->errors)
+			break;
+		if (ret < 0)
+			return -1;
+
+		/* a) partial sync didn't work
+		   b) we ran out of UIDs
+		   c) syncing had errors */
+		if (sync_ctx->delay_writes &&
+		    (sync_ctx->errors || sync_ctx->renumber_uids)) {
+			/* fixing a broken mbox state, be sure to write
+			   the changes (except if we're readonly). */
+			if (!sync_ctx->readonly)
+				sync_ctx->delay_writes = FALSE;
+		}
+		if (++i == 3)
+			break;
+
+		mbox_sync_restart(sync_ctx);
+		partial = FALSE;
+	}
+
+	if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
+		return -1;
+
+	/* only syncs left should be just appends (and their updates)
+	   which weren't synced yet for some reason (crash). we'll just
+	   ignore them, as we've overwritten them above. */
+	index_sync_changes_reset(sync_ctx->sync_changes);
+
+	if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 &&
+	    ret > 0 && !sync_ctx->delay_writes &&
+	    sync_ctx->base_uid_last_offset != 0) {
+		/* Rewrite uid_last in X-IMAPbase header if we've seen it
+		   (ie. the file isn't empty) */
+                ret = mbox_rewrite_base_uid_last(sync_ctx);
+	} else {
+		ret = 0;
+	}
+
+	if (mbox_sync_update_index_header(sync_ctx) < 0)
+		return -1;
+	return ret;
+}
+
+int mbox_sync_header_refresh(struct mbox_mailbox *mbox)
+{
+	const void *data;
+	size_t data_size;
+
+	if (mail_index_refresh(mbox->box.index) < 0) {
+		mailbox_set_index_error(&mbox->box);
+		return -1;
+	}
+
+	mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx,
+				  &data, &data_size);
+	if (data_size == 0) {
+		/* doesn't exist yet. */
+		i_zero(&mbox->mbox_hdr);
+		return 0;
+	}
+
+	memcpy(&mbox->mbox_hdr, data, I_MIN(sizeof(mbox->mbox_hdr), data_size));
+	if (mbox->mbox_broken_offsets)
+		mbox->mbox_hdr.dirty_flag = 1;
+	return 0;
+}
+
+int mbox_sync_get_guid(struct mbox_mailbox *mbox)
+{
+	struct mail_index_transaction *trans;
+	unsigned int lock_id;
+	int ret;
+
+	if (mbox_lock(mbox, F_WRLCK, &lock_id) <= 0)
+		return -1;
+
+	ret = mbox_sync_header_refresh(mbox);
+	if (ret == 0) {
+		trans = mail_index_transaction_begin(mbox->box.view,
+				MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+		mbox_sync_index_update_ext_header(mbox, trans);
+		ret = mail_index_transaction_commit(&trans);
+	}
+	mbox_unlock(mbox, lock_id);
+	return ret;
+}
+
+int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty)
+{
+	const struct stat *st;
+	struct stat statbuf;
+
+	if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) {
+		/* read-only stream */
+		if (i_stream_stat(mbox->mbox_file_stream, FALSE, &st) < 0) {
+			if (errno == ENOENT) {
+				mailbox_set_deleted(&mbox->box);
+				return 0;
+			}
+			mbox_istream_set_syscall_error(mbox,
+				mbox->mbox_file_stream, "i_stream_stat()");
+			return -1;
+		}
+	} else {
+		if (stat(mailbox_get_path(&mbox->box), &statbuf) < 0) {
+			if (errno == ENOENT) {
+				mailbox_set_deleted(&mbox->box);
+				return 0;
+			}
+			mbox_set_syscall_error(mbox, "stat()");
+			return -1;
+		}
+		st = &statbuf;
+	}
+
+	if (mbox_sync_header_refresh(mbox) < 0)
+		return -1;
+
+	if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
+		/* need to assign mailbox GUID */
+		return 1;
+	}
+
+	if ((uint32_t)st->st_mtime == mbox->mbox_hdr.sync_mtime &&
+	    (uint64_t)st->st_size == mbox->mbox_hdr.sync_size) {
+		/* fully synced */
+		if (mbox->mbox_hdr.dirty_flag != 0 || leave_dirty)
+			return 0;
+		/* flushing dirtiness */
+	}
+
+	/* file changed */
+	return 1;
+}
+
+static void mbox_sync_context_free(struct mbox_sync_context *sync_ctx)
+{
+	index_sync_changes_deinit(&sync_ctx->sync_changes);
+	index_storage_expunging_deinit(&sync_ctx->mbox->box);
+	if (sync_ctx->index_sync_ctx != NULL)
+		mail_index_sync_rollback(&sync_ctx->index_sync_ctx);
+	pool_unref(&sync_ctx->mail_keyword_pool);
+	pool_unref(&sync_ctx->saved_keywords_pool);
+	str_free(&sync_ctx->header);
+	str_free(&sync_ctx->from_line);
+	array_free(&sync_ctx->mails);
+}
+
+static int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags,
+			 unsigned int *lock_id)
+{
+	struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *trans;
+	struct mbox_sync_context sync_ctx;
+	enum mail_index_sync_flags sync_flags;
+	int ret;
+	bool changed, delay_writes, readonly;
+
+	readonly = mbox_is_backend_readonly(mbox) ||
+		(flags & MBOX_SYNC_READONLY) != 0;
+	delay_writes = readonly ||
+		((flags & MBOX_SYNC_REWRITE) == 0 &&
+		 mbox->storage->set->mbox_lazy_writes);
+
+	if (!mbox->storage->set->mbox_dirty_syncs &&
+	    !mbox->storage->set->mbox_very_dirty_syncs)
+		flags |= MBOX_SYNC_UNDIRTY;
+
+	if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
+		if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0)
+			return -1;
+	}
+
+	if ((flags & MBOX_SYNC_HEADER) != 0 ||
+	    (flags & MBOX_SYNC_FORCE_SYNC) != 0) {
+		if (mbox_sync_header_refresh(mbox) < 0)
+			return -1;
+		changed = TRUE;
+	} else {
+		bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
+		if ((ret = mbox_sync_has_changed(mbox, leave_dirty)) < 0)
+			return -1;
+		changed = ret > 0;
+	}
+
+	if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
+		/* we just want to lock it for reading. if mbox hasn't been
+		   modified don't do any syncing. */
+		if (!changed)
+			return 0;
+
+		/* have to sync to make sure offsets have stayed the same */
+		mbox_unlock(mbox, *lock_id);
+		*lock_id = 0;
+	}
+
+	/* flush input streams' buffers */
+	if (mbox->mbox_stream != NULL)
+		i_stream_sync(mbox->mbox_stream);
+	if (mbox->mbox_file_stream != NULL)
+		i_stream_sync(mbox->mbox_file_stream);
+
+again:
+	if (changed) {
+		/* we're most likely modifying the mbox while syncing, just
+		   lock it for writing immediately. the mbox must be locked
+		   before index syncing is started to avoid deadlocks, so we
+		   don't have much choice either (well, easy ones anyway). */
+		int lock_type = readonly ? F_RDLCK : F_WRLCK;
+
+		if ((ret = mbox_lock(mbox, lock_type, lock_id)) <= 0) {
+			if (ret == 0 || lock_type == F_RDLCK)
+				return -1;
+
+			/* try as read-only */
+			if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0)
+				return -1;
+			mbox->backend_readonly = readonly = TRUE;
+			mbox->backend_readonly_set = TRUE;
+			delay_writes = TRUE;
+		}
+	}
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box);
+	if ((flags & MBOX_SYNC_REWRITE) != 0)
+		sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
+
+	ret = index_storage_expunged_sync_begin(&mbox->box, &index_sync_ctx,
+						&sync_view, &trans, sync_flags);
+	if (ret <= 0)
+		return ret;
+
+	if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) {
+		/* see if we need to drop recent flags */
+		sync_ctx.hdr = mail_index_get_header(sync_view);
+		if (sync_ctx.hdr->first_recent_uid < sync_ctx.hdr->next_uid)
+			changed = TRUE;
+	}
+
+	if (!changed && !mail_index_sync_have_more(index_sync_ctx)) {
+		/* nothing to do */
+	nothing_to_do:
+		/* index may need to do internal syncing though, so commit
+		   instead of rolling back. */
+		index_storage_expunging_deinit(&mbox->box);
+		if (mail_index_sync_commit(&index_sync_ctx) < 0) {
+			mailbox_set_index_error(&mbox->box);
+			return -1;
+		}
+		return 0;
+	}
+
+	i_zero(&sync_ctx);
+	sync_ctx.mbox = mbox;
+	sync_ctx.keep_recent =
+		(mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0;
+
+	sync_ctx.hdr = mail_index_get_header(sync_view);
+	sync_ctx.from_line = str_new(default_pool, 256);
+	sync_ctx.header = str_new(default_pool, 4096);
+
+	sync_ctx.index_sync_ctx = index_sync_ctx;
+	sync_ctx.sync_view = sync_view;
+	sync_ctx.t = trans;
+	sync_ctx.mail_keyword_pool =
+		pool_alloconly_create("mbox keywords", 512);
+	sync_ctx.saved_keywords_pool =
+		pool_alloconly_create("mbox saved keywords", 4096);
+
+	/* make sure we've read the latest keywords in index */
+	(void)mail_index_get_keywords(mbox->box.index);
+
+	i_array_init(&sync_ctx.mails, 64);
+
+	sync_ctx.flags = flags;
+	sync_ctx.readonly = readonly;
+	sync_ctx.delay_writes = delay_writes;
+
+	sync_ctx.sync_changes =
+		index_sync_changes_init(index_sync_ctx, sync_view, trans,
+					sync_ctx.delay_writes);
+
+	if (!changed && delay_writes) {
+		/* if we have only flag changes, we don't need to open the
+		   mbox file */
+		bool expunged;
+		uint32_t uid;
+
+		mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged);
+		uid = expunged ? 1 :
+			index_sync_changes_get_next_uid(sync_ctx.sync_changes);
+		if (uid == 0) {
+			sync_ctx.index_sync_ctx = NULL;
+			mbox_sync_context_free(&sync_ctx);
+			goto nothing_to_do;
+		}
+	}
+
+	if (*lock_id == 0) {
+		/* ok, we have something to do but no locks. we'll have to
+		   restart syncing to avoid deadlocking. */
+		mbox_sync_context_free(&sync_ctx);
+		changed = TRUE;
+		goto again;
+	}
+
+	if (mbox_file_open_stream(mbox) < 0) {
+		mbox_sync_context_free(&sync_ctx);
+		return -1;
+	}
+
+	sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream;
+	sync_ctx.input = sync_ctx.mbox->mbox_stream;
+	sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 :
+		sync_ctx.mbox->mbox_fd;
+
+	ret = mbox_sync_do(&sync_ctx, flags);
+
+	if (ret < 0)
+		mail_index_sync_rollback(&index_sync_ctx);
+	else if (mail_index_sync_commit(&index_sync_ctx) < 0) {
+		mailbox_set_index_error(&mbox->box);
+		ret = -1;
+	}
+	sync_ctx.t = NULL;
+	sync_ctx.index_sync_ctx = NULL;
+
+	if (ret == 0 && mbox->mbox_fd != -1 && sync_ctx.keep_recent &&
+	    !readonly) {
+		/* try to set atime back to its original value.
+		   (it'll fail with EPERM for shared mailboxes where we aren't
+		   the file's owner) */
+		struct utimbuf buf;
+		struct stat st;
+
+		if (fstat(mbox->mbox_fd, &st) < 0)
+			mbox_set_syscall_error(mbox, "fstat()");
+		else {
+			buf.modtime = st.st_mtime;
+			buf.actime = sync_ctx.orig_atime;
+			if (utime(mailbox_get_path(&mbox->box), &buf) < 0 &&
+			    errno != EPERM)
+				mbox_set_syscall_error(mbox, "utime()");
+		}
+	}
+
+	i_assert(*lock_id != 0);
+
+	if (mbox->storage->storage.set->mail_nfs_storage &&
+	    mbox->mbox_fd != -1) {
+		if (fdatasync(mbox->mbox_fd) < 0) {
+			mbox_set_syscall_error(mbox, "fdatasync()");
+			ret = -1;
+		}
+	}
+
+	mbox_sync_context_free(&sync_ctx);
+	return ret;
+}
+
+int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags)
+{
+	unsigned int lock_id = 0;
+	int ret;
+
+	i_assert(mbox->mbox_lock_type != F_RDLCK ||
+		 (flags & MBOX_SYNC_READONLY) != 0);
+
+	mbox->syncing = TRUE;
+	ret = mbox_sync_int(mbox, flags, &lock_id);
+	mbox->syncing = FALSE;
+
+	if (lock_id != 0) {
+		if (ret < 0) {
+			/* syncing failed, don't leave it locked */
+			mbox_unlock(mbox, lock_id);
+		} else if ((flags & MBOX_SYNC_LOCK_READING) == 0) {
+			if (mbox_unlock(mbox, lock_id) < 0)
+				ret = -1;
+		} else if (mbox->mbox_lock_type != F_RDLCK) {
+			/* drop to read lock */
+			unsigned int read_lock_id = 0;
+
+			if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0)
+				ret = -1;
+			if (mbox_unlock(mbox, lock_id) < 0)
+				ret = -1;
+		}
+	}
+
+	mailbox_sync_notify(&mbox->box, 0, 0);
+	return ret;
+}
+
+struct mailbox_sync_context *
+mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct mbox_mailbox *mbox = MBOX_MAILBOX(box);
+	enum mbox_sync_flags mbox_sync_flags = 0;
+	int ret = 0;
+
+	if (index_mailbox_want_full_sync(&mbox->box, flags)) {
+		if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
+		    !mbox->storage->set->mbox_very_dirty_syncs)
+			mbox_sync_flags |= MBOX_SYNC_UNDIRTY;
+		if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)
+			mbox_sync_flags |= MBOX_SYNC_REWRITE;
+		if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) {
+			mbox_sync_flags |= MBOX_SYNC_UNDIRTY |
+				MBOX_SYNC_REWRITE | MBOX_SYNC_FORCE_SYNC;
+		}
+
+		ret = mbox_sync(mbox, mbox_sync_flags);
+	}
+
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
diff --git a/src/lib-storage/index/pop3c/Makefile.am b/src/lib-storage/index/pop3c/Makefile.am
new file mode 100644
index 0000000..868dc1e
--- /dev/null
+++ b/src/lib-storage/index/pop3c/Makefile.am
@@ -0,0 +1,28 @@
+noinst_LTLIBRARIES = libstorage_pop3c.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-dns \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_pop3c_la_SOURCES = \
+	pop3c-client.c \
+	pop3c-mail.c \
+	pop3c-settings.c \
+	pop3c-storage.c \
+	pop3c-sync.c
+
+headers = \
+	pop3c-client.h \
+	pop3c-settings.h \
+	pop3c-storage.h \
+	pop3c-sync.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/pop3c/Makefile.in b/src/lib-storage/index/pop3c/Makefile.in
new file mode 100644
index 0000000..e66bdcb
--- /dev/null
+++ b/src/lib-storage/index/pop3c/Makefile.in
@@ -0,0 +1,837 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/pop3c
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_pop3c_la_LIBADD =
+am_libstorage_pop3c_la_OBJECTS = pop3c-client.lo pop3c-mail.lo \
+	pop3c-settings.lo pop3c-storage.lo pop3c-sync.lo
+libstorage_pop3c_la_OBJECTS = $(am_libstorage_pop3c_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/pop3c-client.Plo \
+	./$(DEPDIR)/pop3c-mail.Plo ./$(DEPDIR)/pop3c-settings.Plo \
+	./$(DEPDIR)/pop3c-storage.Plo ./$(DEPDIR)/pop3c-sync.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_pop3c_la_SOURCES)
+DIST_SOURCES = $(libstorage_pop3c_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_pop3c.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-dns \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_pop3c_la_SOURCES = \
+	pop3c-client.c \
+	pop3c-mail.c \
+	pop3c-settings.c \
+	pop3c-storage.c \
+	pop3c-sync.c
+
+headers = \
+	pop3c-client.h \
+	pop3c-settings.h \
+	pop3c-storage.h \
+	pop3c-sync.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/pop3c/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/pop3c/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_pop3c.la: $(libstorage_pop3c_la_OBJECTS) $(libstorage_pop3c_la_DEPENDENCIES) $(EXTRA_libstorage_pop3c_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_pop3c_la_OBJECTS) $(libstorage_pop3c_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-client.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-settings.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pop3c-sync.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/pop3c-client.Plo
+	-rm -f ./$(DEPDIR)/pop3c-mail.Plo
+	-rm -f ./$(DEPDIR)/pop3c-settings.Plo
+	-rm -f ./$(DEPDIR)/pop3c-storage.Plo
+	-rm -f ./$(DEPDIR)/pop3c-sync.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/pop3c-client.Plo
+	-rm -f ./$(DEPDIR)/pop3c-mail.Plo
+	-rm -f ./$(DEPDIR)/pop3c-settings.Plo
+	-rm -f ./$(DEPDIR)/pop3c-storage.Plo
+	-rm -f ./$(DEPDIR)/pop3c-sync.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/pop3c/pop3c-client.c b/src/lib-storage/index/pop3c/pop3c-client.c
new file mode 100644
index 0000000..44544a3
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-client.c
@@ -0,0 +1,902 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "net.h"
+#include "istream.h"
+#include "istream-chain.h"
+#include "istream-dot.h"
+#include "istream-seekable.h"
+#include "ostream.h"
+#include "iostream-rawlog.h"
+#include "iostream-ssl.h"
+#include "safe-mkstemp.h"
+#include "base64.h"
+#include "str.h"
+#include "dns-lookup.h"
+#include "pop3c-client.h"
+
+#include <unistd.h>
+
+#define POP3C_MAX_INBUF_SIZE (1024*32)
+#define POP3C_DNS_LOOKUP_TIMEOUT_MSECS (1000*30)
+#define POP3C_CONNECT_TIMEOUT_MSECS (1000*30)
+#define POP3C_COMMAND_TIMEOUT_MSECS (1000*60*5)
+
+enum pop3c_client_state {
+	/* No connection */
+	POP3C_CLIENT_STATE_DISCONNECTED = 0,
+	/* Trying to connect */
+	POP3C_CLIENT_STATE_CONNECTING,
+	POP3C_CLIENT_STATE_STARTTLS,
+	/* Connected, trying to authenticate */
+	POP3C_CLIENT_STATE_USER,
+	POP3C_CLIENT_STATE_AUTH,
+	POP3C_CLIENT_STATE_PASS,
+	/* Post-authentication, asking for capabilities */
+	POP3C_CLIENT_STATE_CAPA,
+	/* Authenticated, ready to accept commands */
+	POP3C_CLIENT_STATE_DONE
+};
+
+struct pop3c_client_sync_cmd_ctx {
+	enum pop3c_command_state state;
+	char *reply;
+};
+
+struct pop3c_client_cmd {
+	struct istream *input;
+	struct istream_chain *chain;
+	bool reading_dot;
+
+	pop3c_cmd_callback_t *callback;
+	void *context;
+};
+
+struct pop3c_client {
+	pool_t pool;
+	struct event *event;
+	struct pop3c_client_settings set;
+	struct ssl_iostream_context *ssl_ctx;
+	struct ip_addr ip;
+
+	int fd;
+	struct io *io;
+	struct istream *input, *raw_input;
+	struct ostream *output, *raw_output;
+	struct ssl_iostream *ssl_iostream;
+	struct timeout *to;
+	struct dns_lookup *dns_lookup;
+
+	enum pop3c_client_state state;
+	enum pop3c_capability capabilities;
+	const char *auth_mech;
+
+	pop3c_login_callback_t *login_callback;
+	void *login_context;
+
+	ARRAY(struct pop3c_client_cmd) commands;
+	const char *input_line;
+	struct istream *dot_input;
+
+	bool running:1;
+};
+
+static void
+pop3c_dns_callback(const struct dns_lookup_result *result,
+		   struct pop3c_client *client);
+static void pop3c_client_connect_ip(struct pop3c_client *client);
+static int pop3c_client_ssl_init(struct pop3c_client *client);
+static void pop3c_client_input(struct pop3c_client *client);
+
+struct pop3c_client *
+pop3c_client_init(const struct pop3c_client_settings *set,
+		  struct event *event_parent)
+{
+	struct pop3c_client *client;
+	const char *error;
+	pool_t pool;
+
+	pool = pool_alloconly_create("pop3c client", 1024);
+	client = p_new(pool, struct pop3c_client, 1);
+	client->pool = pool;
+	client->event = event_create(event_parent);
+	client->fd = -1;
+	p_array_init(&client->commands, pool, 16);
+
+	client->set.debug = set->debug;
+	client->set.host = p_strdup(pool, set->host);
+	client->set.port = set->port;
+	client->set.master_user = p_strdup_empty(pool, set->master_user);
+	client->set.username = p_strdup(pool, set->username);
+	client->set.password = p_strdup(pool, set->password);
+	client->set.dns_client_socket_path =
+		p_strdup(pool, set->dns_client_socket_path);
+	client->set.temp_path_prefix = p_strdup(pool, set->temp_path_prefix);
+	client->set.rawlog_dir = p_strdup(pool, set->rawlog_dir);
+	client->set.ssl_mode = set->ssl_mode;
+
+	if (set->ssl_mode != POP3C_CLIENT_SSL_MODE_NONE) {
+		ssl_iostream_settings_init_from(client->pool, &client->set.ssl_set, &set->ssl_set);
+		client->set.ssl_set.verbose_invalid_cert = !client->set.ssl_set.allow_invalid_cert;
+		if (ssl_iostream_client_context_cache_get(&set->ssl_set,
+							  &client->ssl_ctx,
+							  &error) < 0) {
+			i_error("pop3c(%s:%u): Couldn't initialize SSL context: %s",
+				set->host, set->port, error);
+		}
+	}
+	return client;
+}
+
+static void
+client_login_callback(struct pop3c_client *client,
+		      enum pop3c_command_state state, const char *reason)
+{
+	pop3c_login_callback_t *callback = client->login_callback;
+	void *context = client->login_context;
+
+	if (client->login_callback != NULL) {
+		client->login_callback = NULL;
+		client->login_context = NULL;
+		callback(state, reason, context);
+	}
+}
+
+static void
+pop3c_client_async_callback(struct pop3c_client *client,
+			    enum pop3c_command_state state, const char *reply)
+{
+	struct pop3c_client_cmd *cmd, cmd_copy;
+	bool running = client->running;
+
+	i_assert(reply != NULL);
+	i_assert(array_count(&client->commands) > 0);
+
+	cmd = array_front_modifiable(&client->commands);
+	if (cmd->input != NULL && state == POP3C_COMMAND_STATE_OK &&
+	    !cmd->reading_dot) {
+		/* read the full input into seekable-istream before calling
+		   the callback */
+		i_assert(client->dot_input == NULL);
+		i_stream_chain_append(cmd->chain, client->input);
+		client->dot_input = cmd->input;
+		cmd->reading_dot = TRUE;
+		return;
+	}
+	cmd_copy = *cmd;
+	array_pop_front(&client->commands);
+
+	if (cmd_copy.input != NULL) {
+		i_stream_seek(cmd_copy.input, 0);
+		i_stream_unref(&cmd_copy.input);
+	}
+	if (cmd_copy.callback != NULL)
+		cmd_copy.callback(state, reply, cmd_copy.context);
+	if (running)
+		io_loop_stop(current_ioloop);
+}
+
+static void
+pop3c_client_async_callback_disconnected(struct pop3c_client *client)
+{
+	pop3c_client_async_callback(client, POP3C_COMMAND_STATE_DISCONNECTED,
+				    "Disconnected");
+}
+
+static void pop3c_client_disconnect(struct pop3c_client *client)
+{
+	client->state = POP3C_CLIENT_STATE_DISCONNECTED;
+
+	if (client->running)
+		io_loop_stop(current_ioloop);
+
+	if (client->dns_lookup != NULL)
+		dns_lookup_abort(&client->dns_lookup);
+	timeout_remove(&client->to);
+	io_remove(&client->io);
+	i_stream_destroy(&client->input);
+	o_stream_destroy(&client->output);
+	ssl_iostream_destroy(&client->ssl_iostream);
+	i_close_fd(&client->fd);
+	while (array_count(&client->commands) > 0)
+		pop3c_client_async_callback_disconnected(client);
+	client_login_callback(client, POP3C_COMMAND_STATE_DISCONNECTED,
+			      "Disconnected");
+}
+
+void pop3c_client_deinit(struct pop3c_client **_client)
+{
+	struct pop3c_client *client = *_client;
+
+	pop3c_client_disconnect(client);
+	if (client->ssl_ctx != NULL)
+		ssl_iostream_context_unref(&client->ssl_ctx);
+	event_unref(&client->event);
+	pool_unref(&client->pool);
+}
+
+static void pop3c_client_ioloop_changed(struct pop3c_client *client)
+{
+	if (client->to != NULL)
+		client->to = io_loop_move_timeout(&client->to);
+	if (client->io != NULL)
+		client->io = io_loop_move_io(&client->io);
+	if (client->output != NULL)
+		o_stream_switch_ioloop(client->output);
+}
+
+static void pop3c_client_timeout(struct pop3c_client *client)
+{
+	switch (client->state) {
+	case POP3C_CLIENT_STATE_CONNECTING:
+		i_error("pop3c(%s): connect(%s, %u) timed out after %u seconds",
+			client->set.host, net_ip2addr(&client->ip),
+			client->set.port, POP3C_CONNECT_TIMEOUT_MSECS/1000);
+		break;
+	case POP3C_CLIENT_STATE_DONE:
+		i_error("pop3c(%s): Command timed out after %u seconds",
+			client->set.host, POP3C_COMMAND_TIMEOUT_MSECS/1000);
+		break;
+	default:
+		i_error("pop3c(%s): Authentication timed out after %u seconds",
+			client->set.host, POP3C_CONNECT_TIMEOUT_MSECS/1000);
+		break;
+	}
+	pop3c_client_disconnect(client);
+}
+
+static int pop3c_client_dns_lookup(struct pop3c_client *client)
+{
+	struct dns_lookup_settings dns_set;
+
+	i_assert(client->state == POP3C_CLIENT_STATE_CONNECTING);
+
+	if (client->set.dns_client_socket_path[0] == '\0') {
+		struct ip_addr *ips;
+		unsigned int ips_count;
+		int ret;
+
+		ret = net_gethostbyname(client->set.host, &ips, &ips_count);
+		if (ret != 0) {
+			i_error("pop3c(%s): net_gethostbyname() failed: %s",
+				client->set.host, net_gethosterror(ret));
+			return -1;
+		}
+		i_assert(ips_count > 0);
+		client->ip = ips[0];
+		pop3c_client_connect_ip(client);
+	} else {
+		i_zero(&dns_set);
+		dns_set.dns_client_socket_path =
+			client->set.dns_client_socket_path;
+		dns_set.timeout_msecs = POP3C_DNS_LOOKUP_TIMEOUT_MSECS;
+		dns_set.event_parent = client->event;
+		if (dns_lookup(client->set.host, &dns_set,
+			       pop3c_dns_callback, client,
+			       &client->dns_lookup) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+void pop3c_client_wait_one(struct pop3c_client *client)
+{
+	struct ioloop *ioloop, *prev_ioloop = current_ioloop;
+	bool timeout_added = FALSE, failed = FALSE;
+
+	if (client->state == POP3C_CLIENT_STATE_DISCONNECTED &&
+	    array_count(&client->commands) > 0) {
+		while (array_count(&client->commands) > 0)
+			pop3c_client_async_callback_disconnected(client);
+		return;
+	}
+
+	i_assert(client->fd != -1 ||
+		 client->state == POP3C_CLIENT_STATE_CONNECTING);
+	i_assert(array_count(&client->commands) > 0 ||
+		 client->state == POP3C_CLIENT_STATE_CONNECTING);
+
+	ioloop = io_loop_create();
+	pop3c_client_ioloop_changed(client);
+
+	if (client->ip.family == 0) {
+		/* we're connecting, start DNS lookup after our ioloop
+		   is created */
+		if (pop3c_client_dns_lookup(client) < 0)
+			failed = TRUE;
+	} else if (client->to == NULL) {
+		client->to = timeout_add(POP3C_COMMAND_TIMEOUT_MSECS,
+					 pop3c_client_timeout, client);
+		timeout_added = TRUE;
+	}
+
+	if (!failed) {
+		client->running = TRUE;
+		io_loop_run(ioloop);
+		client->running = FALSE;
+	}
+
+	if (timeout_added && client->to != NULL)
+		timeout_remove(&client->to);
+
+	io_loop_set_current(prev_ioloop);
+	pop3c_client_ioloop_changed(client);
+	io_loop_set_current(ioloop);
+	io_loop_destroy(&ioloop);
+}
+
+static void pop3c_client_starttls(struct pop3c_client *client)
+{
+	o_stream_nsend_str(client->output, "STLS\r\n");
+	client->state = POP3C_CLIENT_STATE_STARTTLS;
+}
+
+static void pop3c_client_authenticate1(struct pop3c_client *client)
+{
+	const struct pop3c_client_settings *set = &client->set;
+
+	if (client->set.debug) {
+		if (set->master_user == NULL) {
+			i_debug("pop3c(%s): Authenticating as '%s' (with USER+PASS)",
+				client->set.host, set->username);
+		} else {
+			i_debug("pop3c(%s): Authenticating as master user '%s' for user '%s' (with SASL PLAIN)",
+				client->set.host, set->master_user,
+				set->username);
+		}
+	}
+
+	if (set->master_user == NULL) {
+		o_stream_nsend_str(client->output,
+			t_strdup_printf("USER %s\r\n", set->username));
+		client->state = POP3C_CLIENT_STATE_USER;
+	} else {
+		client->state = POP3C_CLIENT_STATE_AUTH;
+		o_stream_nsend_str(client->output, "AUTH PLAIN\r\n");
+	}
+}
+
+static const char *
+pop3c_client_get_sasl_plain_request(struct pop3c_client *client)
+{
+	const struct pop3c_client_settings *set = &client->set;
+	string_t *in, *out;
+
+	in = t_str_new(128);
+	if (set->master_user != NULL) {
+		str_append(in, set->username);
+		str_append_c(in, '\0');
+		str_append(in, set->master_user);
+	} else {
+		str_append_c(in, '\0');
+		str_append(in, set->username);
+	}
+	str_append_c(in, '\0');
+	str_append(in, set->password);
+
+	out = t_str_new(128);
+	base64_encode(str_data(in), str_len(in), out);
+	str_append(out, "\r\n");
+	return str_c(out);
+}
+
+static void pop3c_client_login_finished(struct pop3c_client *client)
+{
+	io_remove(&client->io);
+	client->io = io_add(client->fd, IO_READ, pop3c_client_input, client);
+
+	timeout_remove(&client->to);
+	client->state = POP3C_CLIENT_STATE_DONE;
+
+	if (client->running)
+		io_loop_stop(current_ioloop);
+}
+
+static int
+pop3c_client_prelogin_input_line(struct pop3c_client *client, const char *line)
+{
+	bool success = line[0] == '+';
+	const char *reply;
+
+	switch (client->state) {
+	case POP3C_CLIENT_STATE_CONNECTING:
+		if (!success) {
+			i_error("pop3c(%s): Server sent invalid banner: %s",
+				client->set.host, line);
+			return -1;
+		}
+		if (client->set.ssl_mode == POP3C_CLIENT_SSL_MODE_STARTTLS)
+			pop3c_client_starttls(client);
+		else
+			pop3c_client_authenticate1(client);
+		break;
+	case POP3C_CLIENT_STATE_STARTTLS:
+		if (!success) {
+			i_error("pop3c(%s): STLS failed: %s",
+				client->set.host, line);
+			return -1;
+		}
+		if (pop3c_client_ssl_init(client) < 0)
+			pop3c_client_disconnect(client);
+		break;
+	case POP3C_CLIENT_STATE_USER:
+		if (!success) {
+			i_error("pop3c(%s): USER failed: %s",
+				client->set.host, line);
+			return -1;
+		}
+
+		/* the PASS reply can take a long time.
+		   switch to command timeout. */
+		timeout_remove(&client->to);
+		client->to = timeout_add(POP3C_COMMAND_TIMEOUT_MSECS,
+					 pop3c_client_timeout, client);
+
+		o_stream_nsend_str(client->output,
+			t_strdup_printf("PASS %s\r\n", client->set.password));
+		client->state = POP3C_CLIENT_STATE_PASS;
+		client->auth_mech = "USER+PASS";
+		break;
+	case POP3C_CLIENT_STATE_AUTH:
+		if (line[0] != '+') {
+			i_error("pop3c(%s): AUTH PLAIN failed: %s",
+				client->set.host, line);
+			return -1;
+		}
+		o_stream_nsend_str(client->output,
+			pop3c_client_get_sasl_plain_request(client));
+		client->state = POP3C_CLIENT_STATE_PASS;
+		client->auth_mech = "AUTH PLAIN";
+		break;
+	case POP3C_CLIENT_STATE_PASS:
+		if (client->login_callback != NULL) {
+			reply = strncasecmp(line, "+OK ", 4) == 0 ? line + 4 :
+				strncasecmp(line, "-ERR ", 5) == 0 ? line + 5 :
+				line;
+			client_login_callback(client, success ?
+					      POP3C_COMMAND_STATE_OK :
+					      POP3C_COMMAND_STATE_ERR, reply);
+		} else if (!success) {
+			i_error("pop3c(%s): Authentication via %s failed: %s",
+				client->set.host, client->auth_mech, line);
+		}
+		if (!success)
+			return -1;
+
+		o_stream_nsend_str(client->output, "CAPA\r\n");
+		client->state = POP3C_CLIENT_STATE_CAPA;
+		break;
+	case POP3C_CLIENT_STATE_CAPA:
+		if (strncasecmp(line, "-ERR", 4) == 0) {
+			/* CAPA command not supported. some commands still
+			   support UIDL though. */
+			client->capabilities |= POP3C_CAPABILITY_UIDL;
+			pop3c_client_login_finished(client);
+			break;
+		} else if (strcmp(line, ".") == 0) {
+			pop3c_client_login_finished(client);
+			break;
+		}
+		if ((client->set.parsed_features & POP3C_FEATURE_NO_PIPELINING) == 0 &&
+		    strcasecmp(line, "PIPELINING") == 0)
+			client->capabilities |= POP3C_CAPABILITY_PIPELINING;
+		else if (strcasecmp(line, "TOP") == 0)
+			client->capabilities |= POP3C_CAPABILITY_TOP;
+		else if (strcasecmp(line, "UIDL") == 0)
+			client->capabilities |= POP3C_CAPABILITY_UIDL;
+		break;
+	case POP3C_CLIENT_STATE_DISCONNECTED:
+	case POP3C_CLIENT_STATE_DONE:
+		i_unreached();
+	}
+	return 0;
+}
+
+static void pop3c_client_prelogin_input(struct pop3c_client *client)
+{
+	const char *line, *errstr;
+
+	i_assert(client->state != POP3C_CLIENT_STATE_DONE);
+
+	/* we need to read as much as we can with SSL streams to avoid
+	   hanging */
+	while ((line = i_stream_read_next_line(client->input)) != NULL) {
+		if (pop3c_client_prelogin_input_line(client, line) < 0) {
+			pop3c_client_disconnect(client);
+			return;
+		}
+	}
+
+	if (client->input->closed || client->input->eof ||
+	    client->input->stream_errno != 0) {
+		/* disconnected */
+		if (client->ssl_iostream == NULL) {
+			i_error("pop3c(%s): Server disconnected unexpectedly",
+				client->set.host);
+		} else {
+			errstr = ssl_iostream_get_last_error(client->ssl_iostream);
+			if (errstr == NULL) {
+				errstr = client->input->stream_errno == 0 ? "EOF" :
+					strerror(client->input->stream_errno);
+			}
+			i_error("pop3c(%s): Server disconnected: %s",
+				client->set.host, errstr);
+		}
+		pop3c_client_disconnect(client);
+	}
+}
+
+static int pop3c_client_ssl_handshaked(const char **error_r, void *context)
+{
+	struct pop3c_client *client = context;
+	const char *error;
+
+	if (ssl_iostream_check_cert_validity(client->ssl_iostream,
+					     client->set.host, &error) == 0) {
+		if (client->set.debug) {
+			i_debug("pop3c(%s): SSL handshake successful",
+				client->set.host);
+		}
+		return 0;
+	} else if (client->set.ssl_set.allow_invalid_cert) {
+		if (client->set.debug) {
+			i_debug("pop3c(%s): SSL handshake successful, "
+				"ignoring invalid certificate: %s",
+				client->set.host, error);
+		}
+		return 0;
+	} else {
+		*error_r = error;
+		return -1;
+	}
+}
+
+static int pop3c_client_ssl_init(struct pop3c_client *client)
+{
+	const char *error;
+
+	if (client->ssl_ctx == NULL) {
+		i_error("pop3c(%s): No SSL context", client->set.host);
+		return -1;
+	}
+
+	if (client->set.debug)
+		i_debug("pop3c(%s): Starting SSL handshake", client->set.host);
+
+	if (client->raw_input != client->input) {
+		/* recreate rawlog after STARTTLS */
+		i_stream_ref(client->raw_input);
+		o_stream_ref(client->raw_output);
+		i_stream_destroy(&client->input);
+		o_stream_destroy(&client->output);
+		client->input = client->raw_input;
+		client->output = client->raw_output;
+	}
+
+	if (io_stream_create_ssl_client(client->ssl_ctx, client->set.host,
+					&client->set.ssl_set, &client->input,
+					&client->output, &client->ssl_iostream, &error) < 0) {
+		i_error("pop3c(%s): Couldn't initialize SSL client: %s",
+			client->set.host, error);
+		return -1;
+	}
+	ssl_iostream_set_handshake_callback(client->ssl_iostream,
+					    pop3c_client_ssl_handshaked,
+					    client);
+	if (ssl_iostream_handshake(client->ssl_iostream) < 0) {
+		i_error("pop3c(%s): SSL handshake failed: %s", client->set.host,
+			ssl_iostream_get_last_error(client->ssl_iostream));
+		return -1;
+	}
+
+	if (*client->set.rawlog_dir != '\0') {
+		iostream_rawlog_create(client->set.rawlog_dir,
+				       &client->input, &client->output);
+	}
+	return 0;
+}
+
+static void pop3c_client_connected(struct pop3c_client *client)
+{
+	int err;
+
+	err = net_geterror(client->fd);
+	if (err != 0) {
+		i_error("pop3c(%s): connect(%s, %u) failed: %s",
+			client->set.host, net_ip2addr(&client->ip),
+			client->set.port, strerror(err));
+		pop3c_client_disconnect(client);
+		return;
+	}
+	io_remove(&client->io);
+	client->io = io_add(client->fd, IO_READ,
+			    pop3c_client_prelogin_input, client);
+
+	if (client->set.ssl_mode == POP3C_CLIENT_SSL_MODE_IMMEDIATE) {
+		if (pop3c_client_ssl_init(client) < 0)
+			pop3c_client_disconnect(client);
+	}
+}
+
+static void pop3c_client_connect_ip(struct pop3c_client *client)
+{
+	client->fd = net_connect_ip(&client->ip, client->set.port, NULL);
+	if (client->fd == -1) {
+		pop3c_client_disconnect(client);
+		return;
+	}
+
+	client->input = client->raw_input =
+		i_stream_create_fd(client->fd, POP3C_MAX_INBUF_SIZE);
+	client->output = client->raw_output =
+		o_stream_create_fd(client->fd, SIZE_MAX);
+	o_stream_set_no_error_handling(client->output, TRUE);
+
+	if (*client->set.rawlog_dir != '\0' &&
+	    client->set.ssl_mode != POP3C_CLIENT_SSL_MODE_IMMEDIATE) {
+		iostream_rawlog_create(client->set.rawlog_dir,
+				       &client->input, &client->output);
+	}
+	client->io = io_add(client->fd, IO_WRITE,
+			    pop3c_client_connected, client);
+	client->to = timeout_add(POP3C_CONNECT_TIMEOUT_MSECS,
+				 pop3c_client_timeout, client);
+	if (client->set.debug) {
+		i_debug("pop3c(%s): Connecting to %s:%u", client->set.host,
+			net_ip2addr(&client->ip), client->set.port);
+	}
+}
+
+static void
+pop3c_dns_callback(const struct dns_lookup_result *result,
+		   struct pop3c_client *client)
+{
+	client->dns_lookup = NULL;
+
+	if (result->ret != 0) {
+		i_error("pop3c(%s): dns_lookup() failed: %s",
+			client->set.host, result->error);
+		pop3c_client_disconnect(client);
+		return;
+	}
+
+	i_assert(result->ips_count > 0);
+	client->ip = result->ips[0];
+	pop3c_client_connect_ip(client);
+}
+
+void pop3c_client_login(struct pop3c_client *client,
+			pop3c_login_callback_t *callback, void *context)
+{
+	if (client->fd != -1) {
+		i_assert(callback == NULL);
+		return;
+	}
+	i_assert(client->login_callback == NULL);
+	client->login_callback = callback;
+	client->login_context = context;
+	client->state = POP3C_CLIENT_STATE_CONNECTING;
+
+	if (client->set.debug)
+		i_debug("pop3c(%s): Looking up IP address", client->set.host);
+}
+
+bool pop3c_client_is_connected(struct pop3c_client *client)
+{
+	return client->fd != -1;
+}
+
+enum pop3c_capability
+pop3c_client_get_capabilities(struct pop3c_client *client)
+{
+	return client->capabilities;
+}
+
+static int pop3c_client_dot_input(struct pop3c_client *client)
+{
+	ssize_t ret;
+
+	while ((ret = i_stream_read(client->dot_input)) > 0 || ret == -2) {
+		i_stream_skip(client->dot_input,
+			      i_stream_get_data_size(client->dot_input));
+	}
+	if (ret == 0)
+		return 0;
+	i_assert(ret == -1);
+
+	if (client->dot_input->stream_errno == 0)
+		ret = 1;
+	client->dot_input = NULL;
+
+	if (ret > 0) {
+		/* currently we don't actually care about preserving the
+		   +OK reply line for multi-line replies, so just return
+		   it as empty */
+		pop3c_client_async_callback(client, POP3C_COMMAND_STATE_OK, "");
+		return 1;
+	} else {
+		pop3c_client_async_callback_disconnected(client);
+		return -1;
+	}
+}
+
+static int
+pop3c_client_input_next_reply(struct pop3c_client *client)
+{
+	const char *line;
+	enum pop3c_command_state state;
+
+	line = i_stream_read_next_line(client->input);
+	if (line == NULL)
+		return client->input->eof ? -1 : 0;
+
+	if (strncasecmp(line, "+OK", 3) == 0) {
+		line += 3;
+		state = POP3C_COMMAND_STATE_OK;
+	} else if (strncasecmp(line, "-ERR", 4) == 0) {
+		line += 4;
+		state = POP3C_COMMAND_STATE_ERR;
+	} else {
+		i_error("pop3c(%s): Server sent unrecognized line: %s",
+			client->set.host, line);
+		state = POP3C_COMMAND_STATE_ERR;
+	}
+	if (line[0] == ' ')
+		line++;
+	if (array_count(&client->commands) == 0) {
+		i_error("pop3c(%s): Server sent line when no command was running: %s",
+			client->set.host, line);
+	} else {
+		pop3c_client_async_callback(client, state, line);
+	}
+	return 1;
+}
+
+static void pop3c_client_input(struct pop3c_client *client)
+{
+	int ret;
+
+	if (client->to != NULL)
+		timeout_reset(client->to);
+	do {
+		if (client->dot_input != NULL) {
+			/* continue reading the current multiline reply */
+			if ((ret = pop3c_client_dot_input(client)) == 0)
+				return;
+		} else {
+			ret = pop3c_client_input_next_reply(client);
+		}
+	} while (ret > 0);
+
+	if (ret < 0) {
+		i_error("pop3c(%s): Server disconnected unexpectedly",
+			client->set.host);
+		pop3c_client_disconnect(client);
+	}
+}
+
+static void pop3c_client_cmd_reply(enum pop3c_command_state state,
+				   const char *reply, void *context)
+{
+	struct pop3c_client_sync_cmd_ctx *ctx = context;
+
+	i_assert(ctx->reply == NULL);
+
+	ctx->state = state;
+	ctx->reply = i_strdup(reply);
+}
+
+int pop3c_client_cmd_line(struct pop3c_client *client, const char *cmdline,
+			  const char **reply_r)
+{
+	struct pop3c_client_sync_cmd_ctx ctx;
+
+	i_zero(&ctx);
+	pop3c_client_cmd_line_async(client, cmdline, pop3c_client_cmd_reply, &ctx);
+	while (ctx.reply == NULL)
+		pop3c_client_wait_one(client);
+	*reply_r = t_strdup(ctx.reply);
+	i_free(ctx.reply);
+	return ctx.state == POP3C_COMMAND_STATE_OK ? 0 : -1;
+}
+
+struct pop3c_client_cmd *
+pop3c_client_cmd_line_async(struct pop3c_client *client, const char *cmdline,
+			    pop3c_cmd_callback_t *callback, void *context)
+{
+	struct pop3c_client_cmd *cmd;
+
+	if ((client->capabilities & POP3C_CAPABILITY_PIPELINING) == 0) {
+		while (array_count(&client->commands) > 0)
+			pop3c_client_wait_one(client);
+	}
+	i_assert(client->state == POP3C_CLIENT_STATE_DISCONNECTED ||
+		 client->state == POP3C_CLIENT_STATE_DONE);
+	if (client->state == POP3C_CLIENT_STATE_DONE)
+		o_stream_nsend_str(client->output, cmdline);
+
+	cmd = array_append_space(&client->commands);
+	cmd->callback = callback;
+	cmd->context = context;
+	return cmd;
+}
+
+void pop3c_client_cmd_line_async_nocb(struct pop3c_client *client,
+				      const char *cmdline)
+{
+	pop3c_client_cmd_line_async(client, cmdline, NULL, NULL);
+}
+
+static int seekable_fd_callback(const char **path_r, void *context)
+{
+	struct pop3c_client *client = context;
+	string_t *path;
+	int fd;
+
+	path = t_str_new(128);
+	str_append(path, client->set.temp_path_prefix);
+	fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1);
+	if (fd == -1) {
+		i_error("safe_mkstemp(%s) failed: %m", str_c(path));
+		return -1;
+	}
+
+	/* we just want the fd, unlink it */
+	if (i_unlink(str_c(path)) < 0) {
+		/* shouldn't happen.. */
+		i_close_fd(&fd);
+		return -1;
+	}
+
+	*path_r = str_c(path);
+	return fd;
+}
+
+int pop3c_client_cmd_stream(struct pop3c_client *client, const char *cmdline,
+			    struct istream **input_r, const char **error_r)
+{
+	struct pop3c_client_sync_cmd_ctx ctx;
+	const char *reply;
+
+	if (client->state == POP3C_CLIENT_STATE_DISCONNECTED) {
+		*error_r = "Disconnected from server";
+		return -1;
+	}
+
+	i_zero(&ctx);
+	*input_r = pop3c_client_cmd_stream_async(client, cmdline,
+						 pop3c_client_cmd_reply, &ctx);
+	while (ctx.reply == NULL)
+		pop3c_client_wait_one(client);
+	reply = t_strdup(ctx.reply);
+	i_free(ctx.reply);
+
+	if (ctx.state == POP3C_COMMAND_STATE_OK)
+		return 0;
+	i_stream_unref(input_r);
+	*error_r = reply;
+	return -1;
+}
+
+struct istream *
+pop3c_client_cmd_stream_async(struct pop3c_client *client, const char *cmdline,
+			      pop3c_cmd_callback_t *callback, void *context)
+{
+	struct istream *input, *inputs[2];
+	struct pop3c_client_cmd *cmd;
+
+	cmd = pop3c_client_cmd_line_async(client, cmdline, callback, context);
+
+	input = i_stream_create_chain(&cmd->chain, POP3C_MAX_INBUF_SIZE);
+	inputs[0] = i_stream_create_dot(input, TRUE);
+	inputs[1] = NULL;
+	cmd->input = i_stream_create_seekable(inputs, POP3C_MAX_INBUF_SIZE,
+					      seekable_fd_callback, client);
+	i_stream_unref(&input);
+	i_stream_unref(&inputs[0]);
+
+	i_stream_ref(cmd->input);
+	return cmd->input;
+}
diff --git a/src/lib-storage/index/pop3c/pop3c-client.h b/src/lib-storage/index/pop3c/pop3c-client.h
new file mode 100644
index 0000000..f0bbd64
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-client.h
@@ -0,0 +1,86 @@
+#ifndef POP3C_CLIENT_H
+#define POP3C_CLIENT_H
+
+#include "net.h"
+#include "pop3c-settings.h"
+#include "iostream-ssl.h"
+
+enum pop3c_capability {
+	POP3C_CAPABILITY_PIPELINING	= 0x01,
+	POP3C_CAPABILITY_TOP		= 0x02,
+	POP3C_CAPABILITY_UIDL		= 0x04
+};
+
+enum pop3c_command_state {
+	POP3C_COMMAND_STATE_OK,
+	POP3C_COMMAND_STATE_ERR,
+	POP3C_COMMAND_STATE_DISCONNECTED
+};
+
+enum pop3c_client_ssl_mode {
+	POP3C_CLIENT_SSL_MODE_NONE,
+	POP3C_CLIENT_SSL_MODE_IMMEDIATE,
+	POP3C_CLIENT_SSL_MODE_STARTTLS
+};
+
+struct pop3c_client_settings {
+	const char *host;
+	in_port_t port;
+
+	const char *master_user;
+	const char *username;
+	const char *password;
+
+	const char *dns_client_socket_path;
+	const char *temp_path_prefix;
+
+	enum pop3c_client_ssl_mode ssl_mode;
+	enum pop3c_features parsed_features;
+	struct ssl_iostream_settings ssl_set;
+
+	const char *rawlog_dir;
+	const char *ssl_crypto_device;
+	bool debug;
+};
+
+typedef void pop3c_login_callback_t(enum pop3c_command_state state,
+				    const char *reply, void *context);
+typedef void pop3c_cmd_callback_t(enum pop3c_command_state state,
+				  const char *reply, void *context);
+
+struct pop3c_client *
+pop3c_client_init(const struct pop3c_client_settings *set,
+		  struct event *event_parent);
+void pop3c_client_deinit(struct pop3c_client **client);
+
+void pop3c_client_login(struct pop3c_client *client,
+			pop3c_login_callback_t *callback, void *context);
+
+bool pop3c_client_is_connected(struct pop3c_client *client);
+enum pop3c_capability
+pop3c_client_get_capabilities(struct pop3c_client *client);
+
+/* Returns 0 if received +OK reply, reply contains the text without the +OK.
+   Returns -1 if received -ERR reply or disconnected. */
+int pop3c_client_cmd_line(struct pop3c_client *client, const char *cmdline,
+			  const char **reply_r);
+/* Start the command asynchronously. Call the callback when finished. */
+struct pop3c_client_cmd *
+pop3c_client_cmd_line_async(struct pop3c_client *client, const char *cmdline,
+			    pop3c_cmd_callback_t *callback, void *context);
+/* Send a command, don't care if it succeeds or not. */
+void pop3c_client_cmd_line_async_nocb(struct pop3c_client *client,
+				      const char *cmdline);
+/* Returns 0 and stream if succeeded, -1 and error if received -ERR reply or
+   disconnected. */
+int pop3c_client_cmd_stream(struct pop3c_client *client, const char *cmdline,
+			    struct istream **input_r, const char **error_r);
+/* Start the command asynchronously. Call the callback when finished. */
+struct istream *
+pop3c_client_cmd_stream_async(struct pop3c_client *client, const char *cmdline,
+			      pop3c_cmd_callback_t *callback, void *context);
+/* Wait for the next async command to finish. It's an error to call this when
+   there are no pending async commands. */
+void pop3c_client_wait_one(struct pop3c_client *client);
+
+#endif
diff --git a/src/lib-storage/index/pop3c/pop3c-mail.c b/src/lib-storage/index/pop3c/pop3c-mail.c
new file mode 100644
index 0000000..27a8427
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-mail.c
@@ -0,0 +1,304 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "index-mail.h"
+#include "pop3c-client.h"
+#include "pop3c-sync.h"
+#include "pop3c-storage.h"
+
+struct mail *
+pop3c_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers)
+{
+	struct pop3c_mail *mail;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mail", 2048);
+	mail = p_new(pool, struct pop3c_mail, 1);
+
+	index_mail_init(&mail->imail, t, wanted_fields, wanted_headers, pool, NULL);
+	return &mail->imail.mail.mail;
+}
+
+static void pop3c_mail_close(struct mail *_mail)
+{
+	struct pop3c_mail *pmail = POP3C_MAIL(_mail);
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box);
+
+	/* wait for any prefetch to finish before closing the mail */
+	while (pmail->prefetching)
+		pop3c_client_wait_one(mbox->client);
+	i_stream_unref(&pmail->prefetch_stream);
+	index_mail_close(_mail);
+}
+
+static int pop3c_mail_get_received_date(struct mail *_mail, time_t *date_r)
+{
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box);
+	int tz;
+
+	if (mbox->storage->set->pop3c_quick_received_date) {
+		/* we don't care about the date, just return the current date */
+		*date_r = ioloop_time;
+		return 0;
+	}
+
+	/* FIXME: we could also parse the first Received: header and get
+	   the date from there, but since this code is unlikely to be called
+	   except during migration, I don't think it really matters. */
+	return index_mail_get_date(_mail, date_r, &tz);
+}
+
+static int pop3c_mail_get_save_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct index_mail_data *data = &mail->data;
+
+	if (data->save_date == (time_t)-1) {
+		/* FIXME: we could use a value stored in cache */
+		if (pop3c_mail_get_received_date(_mail, date_r) < 0)
+			return -1;
+		return 0;
+	}
+	*date_r = data->save_date;
+	return 0;
+}
+
+static int pop3c_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box);
+	struct message_size hdr_size, body_size;
+	struct istream *input;
+
+	if (mail->data.virtual_size != UOFF_T_MAX) {
+		/* virtual size is already known. it's the same as our
+		   (correct) physical size */
+		*size_r = mail->data.virtual_size;
+		return 0;
+	}
+	if (index_mail_get_physical_size(_mail, size_r) == 0) {
+		*size_r = mail->data.physical_size;
+		return 0;
+	}
+
+	if (_mail->lookup_abort == MAIL_LOOKUP_ABORT_READ_MAIL &&
+	    (_mail->box->flags & MAILBOX_FLAG_POP3_SESSION) != 0) {
+		/* kludge: we want output for POP3 LIST with
+		   pop3_fast_size_lookups=yes. use the remote's LIST values
+		   regardless of their correctness */
+		if (mbox->msg_sizes == NULL) {
+			if (pop3c_sync_get_sizes(mbox) < 0)
+				return -1;
+		}
+		i_assert(_mail->seq <= mbox->msg_count);
+		*size_r = mbox->msg_sizes[_mail->seq-1];
+		return 0;
+	}
+
+	/* slow way: get the whole message body */
+	if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0)
+		return -1;
+
+	i_assert(mail->data.physical_size != UOFF_T_MAX);
+	*size_r = mail->data.physical_size;
+	return 0;
+}
+
+static void pop3c_mail_cache_size(struct index_mail *mail)
+{
+	uoff_t size;
+
+	if (i_stream_get_size(mail->data.stream, TRUE, &size) <= 0)
+		return;
+	mail->data.virtual_size = size;
+	/* it'll be actually added to index when closing the mail in
+	   index_mail_cache_sizes() */
+}
+
+static void
+pop3c_mail_prefetch_done(enum pop3c_command_state state,
+			 const char *reply ATTR_UNUSED, void *context)
+{
+	struct pop3c_mail *pmail = context;
+
+	switch (state) {
+	case POP3C_COMMAND_STATE_OK:
+		break;
+	case POP3C_COMMAND_STATE_ERR:
+	case POP3C_COMMAND_STATE_DISCONNECTED:
+		i_stream_unref(&pmail->prefetch_stream);
+		/* let pop3c_mail_get_stream() figure out the error handling.
+		   in case of a -ERR a retry might even work. */
+		break;
+	}
+	pmail->prefetching = FALSE;
+}
+
+static bool pop3c_mail_prefetch(struct mail *_mail)
+{
+	struct pop3c_mail *pmail = POP3C_MAIL(_mail);
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box);
+	enum pop3c_capability capa;
+	const char *cmd;
+
+	if (pmail->imail.data.access_part != 0 &&
+	    pmail->imail.data.stream == NULL &&
+	    mail_stream_access_start(_mail)) {
+		capa = pop3c_client_get_capabilities(mbox->client);
+		pmail->prefetching_body = (capa & POP3C_CAPABILITY_TOP) == 0 ||
+			(pmail->imail.data.access_part & (READ_BODY | PARSE_BODY)) != 0;
+		if (pmail->prefetching_body)
+			cmd = t_strdup_printf("RETR %u\r\n", _mail->seq);
+		else
+			cmd = t_strdup_printf("TOP %u 0\r\n", _mail->seq);
+
+		pmail->prefetching = TRUE;
+		pmail->prefetch_stream =
+			pop3c_client_cmd_stream_async(mbox->client, cmd,
+				pop3c_mail_prefetch_done, pmail);
+		i_stream_set_name(pmail->prefetch_stream, t_strcut(cmd, '\r'));
+		return !pmail->prefetching;
+	}
+	return index_mail_prefetch(_mail);
+}
+
+static int
+pop3c_mail_get_stream(struct mail *_mail, bool get_body,
+		      struct message_size *hdr_size,
+		      struct message_size *body_size, struct istream **stream_r)
+{
+	struct pop3c_mail *pmail = POP3C_MAIL(_mail);
+	struct index_mail *mail = &pmail->imail;
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box);
+	enum pop3c_capability capa;
+	const char *name, *cmd, *error;
+	struct istream *input;
+	bool new_stream = FALSE;
+
+	if ((mail->data.access_part & (READ_BODY | PARSE_BODY)) != 0)
+		get_body = TRUE;
+
+	while (pmail->prefetching) {
+		/* wait for prefetch to finish */
+		pop3c_client_wait_one(mbox->client);
+	}
+
+	if (pmail->prefetch_stream != NULL && mail->data.stream == NULL) {
+		mail->data.stream = pmail->prefetch_stream;
+		pmail->prefetch_stream = NULL;
+		new_stream = TRUE;
+	}
+
+	if (get_body && mail->data.stream != NULL) {
+		name = i_stream_get_name(mail->data.stream);
+		if (str_begins(name, "RETR")) {
+			/* we've fetched the body */
+		} else if (str_begins(name, "TOP")) {
+			/* we've fetched the header, but we need the body
+			   now too */
+			index_mail_close_streams(mail);
+		} else {
+			i_panic("Unexpected POP3 stream name: %s", name);
+		}
+	}
+
+	if (mail->data.stream == NULL) {
+		if (!mail_stream_access_start(_mail))
+			return -1;
+		capa = pop3c_client_get_capabilities(mbox->client);
+		if (get_body || (capa & POP3C_CAPABILITY_TOP) == 0) {
+			cmd = t_strdup_printf("RETR %u\r\n", _mail->seq);
+			get_body = TRUE;
+		} else {
+			cmd = t_strdup_printf("TOP %u 0\r\n", _mail->seq);
+		}
+		if (pop3c_client_cmd_stream(mbox->client, cmd,
+					    &input, &error) < 0) {
+			mail_storage_set_error(mbox->box.storage,
+				!pop3c_client_is_connected(mbox->client) ?
+				MAIL_ERROR_TEMP : MAIL_ERROR_EXPUNGED, error);
+			return -1;
+		}
+		mail->data.stream = input;
+		i_stream_set_name(mail->data.stream, t_strcut(cmd, '\r'));
+		new_stream = TRUE;
+	}
+	if (new_stream) {
+		if (mail->mail.v.istream_opened != NULL) {
+			if (mail->mail.v.istream_opened(_mail,
+							&mail->data.stream) < 0) {
+				index_mail_close_streams(mail);
+				return -1;
+			}
+		}
+		if (get_body)
+			pop3c_mail_cache_size(mail);
+	}
+	/* if this stream is used by some filter stream, make the
+	   filter stream blocking */
+	mail->data.stream->blocking = TRUE;
+	return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
+}
+
+static int
+pop3c_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+		       const char **value_r)
+{
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(_mail->box);
+
+	switch (field) {
+	case MAIL_FETCH_UIDL_BACKEND:
+	case MAIL_FETCH_GUID:
+		if (mbox->msg_uidls == NULL) {
+			if (pop3c_sync_get_uidls(mbox) < 0)
+				return -1;
+		}
+		i_assert(_mail->seq <= mbox->msg_count);
+		*value_r = mbox->msg_uidls[_mail->seq-1];
+		return 0;
+	default:
+		return index_mail_get_special(_mail, field, value_r);
+	}
+}
+
+struct mail_vfuncs pop3c_mail_vfuncs = {
+	pop3c_mail_close,
+	index_mail_free,
+	index_mail_set_seq,
+	index_mail_set_uid,
+	index_mail_set_uid_cache_updates,
+	pop3c_mail_prefetch,
+	index_mail_precache,
+	index_mail_add_temp_wanted_fields,
+
+	index_mail_get_flags,
+	index_mail_get_keywords,
+	index_mail_get_keyword_indexes,
+	index_mail_get_modseq,
+	index_mail_get_pvt_modseq,
+	index_mail_get_parts,
+	index_mail_get_date,
+	pop3c_mail_get_received_date,
+	pop3c_mail_get_save_date,
+	index_mail_get_virtual_size,
+	pop3c_mail_get_physical_size,
+	index_mail_get_first_header,
+	index_mail_get_headers,
+	index_mail_get_header_stream,
+	pop3c_mail_get_stream,
+	index_mail_get_binary_stream,
+	pop3c_mail_get_special,
+	index_mail_get_backend_mail,
+	index_mail_update_flags,
+	index_mail_update_keywords,
+	index_mail_update_modseq,
+	index_mail_update_pvt_modseq,
+	NULL,
+	index_mail_expunge,
+	index_mail_set_cache_corrupted,
+	index_mail_opened,
+};
diff --git a/src/lib-storage/index/pop3c/pop3c-settings.c b/src/lib-storage/index/pop3c/pop3c-settings.c
new file mode 100644
index 0000000..db876e1
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-settings.c
@@ -0,0 +1,116 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "settings-parser.h"
+#include "mail-storage-settings.h"
+#include "pop3c-settings.h"
+
+#include <stddef.h>
+
+#undef DEF
+#define DEF(type, name) \
+	SETTING_DEFINE_STRUCT_##type(#name, name, struct pop3c_settings)
+
+static const struct setting_define pop3c_setting_defines[] = {
+	DEF(STR, pop3c_host),
+	DEF(IN_PORT, pop3c_port),
+
+	DEF(STR_VARS, pop3c_user),
+	DEF(STR_VARS, pop3c_master_user),
+	DEF(STR, pop3c_password),
+
+	DEF(ENUM, pop3c_ssl),
+	DEF(BOOL, pop3c_ssl_verify),
+
+	DEF(STR, pop3c_rawlog_dir),
+	DEF(BOOL, pop3c_quick_received_date),
+
+	DEF(STR, pop3c_features),
+
+	SETTING_DEFINE_LIST_END
+};
+
+static const struct pop3c_settings pop3c_default_settings = {
+	.pop3c_host = "",
+	.pop3c_port = 110,
+
+	.pop3c_user = "%u",
+	.pop3c_master_user = "",
+	.pop3c_password = "",
+
+	.pop3c_ssl = "no:pop3s:starttls",
+	.pop3c_ssl_verify = TRUE,
+
+	.pop3c_rawlog_dir = "",
+	.pop3c_quick_received_date = FALSE,
+
+	.pop3c_features = ""
+};
+
+/* <settings checks> */
+struct pop3c_feature_list {
+	const char *name;
+	enum pop3c_features num;
+};
+
+static const struct pop3c_feature_list pop3c_feature_list[] = {
+	{ "no-pipelining", POP3C_FEATURE_NO_PIPELINING },
+	{ NULL, 0 }
+};
+
+static int
+pop3c_settings_parse_features(struct pop3c_settings *set,
+			      const char **error_r)
+{
+	enum pop3c_features features = 0;
+	const struct pop3c_feature_list *list;
+	const char *const *str;
+
+	str = t_strsplit_spaces(set->pop3c_features, " ,");
+	for (; *str != NULL; str++) {
+		list = pop3c_feature_list;
+		for (; list->name != NULL; list++) {
+			if (strcasecmp(*str, list->name) == 0) {
+				features |= list->num;
+				break;
+			}
+		}
+		if (list->name == NULL) {
+			*error_r = t_strdup_printf("pop3c_features: "
+				"Unknown feature: %s", *str);
+			return -1;
+		}
+	}
+	set->parsed_features = features;
+	return 0;
+}
+
+static bool pop3c_settings_check(void *_set, pool_t pool ATTR_UNUSED,
+				 const char **error_r)
+{
+	struct pop3c_settings *set = _set;
+
+	if (pop3c_settings_parse_features(set, error_r) < 0)
+		return FALSE;
+	return TRUE;
+}
+/* </settings checks> */
+
+static const struct setting_parser_info pop3c_setting_parser_info = {
+	.module_name = "pop3c",
+	.defines = pop3c_setting_defines,
+	.defaults = &pop3c_default_settings,
+
+	.type_offset = SIZE_MAX,
+	.struct_size = sizeof(struct pop3c_settings),
+
+	.parent_offset = SIZE_MAX,
+	.parent = &mail_user_setting_parser_info,
+
+        .check_func = pop3c_settings_check
+};
+
+const struct setting_parser_info *pop3c_get_setting_parser_info(void)
+{
+	return &pop3c_setting_parser_info;
+}
diff --git a/src/lib-storage/index/pop3c/pop3c-settings.h b/src/lib-storage/index/pop3c/pop3c-settings.h
new file mode 100644
index 0000000..bf44e24
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-settings.h
@@ -0,0 +1,33 @@
+#ifndef POP3C_SETTINGS_H
+#define POP3C_SETTINGS_H
+
+#include "net.h"
+
+/* <settings checks> */
+enum pop3c_features {
+        POP3C_FEATURE_NO_PIPELINING = 0x1,
+};
+/* </settings checks> */
+
+
+struct pop3c_settings {
+	const char *pop3c_host;
+	in_port_t pop3c_port;
+
+	const char *pop3c_user;
+	const char *pop3c_master_user;
+	const char *pop3c_password;
+
+	const char *pop3c_ssl;
+	bool pop3c_ssl_verify;
+
+	const char *pop3c_rawlog_dir;
+	bool pop3c_quick_received_date;
+
+	const char *pop3c_features;
+	enum pop3c_features parsed_features;
+};
+
+const struct setting_parser_info *pop3c_get_setting_parser_info(void);
+
+#endif
diff --git a/src/lib-storage/index/pop3c/pop3c-storage.c b/src/lib-storage/index/pop3c/pop3c-storage.c
new file mode 100644
index 0000000..f784683
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-storage.c
@@ -0,0 +1,368 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "str.h"
+#include "mail-copy.h"
+#include "mail-user.h"
+#include "mailbox-list-private.h"
+#include "index-mail.h"
+#include "pop3c-client.h"
+#include "pop3c-sync.h"
+#include "pop3c-storage.h"
+
+#define DNS_CLIENT_SOCKET_NAME "dns-client"
+
+extern struct mail_storage pop3c_storage;
+extern struct mailbox pop3c_mailbox;
+
+static struct event_category event_category_pop3c = {
+	.name = "pop3c",
+	.parent = &event_category_storage,
+};
+
+static struct mail_storage *pop3c_storage_alloc(void)
+{
+	struct pop3c_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("pop3c storage", 512+256);
+	storage = p_new(pool, struct pop3c_storage, 1);
+	storage->storage = pop3c_storage;
+	storage->storage.pool = pool;
+	return &storage->storage;
+}
+
+static int
+pop3c_storage_create(struct mail_storage *_storage,
+		     struct mail_namespace *ns,
+		     const char **error_r)
+{
+	struct pop3c_storage *storage = POP3C_STORAGE(_storage);
+
+	storage->set = mail_namespace_get_driver_settings(ns, _storage);
+	if (storage->set->pop3c_host[0] == '\0') {
+		*error_r = "missing pop3c_host";
+		return -1;
+	}
+	if (storage->set->pop3c_password[0] == '\0') {
+		*error_r = "missing pop3c_password";
+		return -1;
+	}
+
+	return 0;
+}
+
+static struct pop3c_client *
+pop3c_client_create_from_set(struct mail_storage *storage,
+			     const struct pop3c_settings *set)
+{
+	struct pop3c_client_settings client_set;
+	string_t *str;
+
+	i_zero(&client_set);
+	client_set.host = set->pop3c_host;
+	client_set.port = set->pop3c_port;
+	client_set.username = set->pop3c_user;
+	client_set.master_user = set->pop3c_master_user;
+	client_set.password = set->pop3c_password;
+	client_set.dns_client_socket_path =
+		storage->user->set->base_dir[0] == '\0' ? "" :
+		t_strconcat(storage->user->set->base_dir, "/",
+			    DNS_CLIENT_SOCKET_NAME, NULL);
+	str = t_str_new(128);
+	mail_user_set_get_temp_prefix(str, storage->user->set);
+	client_set.temp_path_prefix = str_c(str);
+
+	client_set.debug = storage->user->mail_debug;
+	client_set.rawlog_dir =
+		mail_user_home_expand(storage->user, set->pop3c_rawlog_dir);
+
+	mail_user_init_ssl_client_settings(storage->user, &client_set.ssl_set);
+
+	if (!set->pop3c_ssl_verify)
+		client_set.ssl_set.allow_invalid_cert = TRUE;
+
+	if (strcmp(set->pop3c_ssl, "pop3s") == 0)
+		client_set.ssl_mode = POP3C_CLIENT_SSL_MODE_IMMEDIATE;
+	else if (strcmp(set->pop3c_ssl, "starttls") == 0)
+		client_set.ssl_mode = POP3C_CLIENT_SSL_MODE_STARTTLS;
+	else
+		client_set.ssl_mode = POP3C_CLIENT_SSL_MODE_NONE;
+	return pop3c_client_init(&client_set, storage->event);
+}
+
+static void
+pop3c_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED,
+				struct mailbox_list_settings *set)
+{
+	set->layout = MAILBOX_LIST_NAME_FS;
+	if (set->root_dir != NULL && *set->root_dir != '\0' &&
+	    set->index_dir == NULL) {
+	       /* we don't really care about root_dir, but we
+		  just need to get index_dir autocreated. */
+		set->index_dir = set->root_dir;
+	}
+}
+
+static struct mailbox *
+pop3c_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
+		    const char *vname, enum mailbox_flags flags)
+{
+	struct pop3c_mailbox *mbox;
+	pool_t pool;
+
+	pool = pool_alloconly_create("pop3c mailbox", 1024*3);
+	mbox = p_new(pool, struct pop3c_mailbox, 1);
+	mbox->box = pop3c_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.list->props |= MAILBOX_LIST_PROP_AUTOCREATE_DIRS;
+	mbox->box.mail_vfuncs = &pop3c_mail_vfuncs;
+	mbox->storage = POP3C_STORAGE(storage);
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
+	return &mbox->box;
+}
+
+static int
+pop3c_mailbox_exists(struct mailbox *box, bool auto_boxes,
+		     enum mailbox_existence *existence_r)
+{
+	if ((auto_boxes && mailbox_is_autocreated(box)) || box->inbox_any)
+		*existence_r = MAILBOX_EXISTENCE_SELECT;
+	else
+		*existence_r = MAILBOX_EXISTENCE_NONE;
+	return 0;
+}
+
+static void pop3c_login_callback(enum pop3c_command_state state,
+				 const char *reply, void *context)
+{
+	struct pop3c_mailbox *mbox = context;
+
+	switch (state) {
+	case POP3C_COMMAND_STATE_OK:
+		mbox->logged_in = TRUE;
+		break;
+	case POP3C_COMMAND_STATE_ERR:
+		if (str_begins(reply, "[IN-USE] ")) {
+			mail_storage_set_error(mbox->box.storage,
+					       MAIL_ERROR_INUSE, reply + 9);
+		} else {
+			/* authentication failure probably */
+			mail_storage_set_error(mbox->box.storage,
+					       MAIL_ERROR_PARAMS, reply);
+		}
+		break;
+	case POP3C_COMMAND_STATE_DISCONNECTED:
+		mailbox_set_critical(&mbox->box,
+			"pop3c: Disconnected from remote server");
+		break;
+	}
+}
+
+static int pop3c_mailbox_open(struct mailbox *box)
+{
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(box);
+
+	if (strcmp(box->name, "INBOX") != 0) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND,
+				       T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname));
+		return -1;
+	}
+
+	if (index_storage_mailbox_open(box, FALSE) < 0)
+		return -1;
+
+	mbox->client = pop3c_client_create_from_set(box->storage,
+						    mbox->storage->set);
+	pop3c_client_login(mbox->client, pop3c_login_callback, mbox);
+	pop3c_client_wait_one(mbox->client);
+	return mbox->logged_in ? 0 : -1;
+}
+
+static void pop3c_mailbox_close(struct mailbox *box)
+{
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(box);
+
+	pool_unref(&mbox->uidl_pool);
+	i_free_and_null(mbox->msg_uids);
+	i_free_and_null(mbox->msg_sizes);
+	pop3c_client_deinit(&mbox->client);
+	index_storage_mailbox_close(box);
+}
+
+static int
+pop3c_mailbox_create(struct mailbox *box,
+		     const struct mailbox_update *update ATTR_UNUSED,
+		     bool directory ATTR_UNUSED)
+{
+	mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE,
+			       "POP3 mailbox creation isn't supported");
+	return -1;
+}
+
+static int
+pop3c_mailbox_update(struct mailbox *box,
+		     const struct mailbox_update *update ATTR_UNUSED)
+{
+	if (!guid_128_is_empty(update->mailbox_guid) ||
+	    update->uid_validity != 0 || update->min_next_uid != 0 ||
+	    update->min_first_recent_uid != 0) {
+		mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE,
+				       "POP3 mailbox update isn't supported");
+	}
+	return index_storage_mailbox_update(box, update);
+}
+
+static int pop3c_mailbox_get_status(struct mailbox *box,
+				    enum mailbox_status_items items,
+				    struct mailbox_status *status_r)
+{
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(box);
+
+	if (index_storage_get_status(box, items, status_r) < 0)
+		return -1;
+
+	if ((pop3c_client_get_capabilities(mbox->client) &
+	     POP3C_CAPABILITY_UIDL) == 0)
+		status_r->have_guids = FALSE;
+	return 0;
+}
+
+static int pop3c_mailbox_get_metadata(struct mailbox *box,
+				      enum mailbox_metadata_items items,
+				      struct mailbox_metadata *metadata_r)
+{
+	if ((items & MAILBOX_METADATA_GUID) != 0) {
+		/* a bit ugly way to do this, but better than nothing for now.
+		   FIXME: if indexes are enabled, keep this there. */
+		mail_generate_guid_128_hash(box->name, metadata_r->guid);
+		items &= ENUM_NEGATE(MAILBOX_METADATA_GUID);
+	}
+	if (items != 0) {
+		if (index_mailbox_get_metadata(box, items, metadata_r) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static void pop3c_notify_changes(struct mailbox *box ATTR_UNUSED)
+{
+}
+
+static struct mail_save_context *
+pop3c_save_alloc(struct mailbox_transaction_context *t)
+{
+	struct mail_save_context *ctx;
+
+	ctx = i_new(struct mail_save_context, 1);
+	ctx->transaction = t;
+	return ctx;
+}
+
+static int
+pop3c_save_begin(struct mail_save_context *ctx,
+		 struct istream *input ATTR_UNUSED)
+{
+	mail_storage_set_error(ctx->transaction->box->storage,
+		MAIL_ERROR_NOTPOSSIBLE, "POP3 doesn't support saving mails");
+	return -1;
+}
+
+static int pop3c_save_continue(struct mail_save_context *ctx ATTR_UNUSED)
+{
+	return -1;
+}
+
+static int pop3c_save_finish(struct mail_save_context *ctx)
+{
+	index_save_context_free(ctx);
+	return -1;
+}
+
+static void
+pop3c_save_cancel(struct mail_save_context *ctx)
+{
+	index_save_context_free(ctx);
+}
+
+static bool pop3c_storage_is_inconsistent(struct mailbox *box)
+{
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(box);
+
+	return index_storage_is_inconsistent(box) ||
+		!pop3c_client_is_connected(mbox->client);
+}
+
+struct mail_storage pop3c_storage = {
+	.name = POP3C_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT |
+		MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS,
+	.event_category = &event_category_pop3c,
+
+	.v = {
+		pop3c_get_setting_parser_info,
+		pop3c_storage_alloc,
+		pop3c_storage_create,
+		index_storage_destroy,
+		NULL,
+		pop3c_storage_get_list_settings,
+		NULL,
+		pop3c_mailbox_alloc,
+		NULL,
+		NULL,
+	}
+};
+
+struct mailbox pop3c_mailbox = {
+	.v = {
+		index_storage_is_readonly,
+		index_storage_mailbox_enable,
+		pop3c_mailbox_exists,
+		pop3c_mailbox_open,
+		pop3c_mailbox_close,
+		index_storage_mailbox_free,
+		pop3c_mailbox_create,
+		pop3c_mailbox_update,
+		index_storage_mailbox_delete,
+		index_storage_mailbox_rename,
+		pop3c_mailbox_get_status,
+		pop3c_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		index_storage_list_index_has_changed,
+		index_storage_list_index_update_sync,
+		pop3c_storage_sync_init,
+		index_mailbox_sync_next,
+		index_mailbox_sync_deinit,
+		NULL,
+		pop3c_notify_changes,
+		index_transaction_begin,
+		index_transaction_commit,
+		index_transaction_rollback,
+		NULL,
+		pop3c_mail_alloc,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		pop3c_save_alloc,
+		pop3c_save_begin,
+		pop3c_save_continue,
+		pop3c_save_finish,
+		pop3c_save_cancel,
+		mail_storage_copy,
+		NULL,
+		NULL,
+		NULL,
+		pop3c_storage_is_inconsistent
+	}
+};
diff --git a/src/lib-storage/index/pop3c/pop3c-storage.h b/src/lib-storage/index/pop3c/pop3c-storage.h
new file mode 100644
index 0000000..f271a59
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-storage.h
@@ -0,0 +1,51 @@
+#ifndef POP3C_STORAGE_H
+#define POP3C_STORAGE_H
+
+#include "index-storage.h"
+
+#define POP3C_STORAGE_NAME "pop3c"
+
+struct pop3c_storage {
+	struct mail_storage storage;
+	const struct pop3c_settings *set;
+};
+
+struct pop3c_mailbox {
+	struct mailbox box;
+	struct pop3c_storage *storage;
+
+	struct pop3c_client *client;
+
+	pool_t uidl_pool;
+	unsigned int msg_count;
+	/* LIST sizes */
+	uoff_t *msg_sizes;
+	/* UIDL strings */
+	const char *const *msg_uidls;
+	/* index UIDs for each message in this session.
+	   the UID may not exist for the entire session */
+	uint32_t *msg_uids;
+
+	bool logged_in:1;
+};
+
+struct pop3c_mail {
+	struct index_mail imail;
+	struct istream *prefetch_stream;
+
+	bool prefetching:1;
+	bool prefetching_body:1;
+};
+
+#define POP3C_STORAGE(s)	container_of(s, struct pop3c_storage, storage)
+#define POP3C_MAILBOX(s)	container_of(s, struct pop3c_mailbox, box)
+#define POP3C_MAIL(s)		container_of(s, struct pop3c_mail, imail.mail.mail)
+
+struct mail *
+pop3c_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers);
+
+extern struct mail_vfuncs pop3c_mail_vfuncs;
+
+#endif
diff --git a/src/lib-storage/index/pop3c/pop3c-sync.c b/src/lib-storage/index/pop3c/pop3c-sync.c
new file mode 100644
index 0000000..2d2dbe3
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-sync.c
@@ -0,0 +1,361 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "bsearch-insert-pos.h"
+#include "str.h"
+#include "sort.h"
+#include "strnum.h"
+#include "index-mail.h"
+#include "pop3c-client.h"
+#include "pop3c-storage.h"
+#include "pop3c-sync.h"
+#include "mailbox-recent-flags.h"
+
+struct pop3c_sync_msg {
+	uint32_t seq;
+	const char *uidl;
+};
+ARRAY_DEFINE_TYPE(pop3c_sync_msg, struct pop3c_sync_msg);
+
+int pop3c_sync_get_uidls(struct pop3c_mailbox *mbox)
+{
+	ARRAY_TYPE(const_string) uidls;
+	struct istream *input;
+	const char *error, *cline;
+	char *line, *p;
+	unsigned int seq, line_seq;
+
+	if (mbox->msg_uidls != NULL)
+		return 0;
+	if ((pop3c_client_get_capabilities(mbox->client) &
+	     POP3C_CAPABILITY_UIDL) == 0) {
+		mail_storage_set_error(mbox->box.storage,
+				       MAIL_ERROR_NOTPOSSIBLE,
+				       "UIDLs not supported by server");
+		return -1;
+	}
+
+	if (pop3c_client_cmd_stream(mbox->client, "UIDL\r\n",
+				    &input, &error) < 0) {
+		mailbox_set_critical(&mbox->box, "UIDL failed: %s", error);
+		return -1;
+	}
+
+	mbox->uidl_pool = pool_alloconly_create("POP3 UIDLs", 1024*32);
+	p_array_init(&uidls, mbox->uidl_pool, 64); seq = 0;
+	while ((line = i_stream_read_next_line(input)) != NULL) {
+		seq++;
+		p = strchr(line, ' ');
+		if (p == NULL) {
+			mailbox_set_critical(&mbox->box,
+				"Invalid UIDL line: %s", line);
+			break;
+		}
+		*p++ = '\0';
+		if (str_to_uint(line, &line_seq) < 0 || line_seq != seq) {
+			mailbox_set_critical(&mbox->box,
+				"Unexpected UIDL seq: %s != %u", line, seq);
+			break;
+		}
+
+		cline = p_strdup(mbox->uidl_pool, p);
+		array_push_back(&uidls, &cline);
+	}
+	i_stream_destroy(&input);
+	if (line != NULL) {
+		pool_unref(&mbox->uidl_pool);
+		return -1;
+	}
+	if (seq == 0) {
+		/* make msg_uidls non-NULL */
+		array_append_zero(&uidls);
+	}
+	mbox->msg_uidls = array_front(&uidls);
+	mbox->msg_count = seq;
+	return 0;
+}
+
+int pop3c_sync_get_sizes(struct pop3c_mailbox *mbox)
+{
+	struct istream *input;
+	const char *error;
+	char *line, *p;
+	unsigned int seq, line_seq;
+
+	i_assert(mbox->msg_sizes == NULL);
+
+	if (mbox->msg_uidls == NULL) {
+		if (pop3c_sync_get_uidls(mbox) < 0)
+			return -1;
+	}
+	if (mbox->msg_count == 0) {
+		mbox->msg_sizes = i_new(uoff_t, 1);
+		return 0;
+	}
+
+	if (pop3c_client_cmd_stream(mbox->client, "LIST\r\n",
+				    &input, &error) < 0) {
+		mailbox_set_critical(&mbox->box, "LIST failed: %s", error);
+		return -1;
+	}
+
+	mbox->msg_sizes = i_new(uoff_t, mbox->msg_count); seq = 0;
+	while ((line = i_stream_read_next_line(input)) != NULL) {
+		if (++seq > mbox->msg_count) {
+			mailbox_set_critical(&mbox->box,
+				"Too much data in LIST: %s", line);
+			break;
+		}
+		p = strchr(line, ' ');
+		if (p == NULL) {
+			mailbox_set_critical(&mbox->box,
+				"Invalid LIST line: %s", line);
+			break;
+		}
+		*p++ = '\0';
+		if (str_to_uint(line, &line_seq) < 0 || line_seq != seq) {
+			mailbox_set_critical(&mbox->box,
+				"Unexpected LIST seq: %s != %u", line, seq);
+			break;
+		}
+		if (str_to_uoff(p, &mbox->msg_sizes[seq-1]) < 0) {
+			mailbox_set_critical(&mbox->box,
+				"Invalid LIST size: %s", p);
+			break;
+		}
+	}
+	i_stream_destroy(&input);
+	if (line != NULL) {
+		i_free_and_null(mbox->msg_sizes);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+pop3c_get_local_msgs(pool_t pool, ARRAY_TYPE(pop3c_sync_msg) *local_msgs,
+		     uint32_t messages_count,
+		     struct mail_cache_view *cache_view,
+		     unsigned int cache_idx)
+{
+	string_t *str = t_str_new(128);
+	struct pop3c_sync_msg msg;
+	uint32_t seq;
+
+	i_zero(&msg);
+	for (seq = 1; seq <= messages_count; seq++) {
+		str_truncate(str, 0);
+		if (mail_cache_lookup_field(cache_view, str, seq,
+					    cache_idx) > 0)
+			msg.uidl = p_strdup(pool, str_c(str));
+		msg.seq = seq;
+		array_idx_set(local_msgs, seq-1, &msg);
+	}
+}
+
+static void
+pop3c_get_remote_msgs(ARRAY_TYPE(pop3c_sync_msg) *remote_msgs,
+		      struct pop3c_mailbox *mbox)
+{
+	struct pop3c_sync_msg *msg;
+	uint32_t seq;
+
+	for (seq = 1; seq <= mbox->msg_count; seq++) {
+		msg = array_append_space(remote_msgs);
+		msg->seq = seq;
+		msg->uidl = mbox->msg_uidls[seq-1];
+	}
+}
+
+static int pop3c_sync_msg_uidl_cmp(const struct pop3c_sync_msg *msg1,
+				   const struct pop3c_sync_msg *msg2)
+{
+	return null_strcmp(msg1->uidl, msg2->uidl);
+}
+
+static void
+pop3c_sync_messages(struct pop3c_mailbox *mbox,
+		    struct mail_index_view *sync_view,
+		    struct mail_index_transaction *sync_trans,
+		    struct mail_cache_view *cache_view)
+{
+	struct index_mailbox_context *ibox =
+		INDEX_STORAGE_CONTEXT(&mbox->box);
+	const struct mail_index_header *hdr;
+	struct mail_cache_transaction_ctx *cache_trans;
+	ARRAY_TYPE(pop3c_sync_msg) local_msgs, remote_msgs;
+	const struct pop3c_sync_msg *lmsg, *rmsg;
+	uint32_t seq1, seq2, next_uid;
+	unsigned int lidx, ridx, lcount, rcount;
+	unsigned int cache_idx = ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
+	pool_t pool;
+
+	i_assert(mbox->msg_uids == NULL);
+
+	/* set our uidvalidity */
+	hdr = mail_index_get_header(sync_view);
+	if (hdr->uid_validity == 0) {
+		uint32_t uid_validity = ioloop_time;
+		mail_index_update_header(sync_trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+
+	pool = pool_alloconly_create(MEMPOOL_GROWING"pop3c sync", 10240);
+	p_array_init(&local_msgs, pool, hdr->messages_count);
+	pop3c_get_local_msgs(pool, &local_msgs, hdr->messages_count,
+			     cache_view, cache_idx);
+	p_array_init(&remote_msgs, pool, mbox->msg_count);
+	pop3c_get_remote_msgs(&remote_msgs, mbox);
+
+	/* sort the messages by UIDLs, because some servers reorder messages */
+	array_sort(&local_msgs, pop3c_sync_msg_uidl_cmp);
+	array_sort(&remote_msgs, pop3c_sync_msg_uidl_cmp);
+
+	/* skip over existing messages with matching UIDLs and expunge the ones
+	   that no longer exist in remote. */
+	mbox->msg_uids = mbox->msg_count == 0 ?
+		i_new(uint32_t, 1) : /* avoid malloc(0) assert */
+		i_new(uint32_t, mbox->msg_count);
+	cache_trans = mail_cache_get_transaction(cache_view, sync_trans);
+
+	lmsg = array_get(&local_msgs, &lcount);
+	rmsg = array_get(&remote_msgs, &rcount);
+	next_uid = hdr->next_uid;
+	lidx = ridx = 0;
+	while (lidx < lcount || ridx < rcount) {
+		uint32_t lseq = lidx < lcount ? lmsg[lidx].seq : 0;
+		uint32_t rseq = ridx < rcount ? rmsg[ridx].seq : 0;
+		int ret;
+
+		if (lidx >= lcount)
+			ret = 1;
+		else if (ridx >= rcount || lmsg[lidx].uidl == NULL)
+			ret = -1;
+		else
+			ret = strcmp(lmsg[lidx].uidl, rmsg[ridx].uidl);
+		if (ret < 0) {
+			/* message expunged in remote, or we didn't have a
+			   local message's UIDL in cache. */
+			mail_index_expunge(sync_trans, lseq);
+			lidx++;
+		} else if (ret > 0) {
+			/* new message in remote */
+			i_assert(mbox->msg_uids[rseq-1] == 0);
+			mbox->msg_uids[rseq-1] = next_uid;
+			mail_index_append(sync_trans, next_uid++, &lseq);
+			mail_cache_add(cache_trans, lseq, cache_idx,
+				       rmsg[ridx].uidl,
+				       strlen(rmsg[ridx].uidl));
+			ridx++;
+		} else {
+			/* UIDL matched */
+			i_assert(mbox->msg_uids[rseq-1] == 0);
+			mail_index_lookup_uid(sync_view, lseq,
+					      &mbox->msg_uids[rseq-1]);
+			lidx++;
+			ridx++;
+		}
+	}
+
+	/* mark the newly seen messages as recent */
+	if (mail_index_lookup_seq_range(sync_view, hdr->first_recent_uid,
+					hdr->next_uid, &seq1, &seq2))
+		mailbox_recent_flags_set_seqs(&mbox->box, sync_view, seq1, seq2);
+	pool_unref(&pool);
+}
+
+int pop3c_sync(struct pop3c_mailbox *mbox)
+{
+        struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view, *trans_view;
+	struct mail_index_transaction *sync_trans;
+	struct mail_index_sync_rec sync_rec;
+	struct mail_cache_view *cache_view = NULL;
+	enum mail_index_sync_flags sync_flags;
+	unsigned int idx;
+	string_t *str;
+	const char *reply;
+	int ret;
+	bool deletions = FALSE;
+
+	if (pop3c_sync_get_uidls(mbox) < 0)
+		return -1;
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box) |
+		MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
+
+	ret = mail_index_sync_begin(mbox->box.index, &index_sync_ctx,
+				    &sync_view, &sync_trans, sync_flags);
+	if (ret <= 0) {
+		if (ret < 0)
+			mailbox_set_index_error(&mbox->box);
+		return ret;
+	}
+
+	if (mbox->msg_uids == NULL) {
+		trans_view = mail_index_transaction_open_updated_view(sync_trans);
+		cache_view = mail_cache_view_open(mbox->box.cache, trans_view);
+		pop3c_sync_messages(mbox, sync_view, sync_trans, cache_view);
+	}
+
+	/* mark expunges messages as deleted in this pop3 session,
+	   if those exist */
+	str = t_str_new(32);
+	while (mail_index_sync_next(index_sync_ctx, &sync_rec)) {
+		if (sync_rec.type != MAIL_INDEX_SYNC_TYPE_EXPUNGE)
+			continue;
+
+		if (!bsearch_insert_pos(&sync_rec.uid1, mbox->msg_uids,
+					mbox->msg_count, sizeof(uint32_t),
+					uint32_cmp, &idx)) {
+			/* no such messages in this session */
+			continue;
+		}
+		for (; idx < mbox->msg_count; idx++) {
+			i_assert(mbox->msg_uids[idx] >= sync_rec.uid1);
+			if (mbox->msg_uids[idx] > sync_rec.uid2)
+				break;
+
+			str_truncate(str, 0);
+			str_printfa(str, "DELE %u\r\n", idx+1);
+			pop3c_client_cmd_line_async_nocb(mbox->client, str_c(str));
+			deletions = TRUE;
+		}
+	}
+
+	if (mail_index_sync_commit(&index_sync_ctx) < 0) {
+		mailbox_set_index_error(&mbox->box);
+		return -1;
+	}
+	if (cache_view != NULL) {
+		mail_cache_view_close(&cache_view);
+		mail_index_view_close(&trans_view);
+	}
+	if (deletions) {
+		if (pop3c_client_cmd_line(mbox->client, "QUIT\r\n",
+					  &reply) < 0) {
+			mail_storage_set_error(mbox->box.storage,
+					       MAIL_ERROR_TEMP, reply);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+struct mailbox_sync_context *
+pop3c_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct pop3c_mailbox *mbox = POP3C_MAILBOX(box);
+	int ret = 0;
+
+	if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 &&
+	    mbox->msg_uidls == NULL) {
+		/* FIXME: reconnect */
+	}
+
+	ret = pop3c_sync(mbox);
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
diff --git a/src/lib-storage/index/pop3c/pop3c-sync.h b/src/lib-storage/index/pop3c/pop3c-sync.h
new file mode 100644
index 0000000..bf3c802
--- /dev/null
+++ b/src/lib-storage/index/pop3c/pop3c-sync.h
@@ -0,0 +1,14 @@
+#ifndef POP3C_SYNC_H
+#define POP3C_SYNC_H
+
+struct mailbox;
+struct pop3c_mailbox;
+
+struct mailbox_sync_context *
+pop3c_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+int pop3c_sync(struct pop3c_mailbox *mbox);
+
+int pop3c_sync_get_sizes(struct pop3c_mailbox *mbox);
+int pop3c_sync_get_uidls(struct pop3c_mailbox *mbox);
+
+#endif
diff --git a/src/lib-storage/index/raw/Makefile.am b/src/lib-storage/index/raw/Makefile.am
new file mode 100644
index 0000000..58a9df3
--- /dev/null
+++ b/src/lib-storage/index/raw/Makefile.am
@@ -0,0 +1,21 @@
+noinst_LTLIBRARIES = libstorage_raw.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_raw_la_SOURCES = \
+	raw-mail.c \
+	raw-sync.c \
+	raw-storage.c
+
+headers = \
+	raw-storage.h \
+	raw-sync.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/raw/Makefile.in b/src/lib-storage/index/raw/Makefile.in
new file mode 100644
index 0000000..c42d116
--- /dev/null
+++ b/src/lib-storage/index/raw/Makefile.in
@@ -0,0 +1,822 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/raw
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_raw_la_LIBADD =
+am_libstorage_raw_la_OBJECTS = raw-mail.lo raw-sync.lo raw-storage.lo
+libstorage_raw_la_OBJECTS = $(am_libstorage_raw_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/raw-mail.Plo \
+	./$(DEPDIR)/raw-storage.Plo ./$(DEPDIR)/raw-sync.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_raw_la_SOURCES)
+DIST_SOURCES = $(libstorage_raw_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_raw.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_raw_la_SOURCES = \
+	raw-mail.c \
+	raw-sync.c \
+	raw-storage.c
+
+headers = \
+	raw-storage.h \
+	raw-sync.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/raw/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/raw/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_raw.la: $(libstorage_raw_la_OBJECTS) $(libstorage_raw_la_DEPENDENCIES) $(EXTRA_libstorage_raw_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_raw_la_OBJECTS) $(libstorage_raw_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/raw-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/raw-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/raw-sync.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/raw-mail.Plo
+	-rm -f ./$(DEPDIR)/raw-storage.Plo
+	-rm -f ./$(DEPDIR)/raw-sync.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/raw-mail.Plo
+	-rm -f ./$(DEPDIR)/raw-storage.Plo
+	-rm -f ./$(DEPDIR)/raw-sync.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/raw/raw-mail.c b/src/lib-storage/index/raw/raw-mail.c
new file mode 100644
index 0000000..72d3688
--- /dev/null
+++ b/src/lib-storage/index/raw/raw-mail.c
@@ -0,0 +1,152 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "index-mail.h"
+#include "raw-storage.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+static int raw_mail_stat(struct mail *mail)
+{
+	struct raw_mailbox *mbox = RAW_MAILBOX(mail->box);
+	const struct stat *st;
+
+	if (!mail_metadata_access_start(mail))
+		return -1;
+
+	mail->transaction->stats.fstat_lookup_count++;
+	if (i_stream_stat(mail->box->input, TRUE, &st) < 0) {
+		mail_set_critical(mail, "stat(%s) failed: %m",
+				  i_stream_get_name(mail->box->input));
+		return -1;
+	}
+
+	if (mbox->mtime != (time_t)-1)
+		mbox->mtime = st->st_mtime;
+	if (mbox->ctime != (time_t)-1)
+		mbox->ctime = st->st_ctime;
+	mbox->size = (size_t)st->st_size;
+	return 0;
+}
+
+static int raw_mail_get_received_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box);
+
+	if (mbox->mtime == (time_t)-1) {
+		if (raw_mail_stat(_mail) < 0)
+			return -1;
+	}
+
+	*date_r = mail->data.received_date = mbox->mtime;
+	return 0;
+}
+
+static int raw_mail_get_save_date(struct mail *_mail, time_t *date_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box);
+
+	if (mbox->ctime == (time_t)-1) {
+		if (raw_mail_stat(_mail) < 0)
+			return -1;
+	}
+
+	*date_r = mail->data.save_date = mbox->ctime;
+	return 1;
+}
+
+static int raw_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+	struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box);
+
+	if (mbox->size == UOFF_T_MAX) {
+		if (raw_mail_stat(_mail) < 0)
+			return -1;
+	}
+
+	*size_r = mail->data.physical_size = mbox->size;
+	return 0;
+}
+
+static int
+raw_mail_get_stream(struct mail *_mail, bool get_body ATTR_UNUSED,
+		    struct message_size *hdr_size,
+		    struct message_size *body_size, struct istream **stream_r)
+{
+	struct index_mail *mail = INDEX_MAIL(_mail);
+
+	if (mail->data.stream == NULL) {
+		if (!mail_stream_access_start(_mail))
+			return -1;
+		/* we can't just reference mbox->input, because
+		   index_mail_close() expects to be able to free the stream */
+		mail->data.stream =
+			i_stream_create_limit(_mail->box->input, UOFF_T_MAX);
+	}
+
+	return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
+}
+
+static int
+raw_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+		     const char **value_r)
+{
+	struct raw_mailbox *mbox = RAW_MAILBOX(_mail->box);
+
+	switch (field) {
+	case MAIL_FETCH_FROM_ENVELOPE:
+		*value_r = mbox->envelope_sender != NULL ?
+			mbox->envelope_sender : "";
+		return 0;
+	case MAIL_FETCH_STORAGE_ID:
+		*value_r = mbox->have_filename ?
+			mailbox_get_path(_mail->box) : "";
+		return 0;
+	default:
+		return index_mail_get_special(_mail, field, value_r);
+	}
+}
+
+struct mail_vfuncs raw_mail_vfuncs = {
+	index_mail_close,
+	index_mail_free,
+	index_mail_set_seq,
+	index_mail_set_uid,
+	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
+	index_mail_precache,
+	index_mail_add_temp_wanted_fields,
+
+	index_mail_get_flags,
+	index_mail_get_keywords,
+	index_mail_get_keyword_indexes,
+	index_mail_get_modseq,
+	index_mail_get_pvt_modseq,
+	index_mail_get_parts,
+	index_mail_get_date,
+	raw_mail_get_received_date,
+	raw_mail_get_save_date,
+	index_mail_get_virtual_size,
+	raw_mail_get_physical_size,
+	index_mail_get_first_header,
+	index_mail_get_headers,
+	index_mail_get_header_stream,
+	raw_mail_get_stream,
+	index_mail_get_binary_stream,
+	raw_mail_get_special,
+	index_mail_get_backend_mail,
+	index_mail_update_flags,
+	index_mail_update_keywords,
+	index_mail_update_modseq,
+	index_mail_update_pvt_modseq,
+	NULL,
+	index_mail_expunge,
+	index_mail_set_cache_corrupted,
+	index_mail_opened,
+};
diff --git a/src/lib-storage/index/raw/raw-storage.c b/src/lib-storage/index/raw/raw-storage.c
new file mode 100644
index 0000000..e94ae10
--- /dev/null
+++ b/src/lib-storage/index/raw/raw-storage.c
@@ -0,0 +1,269 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "index-mail.h"
+#include "mail-copy.h"
+#include "mailbox-list-private.h"
+#include "raw-sync.h"
+#include "raw-storage.h"
+
+extern struct mail_storage raw_storage;
+extern struct mailbox raw_mailbox;
+
+struct mail_user *
+raw_storage_create_from_set(const struct setting_parser_info *set_info,
+			    const struct mail_user_settings *set)
+{
+	struct mail_user *user;
+	struct mail_namespace *ns;
+	struct mail_namespace_settings *ns_set;
+	struct mail_storage_settings *mail_set;
+	const char *error;
+
+	user = mail_user_alloc(NULL, "raw mail user", set_info, set);
+	user->autocreated = TRUE;
+	mail_user_set_home(user, "/");
+	if (mail_user_init(user, &error) < 0)
+		i_fatal("Raw user initialization failed: %s", error);
+
+	ns_set = p_new(user->pool, struct mail_namespace_settings, 1);
+	ns_set->name = "raw-storage";
+	ns_set->location = ":LAYOUT=none";
+	ns_set->separator = "/";
+
+	ns = mail_namespaces_init_empty(user);
+	/* raw storage doesn't have INBOX. We especially don't want LIST to
+	   return INBOX. */
+	ns->flags &= ENUM_NEGATE(NAMESPACE_FLAG_INBOX_USER);
+	ns->flags |= NAMESPACE_FLAG_NOQUOTA | NAMESPACE_FLAG_NOACL;
+	ns->set = ns_set;
+	/* absolute paths are ok with raw storage */
+	mail_set = p_new(user->pool, struct mail_storage_settings, 1);
+	*mail_set = *ns->mail_set;
+	mail_set->mail_full_filesystem_access = TRUE;
+	ns->mail_set = mail_set;
+
+	if (mail_storage_create(ns, "raw", 0, &error) < 0)
+		i_fatal("Couldn't create internal raw storage: %s", error);
+	if (mail_namespaces_init_finish(ns, &error) < 0)
+		i_fatal("Couldn't create internal raw namespace: %s", error);
+	return user;
+}
+
+static int ATTR_NULL(2, 3)
+raw_mailbox_alloc_common(struct mail_user *user, struct istream *input,
+			 const char *path, time_t received_time,
+			 const char *envelope_sender, struct mailbox **box_r)
+{
+	struct mail_namespace *ns = user->namespaces;
+	struct mailbox *box;
+	struct raw_mailbox *raw_box;
+	const char *name;
+
+	name = path != NULL ? path : i_stream_get_name(input);
+	box = *box_r = mailbox_alloc(ns->list, name,
+				     MAILBOX_FLAG_NO_INDEX_FILES);
+	if (input != NULL) {
+		if (mailbox_open_stream(box, input) < 0)
+			return -1;
+	} else {
+		if (mailbox_open(box) < 0)
+			return -1;
+	}
+	if (mailbox_sync(box, 0) < 0)
+		return -1;
+
+	i_assert(strcmp(box->storage->name, RAW_STORAGE_NAME) == 0);
+	raw_box = RAW_MAILBOX(box);
+	raw_box->envelope_sender = p_strdup(box->pool, envelope_sender);
+	raw_box->mtime = received_time;
+	return 0;
+}
+
+int raw_mailbox_alloc_stream(struct mail_user *user, struct istream *input,
+			     time_t received_time, const char *envelope_sender,
+			     struct mailbox **box_r)
+{
+	return raw_mailbox_alloc_common(user, input, NULL, received_time,
+					envelope_sender, box_r);
+}
+
+int raw_mailbox_alloc_path(struct mail_user *user, const char *path,
+			   time_t received_time, const char *envelope_sender,
+			   struct mailbox **box_r)
+{
+	return raw_mailbox_alloc_common(user, NULL, path, received_time,
+					envelope_sender, box_r);
+}
+
+static struct mail_storage *raw_storage_alloc(void)
+{
+	struct raw_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("raw storage", 512+256);
+	storage = p_new(pool, struct raw_storage, 1);
+	storage->storage = raw_storage;
+	storage->storage.pool = pool;
+	return &storage->storage;
+}
+
+static void
+raw_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED,
+			      struct mailbox_list_settings *set)
+{
+	if (set->layout == NULL)
+		set->layout = MAILBOX_LIST_NAME_FS;
+	if (set->subscription_fname == NULL)
+		set->subscription_fname = RAW_SUBSCRIPTION_FILE_NAME;
+}
+
+static struct mailbox *
+raw_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
+		  const char *vname, enum mailbox_flags flags)
+{
+	struct raw_mailbox *mbox;
+	pool_t pool;
+
+	flags |= MAILBOX_FLAG_READONLY | MAILBOX_FLAG_NO_INDEX_FILES;
+
+	pool = pool_alloconly_create("raw mailbox", 1024*3);
+	mbox = p_new(pool, struct raw_mailbox, 1);
+	mbox->box = raw_mailbox;
+	mbox->box.pool = pool;
+	mbox->box.storage = storage;
+	mbox->box.list = list;
+	mbox->box.mail_vfuncs = &raw_mail_vfuncs;
+
+	index_storage_mailbox_alloc(&mbox->box, vname, flags, "dovecot.index");
+
+	mbox->mtime = mbox->ctime = (time_t)-1;
+	mbox->storage = RAW_STORAGE(storage);
+	mbox->size = UOFF_T_MAX;
+	return &mbox->box;
+}
+
+static int raw_mailbox_open(struct mailbox *box)
+{
+	struct raw_mailbox *mbox = RAW_MAILBOX(box);
+	const char *path;
+	int fd;
+
+	if (box->input != NULL) {
+		mbox->mtime = mbox->ctime = ioloop_time;
+		return index_storage_mailbox_open(box, FALSE);
+	}
+
+	path = box->_path = box->name;
+	mbox->have_filename = TRUE;
+	fd = open(path, O_RDONLY);
+	if (fd == -1) {
+		if (ENOTFOUND(errno)) {
+			mail_storage_set_error(box->storage,
+				MAIL_ERROR_NOTFOUND,
+				T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname));
+		} else if (!mail_storage_set_error_from_errno(box->storage)) {
+			mailbox_set_critical(box, "open(%s) failed: %m", path);
+		}
+		return -1;
+	}
+	box->input = i_stream_create_fd_autoclose(&fd, MAIL_READ_FULL_BLOCK_SIZE);
+	i_stream_set_name(box->input, path);
+	i_stream_set_init_buffer_size(box->input, MAIL_READ_FULL_BLOCK_SIZE);
+	return index_storage_mailbox_open(box, FALSE);
+}
+
+static int
+raw_mailbox_create(struct mailbox *box,
+		   const struct mailbox_update *update ATTR_UNUSED,
+		   bool directory ATTR_UNUSED)
+{
+	mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE,
+			       "Raw mailbox creation isn't supported");
+	return -1;
+}
+
+static int
+raw_mailbox_update(struct mailbox *box,
+		   const struct mailbox_update *update ATTR_UNUSED)
+{
+	mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE,
+			       "Raw mailbox update isn't supported");
+	return -1;
+}
+
+static void raw_notify_changes(struct mailbox *box ATTR_UNUSED)
+{
+}
+
+struct mail_storage raw_storage = {
+	.name = RAW_STORAGE_NAME,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_MAILBOX_IS_FILE |
+		MAIL_STORAGE_CLASS_FLAG_OPEN_STREAMS |
+		MAIL_STORAGE_CLASS_FLAG_BINARY_DATA,
+
+	.v = {
+		NULL,
+		raw_storage_alloc,
+		NULL,
+		index_storage_destroy,
+		NULL,
+		raw_storage_get_list_settings,
+		NULL,
+		raw_mailbox_alloc,
+		NULL,
+		NULL,
+	}
+};
+
+struct mailbox raw_mailbox = {
+	.v = {
+		index_storage_is_readonly,
+		index_storage_mailbox_enable,
+		index_storage_mailbox_exists,
+		raw_mailbox_open,
+		index_storage_mailbox_close,
+		index_storage_mailbox_free,
+		raw_mailbox_create,
+		raw_mailbox_update,
+		index_storage_mailbox_delete,
+		index_storage_mailbox_rename,
+		index_storage_get_status,
+		index_mailbox_get_metadata,
+		index_storage_set_subscribed,
+		index_storage_attribute_set,
+		index_storage_attribute_get,
+		index_storage_attribute_iter_init,
+		index_storage_attribute_iter_next,
+		index_storage_attribute_iter_deinit,
+		index_storage_list_index_has_changed,
+		index_storage_list_index_update_sync,
+		raw_storage_sync_init,
+		index_mailbox_sync_next,
+		index_mailbox_sync_deinit,
+		NULL,
+		raw_notify_changes,
+		index_transaction_begin,
+		index_transaction_commit,
+		index_transaction_rollback,
+		NULL,
+		index_mail_alloc,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
+		index_storage_search_next_match_mail,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		mail_storage_copy,
+		NULL,
+		NULL,
+		NULL,
+		index_storage_is_inconsistent
+	}
+};
diff --git a/src/lib-storage/index/raw/raw-storage.h b/src/lib-storage/index/raw/raw-storage.h
new file mode 100644
index 0000000..124ca7e
--- /dev/null
+++ b/src/lib-storage/index/raw/raw-storage.h
@@ -0,0 +1,41 @@
+#ifndef RAW_STORAGE_H
+#define RAW_STORAGE_H
+
+#include "index-storage.h"
+
+#define RAW_STORAGE_NAME "raw"
+#define RAW_SUBSCRIPTION_FILE_NAME "subscriptions"
+
+struct raw_storage {
+	struct mail_storage storage;
+};
+
+struct raw_mailbox {
+	struct mailbox box;
+	struct raw_storage *storage;
+
+	time_t mtime, ctime;
+	uoff_t size;
+	const char *envelope_sender;
+
+	bool synced:1;
+	bool have_filename:1;
+};
+
+#define RAW_STORAGE(s)		container_of(s, struct raw_storage, storage)
+#define RAW_MAILBOX(s)		container_of(s, struct raw_mailbox, box)
+
+extern struct mail_vfuncs raw_mail_vfuncs;
+
+struct mail_user *
+raw_storage_create_from_set(const struct setting_parser_info *set_info,
+			    const struct mail_user_settings *set);
+
+int raw_mailbox_alloc_stream(struct mail_user *user, struct istream *input,
+			     time_t received_time, const char *envelope_sender,
+			     struct mailbox **box_r);
+int raw_mailbox_alloc_path(struct mail_user *user, const char *path,
+			   time_t received_time, const char *envelope_sender,
+			   struct mailbox **box_r);
+
+#endif
diff --git a/src/lib-storage/index/raw/raw-sync.c b/src/lib-storage/index/raw/raw-sync.c
new file mode 100644
index 0000000..6511f72
--- /dev/null
+++ b/src/lib-storage/index/raw/raw-sync.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "raw-storage.h"
+#include "raw-sync.h"
+#include "mailbox-recent-flags.h"
+
+static int raw_sync(struct raw_mailbox *mbox)
+{
+        struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_sync_rec sync_rec;
+	struct mail_index_transaction *trans;
+	uint32_t seq, uid_validity = ioloop_time;
+	enum mail_index_sync_flags sync_flags;
+	int ret;
+
+	i_assert(!mbox->synced);
+
+	sync_flags = index_storage_get_sync_flags(&mbox->box) |
+		MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY |
+		MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
+
+	if (mail_index_view_get_messages_count(mbox->box.view) > 0) {
+		/* already-synced index was opened via
+		   mail-index-alloc-cache. */
+		return 0;
+	}
+
+	ret = mail_index_sync_begin(mbox->box.index, &index_sync_ctx,
+				    &sync_view, &trans, sync_flags);
+	if (ret <= 0) {
+		if (ret < 0)
+			mailbox_set_index_error(&mbox->box);
+		return ret;
+	}
+
+	/* set our uidvalidity */
+	mail_index_update_header(trans,
+		offsetof(struct mail_index_header, uid_validity),
+		&uid_validity, sizeof(uid_validity), TRUE);
+
+	/* add our one and only message */
+	mail_index_append(trans, 1, &seq);
+	mailbox_recent_flags_set_uid(&mbox->box, 1);
+
+	while (mail_index_sync_next(index_sync_ctx, &sync_rec)) ;
+	if (mail_index_sync_commit(&index_sync_ctx) < 0) {
+		mailbox_set_index_error(&mbox->box);
+		return -1;
+	}
+	mbox->synced = TRUE;
+	return 0;
+}
+
+struct mailbox_sync_context *
+raw_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct raw_mailbox *mbox = RAW_MAILBOX(box);
+	int ret = 0;
+
+	if (!mbox->synced)
+		ret = raw_sync(mbox);
+
+	return index_mailbox_sync_init(box, flags, ret < 0);
+}
diff --git a/src/lib-storage/index/raw/raw-sync.h b/src/lib-storage/index/raw/raw-sync.h
new file mode 100644
index 0000000..2c29c1f
--- /dev/null
+++ b/src/lib-storage/index/raw/raw-sync.h
@@ -0,0 +1,9 @@
+#ifndef RAW_SYNC_H
+#define RAW_SYNC_H
+
+struct mailbox;
+
+struct mailbox_sync_context *
+raw_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+
+#endif
diff --git a/src/lib-storage/index/shared/Makefile.am b/src/lib-storage/index/shared/Makefile.am
new file mode 100644
index 0000000..07980d6
--- /dev/null
+++ b/src/lib-storage/index/shared/Makefile.am
@@ -0,0 +1,19 @@
+noinst_LTLIBRARIES = libstorage_shared.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_shared_la_SOURCES = \
+	shared-list.c \
+	shared-storage.c
+
+headers = \
+	shared-storage.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff --git a/src/lib-storage/index/shared/Makefile.in b/src/lib-storage/index/shared/Makefile.in
new file mode 100644
index 0000000..3c7d4e1
--- /dev/null
+++ b/src/lib-storage/index/shared/Makefile.in
@@ -0,0 +1,817 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/lib-storage/index/shared
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libstorage_shared_la_LIBADD =
+am_libstorage_shared_la_OBJECTS = shared-list.lo shared-storage.lo
+libstorage_shared_la_OBJECTS = $(am_libstorage_shared_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/shared-list.Plo \
+	./$(DEPDIR)/shared-storage.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libstorage_shared_la_SOURCES)
+DIST_SOURCES = $(libstorage_shared_la_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libstorage_shared.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
+
+libstorage_shared_la_SOURCES = \
+	shared-list.c \
+	shared-storage.c
+
+headers = \
+	shared-storage.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-storage/index/shared/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/lib-storage/index/shared/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libstorage_shared.la: $(libstorage_shared_la_OBJECTS) $(libstorage_shared_la_DEPENDENCIES) $(EXTRA_libstorage_shared_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libstorage_shared_la_OBJECTS) $(libstorage_shared_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/shared-list.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/shared-storage.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/shared-list.Plo
+	-rm -f ./$(DEPDIR)/shared-storage.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/shared-list.Plo
+	-rm -f ./$(DEPDIR)/shared-storage.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	cscopelist-am ctags ctags-am distclean distclean-compile \
+	distclean-generic distclean-libtool distclean-tags distdir dvi \
+	dvi-am html html-am info info-am install install-am \
+	install-data install-data-am install-dvi install-dvi-am \
+	install-exec install-exec-am install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-pkginc_libHEADERS install-ps \
+	install-ps-am install-strip installcheck installcheck-am \
+	installdirs maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+	uninstall-am uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-storage/index/shared/shared-list.c b/src/lib-storage/index/shared/shared-list.c
new file mode 100644
index 0000000..c69c4db
--- /dev/null
+++ b/src/lib-storage/index/shared/shared-list.c
@@ -0,0 +1,310 @@
+/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "imap-match.h"
+#include "mailbox-tree.h"
+#include "mailbox-list-private.h"
+#include "index-storage.h"
+#include "shared-storage.h"
+
+extern struct mailbox_list shared_mailbox_list;
+
+static struct mailbox_list *shared_list_alloc(void)
+{
+	struct mailbox_list *list;
+	pool_t pool;
+
+	pool = pool_alloconly_create("shared list", 2048);
+	list = p_new(pool, struct mailbox_list, 1);
+	*list = shared_mailbox_list;
+	list->pool = pool;
+	return list;
+}
+
+static void shared_list_deinit(struct mailbox_list *list)
+{
+	pool_unref(&list->pool);
+}
+
+static void shared_list_copy_error(struct mailbox_list *shared_list,
+				   struct mail_namespace *backend_ns)
+{
+	const char *str;
+	enum mail_error error;
+
+	str = mailbox_list_get_last_error(backend_ns->list, &error);
+	mailbox_list_set_error(shared_list, error, str);
+}
+
+static int
+shared_get_storage(struct mailbox_list **list, const char *vname,
+		   struct mail_storage **storage_r)
+{
+	struct mail_namespace *ns = (*list)->ns;
+	const char *name;
+
+	name = mailbox_list_get_storage_name(*list, vname);
+	if (*name == '\0' && (ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) {
+		/* trying to access the shared/ prefix itself */
+		*storage_r = ns->storage;
+		return 0;
+	}
+
+	if (shared_storage_get_namespace(&ns, &name) < 0)
+		return -1;
+	*list = ns->list;
+	return mailbox_list_get_storage(list, vname, storage_r);
+}
+
+static char shared_list_get_hierarchy_sep(struct mailbox_list *list ATTR_UNUSED)
+{
+	return '/';
+}
+
+static int
+shared_list_get_path(struct mailbox_list *list, const char *name,
+		     enum mailbox_list_path_type type, const char **path_r)
+{
+	struct mail_namespace *ns = list->ns;
+
+	if (mail_namespace_get_default_storage(list->ns) == NULL ||
+	    name == NULL ||
+	    shared_storage_get_namespace(&ns, &name) < 0) {
+		/* we don't have a directory we can use. */
+		*path_r = NULL;
+		return 0;
+	}
+	return mailbox_list_get_path(ns->list, name, type, path_r);
+}
+
+static const char *
+shared_list_get_temp_prefix(struct mailbox_list *list, bool global ATTR_UNUSED)
+{
+	i_panic("shared mailbox list: Can't return a temp prefix for '%s'",
+		list->ns->prefix);
+}
+
+static const char *
+shared_list_join_refpattern(struct mailbox_list *list,
+			    const char *ref, const char *pattern)
+{
+	struct mail_namespace *ns = list->ns;
+	const char *ns_ref, *prefix = list->ns->prefix;
+	size_t prefix_len = strlen(prefix);
+
+	if (*ref != '\0' && str_begins(ref, prefix))
+		ns_ref = ref + prefix_len;
+	else
+		ns_ref = NULL;
+
+	if (ns_ref != NULL && *ns_ref != '\0' &&
+	    shared_storage_get_namespace(&ns, &ns_ref) == 0)
+		return mailbox_list_join_refpattern(ns->list, ref, pattern);
+
+	/* fallback to default behavior */
+	if (*ref != '\0')
+		pattern = t_strconcat(ref, pattern, NULL);
+	return pattern;
+}
+
+static void
+shared_list_create_missing_namespaces(struct mailbox_list *list,
+				      const char *const *patterns)
+{
+	struct mail_namespace *ns;
+	char sep = mail_namespace_get_sep(list->ns);
+	const char *list_pat, *name;
+	unsigned int i;
+
+	for (i = 0; patterns[i] != NULL; i++) {
+		const char *last = NULL, *p;
+
+		/* we'll require that the pattern begins with the list's
+		   namespace prefix. we could also handle other patterns
+		   (e.g. %/user/%), but it's more of a theoretical problem. */
+		if (strncmp(list->ns->prefix, patterns[i],
+			    list->ns->prefix_len) != 0)
+			continue;
+		list_pat = patterns[i] + list->ns->prefix_len;
+
+		for (p = list_pat; *p != '\0'; p++) {
+			if (*p == '%' || *p == '*')
+				break;
+			if (*p == sep)
+				last = p;
+		}
+		if (last != NULL) {
+			ns = list->ns;
+			name = t_strdup_until(list_pat, last);
+			(void)shared_storage_get_namespace(&ns, &name);
+		}
+	}
+}
+
+static struct mailbox_list_iterate_context *
+shared_list_iter_init(struct mailbox_list *list, const char *const *patterns,
+		      enum mailbox_list_iter_flags flags)
+{
+	struct mailbox_list_iterate_context *ctx;
+	pool_t pool;
+	char sep = mail_namespace_get_sep(list->ns);
+
+	pool = pool_alloconly_create("mailbox list shared iter", 1024);
+	ctx = p_new(pool, struct mailbox_list_iterate_context, 1);
+	ctx->pool = pool;
+	ctx->list = list;
+	ctx->flags = flags;
+	ctx->glob = imap_match_init_multiple(pool, patterns, FALSE, sep);
+	array_create(&ctx->module_contexts, pool, sizeof(void *), 5);
+
+	if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0 &&
+	    (list->ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) T_BEGIN {
+		shared_list_create_missing_namespaces(list, patterns);
+	} T_END;
+	return ctx;
+}
+
+static const struct mailbox_info *
+shared_list_iter_next(struct mailbox_list_iterate_context *ctx ATTR_UNUSED)
+{
+	return NULL;
+}
+
+static int shared_list_iter_deinit(struct mailbox_list_iterate_context *ctx)
+{
+	pool_unref(&ctx->pool);
+	return 0;
+}
+
+static int
+shared_list_subscriptions_refresh(struct mailbox_list *src_list,
+				  struct mailbox_list *dest_list)
+{
+	char sep;
+
+	if (dest_list->subscriptions == NULL) {
+		sep = mail_namespace_get_sep(src_list->ns);
+		dest_list->subscriptions = mailbox_tree_init(sep);
+	}
+	return 0;
+}
+
+static int shared_list_set_subscribed(struct mailbox_list *list,
+				      const char *name, bool set)
+{
+	struct mail_namespace *ns = list->ns;
+	int ret;
+
+	if (shared_storage_get_namespace(&ns, &name) < 0)
+		return -1;
+	ret = mailbox_list_set_subscribed(ns->list, name, set);
+	if (ret < 0)
+		shared_list_copy_error(list, ns);
+	return ret;
+}
+
+static int
+shared_list_delete_mailbox(struct mailbox_list *list, const char *name)
+{
+	struct mail_namespace *ns = list->ns;
+	int ret;
+
+	if (shared_storage_get_namespace(&ns, &name) < 0)
+		return -1;
+	ret = ns->list->v.delete_mailbox(ns->list, name);
+	if (ret < 0)
+		shared_list_copy_error(list, ns);
+	return ret;
+}
+
+static int
+shared_list_delete_dir(struct mailbox_list *list, const char *name)
+{
+	struct mail_namespace *ns = list->ns;
+	int ret;
+
+	if (shared_storage_get_namespace(&ns, &name) < 0)
+		return -1;
+	ret = mailbox_list_delete_dir(ns->list, name);
+	if (ret < 0)
+		shared_list_copy_error(list, ns);
+	return ret;
+}
+
+static int
+shared_list_delete_symlink(struct mailbox_list *list, const char *name)
+{
+	struct mail_namespace *ns = list->ns;
+	int ret;
+
+	if (shared_storage_get_namespace(&ns, &name) < 0)
+		return -1;
+	ret = mailbox_list_delete_symlink(ns->list, name);
+	if (ret < 0)
+		shared_list_copy_error(list, ns);
+	return ret;
+}
+
+static int shared_list_rename_get_ns(struct mailbox_list *oldlist,
+				     const char **oldname,
+				     struct mailbox_list *newlist,
+				     const char **newname,
+				     struct mail_namespace **ns_r)
+{
+	struct mail_namespace *old_ns = oldlist->ns, *new_ns = newlist->ns;
+
+	if (shared_storage_get_namespace(&old_ns, oldname) < 0 ||
+	    shared_storage_get_namespace(&new_ns, newname) < 0)
+		return -1;
+	if (old_ns != new_ns) {
+		mailbox_list_set_error(oldlist, MAIL_ERROR_NOTPOSSIBLE,
+			"Can't rename shared mailboxes across storages.");
+		return -1;
+	}
+	*ns_r = old_ns;
+	return 0;
+}
+
+static int
+shared_list_rename_mailbox(struct mailbox_list *oldlist, const char *oldname,
+			   struct mailbox_list *newlist, const char *newname)
+{
+	struct mail_namespace *ns;
+	int ret;
+
+	if (shared_list_rename_get_ns(oldlist, &oldname,
+				      newlist, &newname, &ns) < 0)
+		return -1;
+
+	ret = ns->list->v.rename_mailbox(ns->list, oldname, ns->list, newname);
+	if (ret < 0)
+		shared_list_copy_error(oldlist, ns);
+	return ret;
+}
+
+struct mailbox_list shared_mailbox_list = {
+	.name = "shared",
+	.props = 0,
+	.mailbox_name_max_length = MAILBOX_LIST_NAME_MAX_LENGTH,
+
+	.v = {
+		.alloc = shared_list_alloc,
+		.deinit = shared_list_deinit,
+		.get_storage = shared_get_storage,
+		.get_hierarchy_sep = shared_list_get_hierarchy_sep,
+		.get_vname = mailbox_list_default_get_vname,
+		.get_storage_name = mailbox_list_default_get_storage_name,
+		.get_path = shared_list_get_path,
+		.get_temp_prefix = shared_list_get_temp_prefix,
+		.join_refpattern = shared_list_join_refpattern,
+		.iter_init = shared_list_iter_init,
+		.iter_next = shared_list_iter_next,
+		.iter_deinit = shared_list_iter_deinit,
+		.subscriptions_refresh = shared_list_subscriptions_refresh,
+		.set_subscribed = shared_list_set_subscribed,
+		.delete_mailbox = shared_list_delete_mailbox,
+		.delete_dir = shared_list_delete_dir,
+		.delete_symlink = shared_list_delete_symlink,
+		.rename_mailbox = shared_list_rename_mailbox,
+	}
+};
diff --git a/src/lib-storage/index/shared/shared-storage.c b/src/lib-storage/index/shared/shared-storage.c
new file mode 100644
index 0000000..978949e
--- /dev/null
+++ b/src/lib-storage/index/shared/shared-storage.c
@@ -0,0 +1,379 @@
+/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "ioloop.h"
+#include "var-expand.h"
+#include "index-storage.h"
+#include "mail-storage-service.h"
+#include "mailbox-list-private.h"
+#include "fail-mail-storage.h"
+#include "shared-storage.h"
+
+#include <ctype.h>
+
+extern struct mail_storage shared_storage;
+
+static struct mail_storage *shared_storage_alloc(void)
+{
+	struct shared_storage *storage;
+	pool_t pool;
+
+	pool = pool_alloconly_create("shared storage", 1024);
+	storage = p_new(pool, struct shared_storage, 1);
+	storage->storage = shared_storage;
+	storage->storage.pool = pool;
+	return &storage->storage;
+}
+
+static int
+shared_storage_create(struct mail_storage *_storage, struct mail_namespace *ns,
+		      const char **error_r)
+{
+	struct shared_storage *storage = SHARED_STORAGE(_storage);
+	const char *driver, *p;
+	char *wildcardp, key;
+	bool have_username;
+
+	/* location must begin with the actual mailbox driver */
+	p = strchr(ns->set->location, ':');
+	if (p == NULL) {
+		*error_r = "Shared mailbox location not prefixed with driver";
+		return -1;
+	}
+	driver = t_strdup_until(ns->set->location, p);
+	storage->location = p_strdup(_storage->pool, ns->set->location);
+	storage->unexpanded_location =
+		p_strdup(_storage->pool, ns->unexpanded_set->location);
+	storage->storage_class_name = p_strdup(_storage->pool, driver);
+
+	if (mail_user_get_storage_class(_storage->user, driver) == NULL &&
+	    strcmp(driver, "auto") != 0) {
+		*error_r = t_strconcat("Unknown shared storage driver: ",
+				       driver, NULL);
+		return -1;
+	}
+
+	wildcardp = strchr(ns->prefix, '%');
+	if (wildcardp == NULL) {
+		*error_r = "Shared namespace prefix doesn't contain %";
+		return -1;
+	}
+	storage->ns_prefix_pattern = p_strdup(_storage->pool, wildcardp);
+
+	have_username = FALSE;
+	for (p = storage->ns_prefix_pattern; *p != '\0'; p++) {
+		if (*p != '%')
+			continue;
+
+		key = p[1];
+		if (key == 'u' || key == 'n')
+			have_username = TRUE;
+		else if (key != '%' && key != 'd')
+			break;
+	}
+	if (*p != '\0') {
+		*error_r = "Shared namespace prefix contains unknown variables";
+		return -1;
+	}
+	if (!have_username) {
+		*error_r = "Shared namespace prefix doesn't contain %u or %n";
+		return -1;
+	}
+	if (p[-1] != mail_namespace_get_sep(ns) &&
+	    (ns->flags & (NAMESPACE_FLAG_LIST_PREFIX |
+			  NAMESPACE_FLAG_LIST_CHILDREN)) != 0) {
+		*error_r = "Shared namespace prefix doesn't end with hierarchy separator";
+		return -1;
+	}
+
+	/* truncate prefix after the above checks are done, so they can log
+	   the full prefix in error conditions */
+	*wildcardp = '\0';
+	ns->prefix_len = strlen(ns->prefix);
+	return 0;
+}
+
+static void
+shared_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED,
+				 struct mailbox_list_settings *set)
+{
+	set->layout = "shared";
+}
+
+static void
+get_nonexistent_user_location(struct shared_storage *storage,
+			      const char *username, string_t *location)
+{
+	/* user wasn't found. we'll still need to create the storage
+	   to avoid exposing which users exist and which don't. */
+	str_append(location, storage->storage_class_name);
+	str_append_c(location, ':');
+
+	/* use a reachable but nonexistent path as the mail root directory */
+	str_append(location, storage->storage.user->set->base_dir);
+	str_append(location, "/user-not-found/");
+	str_append(location, username);
+}
+
+static bool shared_namespace_exists(struct mail_namespace *ns)
+{
+	const char *path;
+	struct stat st;
+
+	if (!mailbox_list_get_root_path(ns->list, MAILBOX_LIST_PATH_TYPE_DIR,
+					&path)) {
+		/* we can't know if this exists */
+		return TRUE;
+	}
+	return stat(path, &st) == 0;
+}
+
+int shared_storage_get_namespace(struct mail_namespace **_ns,
+				 const char **_name)
+{
+	struct mail_storage *_storage = (*_ns)->storage;
+	struct mailbox_list *list = (*_ns)->list;
+	struct shared_storage *storage = SHARED_STORAGE(_storage);
+	struct mail_user *user = _storage->user;
+	struct mail_namespace *new_ns, *ns = *_ns;
+	struct mail_namespace_settings *ns_set, *unexpanded_ns_set;
+	struct mail_user *owner;
+	const char *domain = NULL, *username = NULL, *userdomain = NULL;
+	const char *name, *p, *next, **dest, *error;
+	string_t *prefix, *location;
+	char ns_sep = mail_namespace_get_sep(ns);
+	int ret;
+
+	p = storage->ns_prefix_pattern;
+	for (name = *_name; *p != '\0';) {
+		if (*p != '%') {
+			if (*p != *name)
+				break;
+			p++; name++;
+			continue;
+		}
+		switch (*++p) {
+		case 'd':
+			dest = &domain;
+			break;
+		case 'n':
+			dest = &username;
+			break;
+		case 'u':
+			dest = &userdomain;
+			break;
+		default:
+			/* we checked this already above */
+			i_unreached();
+		}
+		p++;
+
+		next = strchr(name, *p != '\0' ? *p : ns_sep);
+		if (next == NULL) {
+			*dest = name;
+			name = "";
+			break;
+		}
+		*dest = t_strdup_until(name, next);
+		name = next;
+	}
+	if (*p != '\0') {
+		if (*name == '\0' ||
+		    (name[1] == '\0' && *name == ns_sep)) {
+			/* trying to open <prefix>/<user> mailbox */
+			name = "INBOX";
+		} else {
+			mailbox_list_set_critical(list,
+					"Invalid namespace prefix %s vs %s",
+					storage->ns_prefix_pattern, *_name);
+			return -1;
+		}
+	}
+
+	/* successfully matched the name. */
+	if (userdomain != NULL) {
+		/* user@domain given */
+		domain = strchr(userdomain, '@');
+		if (domain == NULL)
+			username = userdomain;
+		else {
+			username = t_strdup_until(userdomain, domain);
+			domain++;
+		}
+	} else if (username == NULL) {
+		/* trying to open namespace "shared/domain"
+		   namespace prefix. */
+		mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND,
+				       T_MAIL_ERR_MAILBOX_NOT_FOUND(*_name));
+		return -1;
+	} else {
+		if (domain == NULL) {
+			/* no domain given, use ours (if we have one) */
+			domain = i_strchr_to_next(user->username, '@');
+		}
+		userdomain = domain == NULL ? username :
+			t_strconcat(username, "@", domain, NULL);
+	}
+	if (*userdomain == '\0') {
+		mailbox_list_set_error(list, MAIL_ERROR_PARAMS,
+				       "Empty username doesn't exist");
+		return -1;
+	}
+
+	/* expand the namespace prefix and see if it already exists.
+	   this should normally happen only when the mailbox is being opened */
+	struct var_expand_table tab[] = {
+		{ 'u', userdomain, "user" },
+		{ 'n', username, "username" },
+		{ 'd', domain, "domain" },
+		{ 'h', NULL, "home" },
+		{ '\0', NULL, NULL }
+	};
+
+	prefix = t_str_new(128);
+	str_append(prefix, ns->prefix);
+	if (var_expand(prefix, storage->ns_prefix_pattern, tab, &error) <= 0) {
+		mailbox_list_set_critical(list,
+			"Failed to expand namespace prefix '%s': %s",
+			storage->ns_prefix_pattern, error);
+		return -1;
+	}
+
+	*_ns = mail_namespace_find_prefix(user->namespaces, str_c(prefix));
+	if (*_ns != NULL) {
+		*_name = mailbox_list_get_storage_name(ns->list,
+				t_strconcat(ns->prefix, name, NULL));
+		return 0;
+	}
+
+	owner = mail_user_alloc(event_get_parent(user->event), userdomain,
+				user->set_info, user->unexpanded_set);
+	owner->_service_user = user->_service_user;
+	mail_storage_service_user_ref(owner->_service_user);
+	owner->creator = user;
+	owner->autocreated = TRUE;
+	owner->session_id = p_strdup(owner->pool, user->session_id);
+	if (mail_user_init(owner, &error) < 0) {
+		if (!owner->nonexistent) {
+			mailbox_list_set_critical(list,
+				"Couldn't create namespace '%s' for user %s: %s",
+				ns->prefix, userdomain, error);
+			mail_user_deinit(&owner);
+			return -1;
+		}
+		ret = 0;
+	} else if (!var_has_key(storage->location, 'h', "home")) {
+		ret = 1;
+	} else {
+		/* we'll need to look up the user's home directory */
+		if ((ret = mail_user_get_home(owner, &tab[3].value)) < 0) {
+			mailbox_list_set_critical(list, "Namespace '%s': "
+				"Could not lookup home for user %s",
+				ns->prefix, userdomain);
+			mail_user_deinit(&owner);
+			return -1;
+		}
+	}
+
+	location = t_str_new(256);
+	if (ret > 0 &&
+	    var_expand(location, storage->location, tab, &error) <= 0) {
+		mailbox_list_set_critical(list,
+			"Failed to expand namespace location '%s': %s",
+			storage->location, error);
+		return -1;
+	}
+
+	/* create the new namespace */
+	new_ns = i_new(struct mail_namespace, 1);
+	new_ns->refcount = 1;
+	new_ns->type = MAIL_NAMESPACE_TYPE_SHARED;
+	new_ns->user = user;
+	new_ns->prefix = i_strdup(str_c(prefix));
+	new_ns->owner = owner;
+	new_ns->flags = (NAMESPACE_FLAG_SUBSCRIPTIONS & ns->flags) |
+		NAMESPACE_FLAG_LIST_PREFIX | NAMESPACE_FLAG_HIDDEN |
+		NAMESPACE_FLAG_AUTOCREATED | NAMESPACE_FLAG_INBOX_ANY;
+	new_ns->user_set = user->set;
+	new_ns->mail_set = _storage->set;
+	i_array_init(&new_ns->all_storages, 2);
+
+	if (ret <= 0) {
+		get_nonexistent_user_location(storage, userdomain, location);
+		new_ns->flags |= NAMESPACE_FLAG_UNUSABLE;
+		e_debug(ns->user->event,
+			"shared: Tried to access mails of "
+			"nonexistent user %s", userdomain);
+	}
+
+	ns_set = p_new(user->pool, struct mail_namespace_settings, 1);
+	ns_set->type = "shared";
+	ns_set->separator = p_strdup_printf(user->pool, "%c", ns_sep);
+	ns_set->prefix = new_ns->prefix;
+	ns_set->location = p_strdup(user->pool, str_c(location));
+	ns_set->hidden = TRUE;
+	ns_set->list = "yes";
+	new_ns->set = ns_set;
+
+	unexpanded_ns_set =
+		p_new(user->pool, struct mail_namespace_settings, 1);
+	*unexpanded_ns_set = *ns_set;
+	unexpanded_ns_set->location =
+		p_strdup(user->pool, storage->unexpanded_location);
+	new_ns->unexpanded_set = unexpanded_ns_set;
+
+	/* We need to create a prefix="" namespace for the owner */
+	if (mail_namespaces_init_location(owner, str_c(location), &error) < 0) {
+		/* owner gets freed by namespace deinit */
+		mail_namespace_destroy(new_ns);
+		return -1;
+	}
+
+	if (mail_storage_create(new_ns, NULL, _storage->flags |
+				MAIL_STORAGE_FLAG_NO_AUTOVERIFY, &error) < 0) {
+		mailbox_list_set_critical(list, "Namespace '%s': %s",
+					  new_ns->prefix, error);
+		/* owner gets freed by namespace deinit */
+		mail_namespace_destroy(new_ns);
+		return -1;
+	}
+	if ((new_ns->flags & NAMESPACE_FLAG_UNUSABLE) == 0 &&
+	    !shared_namespace_exists(new_ns)) {
+		/* this user doesn't have a usable storage */
+		new_ns->flags |= NAMESPACE_FLAG_UNUSABLE;
+	}
+	/* mark the shared namespace root as usable, since it now has
+	   child namespaces */
+	ns->flags |= NAMESPACE_FLAG_USABLE;
+	*_name = mailbox_list_get_storage_name(new_ns->list,
+				t_strconcat(new_ns->prefix, name, NULL));
+	*_ns = new_ns;
+	if (_storage->class_flags == 0) {
+		/* flags are unset if we were using "auto" storage */
+		_storage->class_flags =
+			mail_namespace_get_default_storage(new_ns)->class_flags;
+	}
+
+	mail_user_add_namespace(user, &new_ns);
+	return 0;
+}
+
+struct mail_storage shared_storage = {
+	.name = MAIL_SHARED_STORAGE_NAME,
+	.class_flags = 0, /* unknown at this point */
+
+	.v = {
+		NULL,
+		shared_storage_alloc,
+		shared_storage_create,
+		index_storage_destroy,
+		NULL,
+		shared_storage_get_list_settings,
+		NULL,
+		fail_mailbox_alloc,
+		NULL,
+		NULL,
+	}
+};
diff --git a/src/lib-storage/index/shared/shared-storage.h b/src/lib-storage/index/shared/shared-storage.h
new file mode 100644
index 0000000..5463b1c
--- /dev/null
+++ b/src/lib-storage/index/shared/shared-storage.h
@@ -0,0 +1,22 @@
+#ifndef SHARED_STORAGE_H
+#define SHARED_STORAGE_H
+
+struct shared_storage {
+	struct mail_storage storage;
+	union mailbox_list_module_context list_module_ctx;
+
+	const char *ns_prefix_pattern;
+	const char *location, *unexpanded_location;
+
+	const char *storage_class_name;
+};
+
+#define SHARED_STORAGE(s)	container_of(s, struct shared_storage, storage)
+
+struct mailbox_list *shared_mailbox_list_alloc(void);
+
+/* Returns -1 = error, 0 = user doesn't exist, 1 = ok */
+int shared_storage_get_namespace(struct mail_namespace **_ns,
+				 const char **_name);
+
+#endif