Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

66 files changed, 21328 insertions, 0 deletions

diff --git a/src/plugins/fts-lucene/Makefile.am b/src/plugins/fts-lucene/Makefile.am
new file mode 100644
index 0000000..d68e6ae
--- /dev/null
+++ b/src/plugins/fts-lucene/Makefile.am
@@ -0,0 +1,61 @@
+doveadm_moduledir = $(moduledir)/doveadm
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/plugins/fts \
+	-I$(top_srcdir)/src/doveadm
+
+AM_CXXFLAGS = \
+	$(CLUCENE_CFLAGS) \
+	$(LIBEXTTEXTCAT_CFLAGS)
+
+NOPLUGIN_LDFLAGS =
+lib21_fts_lucene_plugin_la_LDFLAGS = -module -avoid-version
+lib20_doveadm_fts_lucene_plugin_la_LDFLAGS = -module -avoid-version
+
+module_LTLIBRARIES = \
+	lib21_fts_lucene_plugin.la
+
+if BUILD_FTS_STEMMER
+STEMMER_LIBS = -lstemmer
+SHOWBALL_SOURCES = Snowball.cc
+endif
+
+if BUILD_FTS_EXTTEXTCAT
+TEXTCAT_LIBS = $(LIBEXTTEXTCAT_LIBS)
+else
+if BUILD_FTS_TEXTCAT
+TEXTCAT_LIBS = -ltextcat
+endif
+endif
+
+lib21_fts_lucene_plugin_la_LIBADD = \
+	$(CLUCENE_LIBS) $(TEXTCAT_LIBS) $(STEMMER_LIBS)
+
+lib21_fts_lucene_plugin_la_SOURCES = \
+	fts-lucene-plugin.c \
+	fts-backend-lucene.c \
+	lucene-wrapper.cc \
+	$(SHOWBALL_SOURCES)
+
+noinst_HEADERS = \
+	fts-lucene-plugin.h \
+	lucene-wrapper.h \
+	SnowballAnalyzer.h \
+	SnowballFilter.h
+
+if BUILD_FTS_TEXTCAT
+exampledir = $(docdir)/example-config
+example_DATA = \
+	textcat.conf
+endif
+EXTRA_DIST = textcat.conf
+
+doveadm_module_LTLIBRARIES = \
+	lib20_doveadm_fts_lucene_plugin.la
+
+lib20_doveadm_fts_lucene_plugin_la_SOURCES = \
+	doveadm-fts-lucene.c
diff --git a/src/plugins/fts-lucene/Makefile.in b/src/plugins/fts-lucene/Makefile.in
new file mode 100644
index 0000000..323982d
--- /dev/null
+++ b/src/plugins/fts-lucene/Makefile.in
@@ -0,0 +1,990 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/plugins/fts-lucene
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(doveadm_moduledir)" \
+	"$(DESTDIR)$(moduledir)" "$(DESTDIR)$(exampledir)"
+LTLIBRARIES = $(doveadm_module_LTLIBRARIES) $(module_LTLIBRARIES)
+lib20_doveadm_fts_lucene_plugin_la_LIBADD =
+am_lib20_doveadm_fts_lucene_plugin_la_OBJECTS = doveadm-fts-lucene.lo
+lib20_doveadm_fts_lucene_plugin_la_OBJECTS =  \
+	$(am_lib20_doveadm_fts_lucene_plugin_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+lib20_doveadm_fts_lucene_plugin_la_LINK = $(LIBTOOL) $(AM_V_lt) \
+	--tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \
+	$(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(lib20_doveadm_fts_lucene_plugin_la_LDFLAGS) $(LDFLAGS) -o $@
+am__DEPENDENCIES_1 =
+@BUILD_FTS_EXTTEXTCAT_TRUE@am__DEPENDENCIES_2 = $(am__DEPENDENCIES_1)
+lib21_fts_lucene_plugin_la_DEPENDENCIES = $(am__DEPENDENCIES_1) \
+	$(am__DEPENDENCIES_2) $(am__DEPENDENCIES_1)
+am__lib21_fts_lucene_plugin_la_SOURCES_DIST = fts-lucene-plugin.c \
+	fts-backend-lucene.c lucene-wrapper.cc Snowball.cc
+@BUILD_FTS_STEMMER_TRUE@am__objects_1 = Snowball.lo
+am_lib21_fts_lucene_plugin_la_OBJECTS = fts-lucene-plugin.lo \
+	fts-backend-lucene.lo lucene-wrapper.lo $(am__objects_1)
+lib21_fts_lucene_plugin_la_OBJECTS =  \
+	$(am_lib21_fts_lucene_plugin_la_OBJECTS)
+lib21_fts_lucene_plugin_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
+	$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
+	$(AM_CXXFLAGS) $(CXXFLAGS) \
+	$(lib21_fts_lucene_plugin_la_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/Snowball.Plo \
+	./$(DEPDIR)/doveadm-fts-lucene.Plo \
+	./$(DEPDIR)/fts-backend-lucene.Plo \
+	./$(DEPDIR)/fts-lucene-plugin.Plo \
+	./$(DEPDIR)/lucene-wrapper.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
+LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CXXFLAGS) $(CXXFLAGS)
+AM_V_CXX = $(am__v_CXX_@AM_V@)
+am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@)
+am__v_CXX_0 = @echo "  CXX     " $@;
+am__v_CXX_1 = 
+CXXLD = $(CXX)
+CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+	$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CXXLD = $(am__v_CXXLD_@AM_V@)
+am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@)
+am__v_CXXLD_0 = @echo "  CXXLD   " $@;
+am__v_CXXLD_1 = 
+SOURCES = $(lib20_doveadm_fts_lucene_plugin_la_SOURCES) \
+	$(lib21_fts_lucene_plugin_la_SOURCES)
+DIST_SOURCES = $(lib20_doveadm_fts_lucene_plugin_la_SOURCES) \
+	$(am__lib21_fts_lucene_plugin_la_SOURCES_DIST)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+DATA = $(example_DATA)
+HEADERS = $(noinst_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = 
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+doveadm_moduledir = $(moduledir)/doveadm
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/plugins/fts \
+	-I$(top_srcdir)/src/doveadm
+
+AM_CXXFLAGS = \
+	$(CLUCENE_CFLAGS) \
+	$(LIBEXTTEXTCAT_CFLAGS)
+
+lib21_fts_lucene_plugin_la_LDFLAGS = -module -avoid-version
+lib20_doveadm_fts_lucene_plugin_la_LDFLAGS = -module -avoid-version
+module_LTLIBRARIES = \
+	lib21_fts_lucene_plugin.la
+
+@BUILD_FTS_STEMMER_TRUE@STEMMER_LIBS = -lstemmer
+@BUILD_FTS_STEMMER_TRUE@SHOWBALL_SOURCES = Snowball.cc
+@BUILD_FTS_EXTTEXTCAT_FALSE@@BUILD_FTS_TEXTCAT_TRUE@TEXTCAT_LIBS = -ltextcat
+@BUILD_FTS_EXTTEXTCAT_TRUE@TEXTCAT_LIBS = $(LIBEXTTEXTCAT_LIBS)
+lib21_fts_lucene_plugin_la_LIBADD = \
+	$(CLUCENE_LIBS) $(TEXTCAT_LIBS) $(STEMMER_LIBS)
+
+lib21_fts_lucene_plugin_la_SOURCES = \
+	fts-lucene-plugin.c \
+	fts-backend-lucene.c \
+	lucene-wrapper.cc \
+	$(SHOWBALL_SOURCES)
+
+noinst_HEADERS = \
+	fts-lucene-plugin.h \
+	lucene-wrapper.h \
+	SnowballAnalyzer.h \
+	SnowballFilter.h
+
+@BUILD_FTS_TEXTCAT_TRUE@exampledir = $(docdir)/example-config
+@BUILD_FTS_TEXTCAT_TRUE@example_DATA = \
+@BUILD_FTS_TEXTCAT_TRUE@	textcat.conf
+
+EXTRA_DIST = textcat.conf
+doveadm_module_LTLIBRARIES = \
+	lib20_doveadm_fts_lucene_plugin.la
+
+lib20_doveadm_fts_lucene_plugin_la_SOURCES = \
+	doveadm-fts-lucene.c
+
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .cc .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/plugins/fts-lucene/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/plugins/fts-lucene/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+install-doveadm_moduleLTLIBRARIES: $(doveadm_module_LTLIBRARIES)
+	@$(NORMAL_INSTALL)
+	@list='$(doveadm_module_LTLIBRARIES)'; test -n "$(doveadm_moduledir)" || list=; \
+	list2=; for p in $$list; do \
+	  if test -f $$p; then \
+	    list2="$$list2 $$p"; \
+	  else :; fi; \
+	done; \
+	test -z "$$list2" || { \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(doveadm_moduledir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(doveadm_moduledir)" || exit 1; \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(doveadm_moduledir)'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(doveadm_moduledir)"; \
+	}
+
+uninstall-doveadm_moduleLTLIBRARIES:
+	@$(NORMAL_UNINSTALL)
+	@list='$(doveadm_module_LTLIBRARIES)'; test -n "$(doveadm_moduledir)" || list=; \
+	for p in $$list; do \
+	  $(am__strip_dir) \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(doveadm_moduledir)/$$f'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(doveadm_moduledir)/$$f"; \
+	done
+
+clean-doveadm_moduleLTLIBRARIES:
+	-test -z "$(doveadm_module_LTLIBRARIES)" || rm -f $(doveadm_module_LTLIBRARIES)
+	@list='$(doveadm_module_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+install-moduleLTLIBRARIES: $(module_LTLIBRARIES)
+	@$(NORMAL_INSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	list2=; for p in $$list; do \
+	  if test -f $$p; then \
+	    list2="$$list2 $$p"; \
+	  else :; fi; \
+	done; \
+	test -z "$$list2" || { \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(moduledir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(moduledir)" || exit 1; \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(moduledir)'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(moduledir)"; \
+	}
+
+uninstall-moduleLTLIBRARIES:
+	@$(NORMAL_UNINSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	for p in $$list; do \
+	  $(am__strip_dir) \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(moduledir)/$$f'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(moduledir)/$$f"; \
+	done
+
+clean-moduleLTLIBRARIES:
+	-test -z "$(module_LTLIBRARIES)" || rm -f $(module_LTLIBRARIES)
+	@list='$(module_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+lib20_doveadm_fts_lucene_plugin.la: $(lib20_doveadm_fts_lucene_plugin_la_OBJECTS) $(lib20_doveadm_fts_lucene_plugin_la_DEPENDENCIES) $(EXTRA_lib20_doveadm_fts_lucene_plugin_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(lib20_doveadm_fts_lucene_plugin_la_LINK) -rpath $(doveadm_moduledir) $(lib20_doveadm_fts_lucene_plugin_la_OBJECTS) $(lib20_doveadm_fts_lucene_plugin_la_LIBADD) $(LIBS)
+
+lib21_fts_lucene_plugin.la: $(lib21_fts_lucene_plugin_la_OBJECTS) $(lib21_fts_lucene_plugin_la_DEPENDENCIES) $(EXTRA_lib21_fts_lucene_plugin_la_DEPENDENCIES) 
+	$(AM_V_CXXLD)$(lib21_fts_lucene_plugin_la_LINK) -rpath $(moduledir) $(lib21_fts_lucene_plugin_la_OBJECTS) $(lib21_fts_lucene_plugin_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Snowball.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-fts-lucene.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-backend-lucene.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-lucene-plugin.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/lucene-wrapper.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+.cc.o:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $<
+
+.cc.obj:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.cc.lo:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-exampleDATA: $(example_DATA)
+	@$(NORMAL_INSTALL)
+	@list='$(example_DATA)'; test -n "$(exampledir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(exampledir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(exampledir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(exampledir)'"; \
+	  $(INSTALL_DATA) $$files "$(DESTDIR)$(exampledir)" || exit $$?; \
+	done
+
+uninstall-exampleDATA:
+	@$(NORMAL_UNINSTALL)
+	@list='$(example_DATA)'; test -n "$(exampledir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(exampledir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(DATA) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(doveadm_moduledir)" "$(DESTDIR)$(moduledir)" "$(DESTDIR)$(exampledir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-doveadm_moduleLTLIBRARIES clean-generic clean-libtool \
+	clean-moduleLTLIBRARIES mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/Snowball.Plo
+	-rm -f ./$(DEPDIR)/doveadm-fts-lucene.Plo
+	-rm -f ./$(DEPDIR)/fts-backend-lucene.Plo
+	-rm -f ./$(DEPDIR)/fts-lucene-plugin.Plo
+	-rm -f ./$(DEPDIR)/lucene-wrapper.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-doveadm_moduleLTLIBRARIES install-exampleDATA \
+	install-moduleLTLIBRARIES
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/Snowball.Plo
+	-rm -f ./$(DEPDIR)/doveadm-fts-lucene.Plo
+	-rm -f ./$(DEPDIR)/fts-backend-lucene.Plo
+	-rm -f ./$(DEPDIR)/fts-lucene-plugin.Plo
+	-rm -f ./$(DEPDIR)/lucene-wrapper.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-doveadm_moduleLTLIBRARIES \
+	uninstall-exampleDATA uninstall-moduleLTLIBRARIES
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-doveadm_moduleLTLIBRARIES clean-generic clean-libtool \
+	clean-moduleLTLIBRARIES cscopelist-am ctags ctags-am distclean \
+	distclean-compile distclean-generic distclean-libtool \
+	distclean-tags distdir dvi dvi-am html html-am info info-am \
+	install install-am install-data install-data-am \
+	install-doveadm_moduleLTLIBRARIES install-dvi install-dvi-am \
+	install-exampleDATA install-exec install-exec-am install-html \
+	install-html-am install-info install-info-am install-man \
+	install-moduleLTLIBRARIES install-pdf install-pdf-am \
+	install-ps install-ps-am install-strip installcheck \
+	installcheck-am installdirs maintainer-clean \
+	maintainer-clean-generic mostlyclean mostlyclean-compile \
+	mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+	tags tags-am uninstall uninstall-am \
+	uninstall-doveadm_moduleLTLIBRARIES uninstall-exampleDATA \
+	uninstall-moduleLTLIBRARIES
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/plugins/fts-lucene/Snowball.cc b/src/plugins/fts-lucene/Snowball.cc
new file mode 100644
index 0000000..43b54e3
--- /dev/null
+++ b/src/plugins/fts-lucene/Snowball.cc
@@ -0,0 +1,151 @@
+/*------------------------------------------------------------------------------
+* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
+*
+* Distributable under the terms of either the Apache License (Version 2.0) or
+* the GNU Lesser General Public License, as specified in the COPYING file.
+------------------------------------------------------------------------------*/
+#include <CLucene.h>
+#include "SnowballAnalyzer.h"
+#include "SnowballFilter.h"
+#include <CLucene/util/CLStreams.h>
+#include <CLucene/analysis/Analyzers.h>
+#include <CLucene/analysis/standard/StandardTokenizer.h>
+#include <CLucene/analysis/standard/StandardFilter.h>
+
+extern "C" {
+#include "lib.h"
+#include "buffer.h"
+#include "unichar.h"
+#include "lucene-wrapper.h"
+};
+
+CL_NS_USE(analysis)
+CL_NS_USE(util)
+CL_NS_USE2(analysis,standard)
+
+CL_NS_DEF2(analysis,snowball)
+
+  /** Builds the named analyzer with no stop words. */
+  SnowballAnalyzer::SnowballAnalyzer(normalizer_func_t *_normalizer, const char* _language)
+      : language(i_strdup(_language)),
+	normalizer(_normalizer),
+	stopSet(NULL),
+	prevstream(NULL)
+  {
+  }
+
+  SnowballAnalyzer::~SnowballAnalyzer()
+  {
+      if (prevstream)
+	  _CLDELETE(prevstream);
+      i_free(language);
+      if ( stopSet != NULL )
+	  _CLDELETE(stopSet);
+  }
+
+  /** Builds the named analyzer with the given stop words.
+  */
+  SnowballAnalyzer::SnowballAnalyzer(const char* language, const TCHAR** stopWords)
+      : language(i_strdup(language)),
+	normalizer(NULL),
+	stopSet(_CLNEW CLTCSetList(true)),
+	prevstream(NULL)
+  {
+      StopFilter::fillStopTable(stopSet,stopWords);
+  }
+
+  TokenStream* SnowballAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) {
+	 return this->tokenStream(fieldName,reader,false);
+  }
+
+  /** Constructs a {@link StandardTokenizer} filtered by a {@link
+      StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
+  TokenStream* SnowballAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader, bool deleteReader) {
+		BufferedReader* bufferedReader = reader->__asBufferedReader();
+		TokenStream* result;
+
+		if ( bufferedReader == NULL )
+			result =  _CLNEW StandardTokenizer( _CLNEW FilteredBufferedReader(reader, deleteReader), true );
+		else
+			result = _CLNEW StandardTokenizer(bufferedReader, deleteReader);
+
+	 result = _CLNEW StandardFilter(result, true);
+    result = _CLNEW CL_NS(analysis)::LowerCaseFilter(result, true);
+    if (stopSet != NULL)
+      result = _CLNEW CL_NS(analysis)::StopFilter(result, true, stopSet);
+    result = _CLNEW SnowballFilter(result, normalizer, language, true);
+    return result;
+  }
+  
+  TokenStream* SnowballAnalyzer::reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) {
+      if (prevstream) _CLDELETE(prevstream);
+      prevstream = this->tokenStream(fieldName, reader);
+      return prevstream;
+  }
+  
+  
+  
+  
+  
+  
+    /** Construct the named stemming filter.
+   *
+   * @param in the input tokens to stem
+   * @param name the name of a stemmer
+   */
+	SnowballFilter::SnowballFilter(TokenStream* in, normalizer_func_t *normalizer, const char* language, bool deleteTS):
+		TokenFilter(in,deleteTS)
+	{
+		stemmer = sb_stemmer_new(language, NULL); //use utf8 encoding
+		this->normalizer = normalizer;
+
+		if ( stemmer == NULL ){
+			_CLTHROWA(CL_ERR_IllegalArgument, "language not available for stemming\n"); //todo: richer error
+		}
+    }
+
+	SnowballFilter::~SnowballFilter(){
+		sb_stemmer_delete(stemmer);
+	}
+
+  /** Returns the next input Token, after being stemmed */
+  Token* SnowballFilter::next(Token* token){
+    if (input->next(token) == NULL)
+      return NULL;
+
+	unsigned char utf8text[LUCENE_MAX_WORD_LEN*5+1];
+	unsigned int len = I_MIN(LUCENE_MAX_WORD_LEN, token->termLength());
+
+	buffer_t buf = { { 0, 0 } };
+	i_assert(sizeof(wchar_t) == sizeof(unichar_t));
+	buffer_create_from_data(&buf, utf8text, sizeof(utf8text));
+	uni_ucs4_to_utf8((const unichar_t *)token->termBuffer(), len, &buf);
+
+    const sb_symbol* stemmed = sb_stemmer_stem(stemmer, utf8text, buf.used);
+	if ( stemmed == NULL )
+		_CLTHROWA(CL_ERR_Runtime,"Out of memory");
+
+	int stemmedLen=sb_stemmer_length(stemmer);
+
+	if (normalizer == NULL) {
+	  unsigned int tchartext_size =
+			  uni_utf8_strlen_n(stemmed, stemmedLen) + 1;
+	  TCHAR tchartext[tchartext_size];
+	  lucene_utf8_n_to_tchar(stemmed, stemmedLen, tchartext, tchartext_size);
+	  token->set(tchartext,token->startOffset(), token->endOffset(), token->type());
+	} else T_BEGIN {
+	  buffer_t *norm_buf = t_buffer_create(stemmedLen);
+	  normalizer(stemmed, stemmedLen, norm_buf);
+
+	  unsigned int tchartext_size =
+			  uni_utf8_strlen_n(norm_buf->data, norm_buf->used) + 1;
+	  TCHAR tchartext[tchartext_size];
+	  lucene_utf8_n_to_tchar((const unsigned char *)norm_buf->data,
+							 norm_buf->used, tchartext, tchartext_size);
+	  token->set(tchartext,token->startOffset(), token->endOffset(), token->type());
+	} T_END;
+	return token;
+  }
+
+
+CL_NS_END2
diff --git a/src/plugins/fts-lucene/SnowballAnalyzer.h b/src/plugins/fts-lucene/SnowballAnalyzer.h
new file mode 100644
index 0000000..45455c5
--- /dev/null
+++ b/src/plugins/fts-lucene/SnowballAnalyzer.h
@@ -0,0 +1,51 @@
+/*------------------------------------------------------------------------------
+* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
+*
+* Distributable under the terms of either the Apache License (Version 2.0) or
+* the GNU Lesser General Public License, as specified in the COPYING file.
+------------------------------------------------------------------------------*/
+#ifndef _lucene_analysis_snowball_analyser_
+#define _lucene_analysis_snowball_analyser_
+
+extern "C" {
+#include "lib.h"
+#include "unichar.h"
+};
+#include "CLucene/analysis/AnalysisHeader.h"
+
+CL_CLASS_DEF(util,BufferedReader)
+CL_NS_DEF2(analysis,snowball)
+
+/** Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
+ * LowerCaseFilter}, {@link StopFilter} and {@link SnowballFilter}.
+ *
+ * Available stemmers are listed in {@link net.sf.snowball.ext}.  The name of a
+ * stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
+ * {@link EnglishStemmer} is named "English".
+ */
+class CLUCENE_CONTRIBS_EXPORT SnowballAnalyzer: public Analyzer {
+  char* language;
+  normalizer_func_t *normalizer;
+  CLTCSetList* stopSet;
+  TokenStream *prevstream;
+
+public:
+  /** Builds the named analyzer with no stop words. */
+  SnowballAnalyzer(normalizer_func_t *normalizer, const char* language="english");
+
+  /** Builds the named analyzer with the given stop words.
+  */
+  SnowballAnalyzer(const char* language, const TCHAR** stopWords);
+
+  ~SnowballAnalyzer();
+
+  /** Constructs a {@link StandardTokenizer} filtered by a {@link
+      StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
+  TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
+  TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader, bool deleteReader);
+  TokenStream* reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
+};
+
+CL_NS_END2
+#endif
+
diff --git a/src/plugins/fts-lucene/SnowballFilter.h b/src/plugins/fts-lucene/SnowballFilter.h
new file mode 100644
index 0000000..6a0ed12
--- /dev/null
+++ b/src/plugins/fts-lucene/SnowballFilter.h
@@ -0,0 +1,42 @@
+/*------------------------------------------------------------------------------
+* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
+*
+* Distributable under the terms of either the Apache License (Version 2.0) or
+* the GNU Lesser General Public License, as specified in the COPYING file.
+------------------------------------------------------------------------------*/
+#ifndef _lucene_analysis_snowball_filter_
+#define _lucene_analysis_snowball_filter_
+
+#include "CLucene/analysis/AnalysisHeader.h"
+#include "libstemmer.h"
+
+CL_NS_DEF2(analysis,snowball)
+
+/** A filter that stems words using a Snowball-generated stemmer.
+ *
+ * Available stemmers are listed in {@link net.sf.snowball.ext}.  The name of a
+ * stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
+ * {@link EnglishStemmer} is named "English".
+ *
+ * Note: todo: This is not thread safe...
+ */
+class CLUCENE_CONTRIBS_EXPORT SnowballFilter: public TokenFilter {
+	struct sb_stemmer * stemmer;
+	normalizer_func_t *normalizer;
+public:
+
+  /** Construct the named stemming filter.
+   *
+   * @param in the input tokens to stem
+   * @param name the name of a stemmer
+   */
+	SnowballFilter(TokenStream* in, normalizer_func_t *normalizer, const char* language, bool deleteTS);
+
+	~SnowballFilter();
+
+    /** Returns the next input Token, after being stemmed */
+    Token* next(Token* token);
+};
+
+CL_NS_END2
+#endif
diff --git a/src/plugins/fts-lucene/doveadm-fts-lucene.c b/src/plugins/fts-lucene/doveadm-fts-lucene.c
new file mode 100644
index 0000000..a761907
--- /dev/null
+++ b/src/plugins/fts-lucene/doveadm-fts-lucene.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "doveadm-dump.h"
+#include "doveadm-fts.h"
+#include "lucene-wrapper.h"
+
+#include <stdio.h>
+#include <sys/stat.h>
+
+const char *doveadm_fts_lucene_plugin_version = DOVECOT_ABI_VERSION;
+
+void doveadm_fts_lucene_plugin_init(struct module *module);
+void doveadm_fts_lucene_plugin_deinit(void);
+
+static void
+cmd_dump_fts_lucene(const char *path, const char *const *args ATTR_UNUSED)
+{
+	struct lucene_index *index;
+	struct lucene_index_iter *iter;
+	guid_128_t prev_guid;
+	const struct lucene_index_record *rec;
+	bool first = TRUE;
+
+	i_zero(&prev_guid);
+	index = lucene_index_init(path, NULL, NULL);
+	iter = lucene_index_iter_init(index);
+	while ((rec = lucene_index_iter_next(iter)) != NULL) {
+		if (memcmp(prev_guid, rec->mailbox_guid,
+			   sizeof(prev_guid)) != 0) {
+			if (first)
+				first = FALSE;
+			else
+				printf("\n");
+			memcpy(prev_guid, rec->mailbox_guid, sizeof(prev_guid));
+			printf("%s: ", guid_128_to_string(prev_guid));
+		}
+		printf("%u", rec->uid);
+		if (rec->part_num != 0)
+			printf("[%u]", rec->part_num);
+		printf("\n");
+	}
+	printf("\n");
+	if (lucene_index_iter_deinit(&iter) < 0)
+		i_error("Lucene index iteration failed");
+	lucene_index_deinit(index);
+}
+
+static bool test_dump_fts_lucene(const char *path)
+{
+	struct stat st;
+
+	path = t_strconcat(path, "/segments.gen", NULL);
+	return stat(path, &st) == 0;
+}
+
+static const struct doveadm_cmd_dump doveadm_cmd_dump_fts_lucene = {
+	"fts-lucene",
+	test_dump_fts_lucene,
+	cmd_dump_fts_lucene
+};
+
+void doveadm_fts_lucene_plugin_init(struct module *module ATTR_UNUSED)
+{
+	doveadm_dump_register(&doveadm_cmd_dump_fts_lucene);
+}
+
+void doveadm_fts_lucene_plugin_deinit(void)
+{
+}
diff --git a/src/plugins/fts-lucene/fts-backend-lucene.c b/src/plugins/fts-lucene/fts-backend-lucene.c
new file mode 100644
index 0000000..963dbdf
--- /dev/null
+++ b/src/plugins/fts-lucene/fts-backend-lucene.c
@@ -0,0 +1,605 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "hex-binary.h"
+#include "strescape.h"
+#include "message-part.h"
+#include "mail-namespace.h"
+#include "mail-storage-private.h"
+#include "fts-expunge-log.h"
+#include "lucene-wrapper.h"
+#include "fts-indexer.h"
+#include "fts-lucene-plugin.h"
+
+#include <wchar.h>
+
+#define LUCENE_INDEX_DIR_NAME "lucene-indexes"
+#define LUCENE_EXPUNGE_LOG_NAME "dovecot-expunges.log"
+#define LUCENE_OPTIMIZE_BATCH_MSGS_COUNT 100
+
+struct lucene_fts_backend {
+	struct fts_backend backend;
+	char *dir_path;
+
+	struct lucene_index *index;
+	struct mailbox *selected_box;
+	unsigned int selected_box_generation;
+	guid_128_t selected_box_guid;
+
+	struct fts_expunge_log *expunge_log;
+
+	bool dir_created:1;
+	bool updating:1;
+};
+
+struct lucene_fts_backend_update_context {
+	struct fts_backend_update_context ctx;
+
+	struct mailbox *box;
+	uint32_t last_uid;
+	uint32_t last_indexed_uid;
+	char *first_box_vname;
+
+	uint32_t uid, part_num;
+	char *hdr_name;
+
+	unsigned int added_msgs;
+	struct fts_expunge_log_append_ctx *expunge_ctx;
+
+	bool lucene_opened;
+	bool last_indexed_uid_set;
+	bool mime_parts;
+};
+
+static int fts_backend_lucene_mkdir(struct lucene_fts_backend *backend)
+{
+	if (backend->dir_created)
+		return 0;
+
+	backend->dir_created = TRUE;
+	if (mailbox_list_mkdir_root(backend->backend.ns->list,
+				    backend->dir_path,
+				    MAILBOX_LIST_PATH_TYPE_INDEX) < 0)
+		return -1;
+	return 0;
+}
+
+static int
+fts_lucene_get_mailbox_guid(struct mailbox *box, guid_128_t guid_r)
+{
+	struct mailbox_metadata metadata;
+
+	if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
+				 &metadata) < 0) {
+		i_error("lucene: Couldn't get mailbox %s GUID: %s",
+			box->vname, mailbox_get_last_internal_error(box, NULL));
+		return -1;
+	}
+	memcpy(guid_r, metadata.guid, GUID_128_SIZE);
+	return 0;
+}
+
+static int
+fts_backend_select(struct lucene_fts_backend *backend, struct mailbox *box)
+{
+	guid_128_t guid;
+	unsigned char guid_hex[MAILBOX_GUID_HEX_LENGTH];
+	wchar_t wguid_hex[MAILBOX_GUID_HEX_LENGTH];
+	buffer_t buf;
+	unsigned int i;
+
+	i_assert(box != NULL);
+
+	if (backend->selected_box == box &&
+	    backend->selected_box_generation == box->generation_sequence)
+		return 0;
+
+	if (fts_lucene_get_mailbox_guid(box, guid) < 0)
+		return -1;
+	buffer_create_from_data(&buf, guid_hex, MAILBOX_GUID_HEX_LENGTH);
+	binary_to_hex_append(&buf, guid, GUID_128_SIZE);
+	for (i = 0; i < N_ELEMENTS(wguid_hex); i++)
+		wguid_hex[i] = guid_hex[i];
+
+	lucene_index_select_mailbox(backend->index, wguid_hex);
+
+	backend->selected_box = box;
+	memcpy(backend->selected_box_guid, guid,
+	       sizeof(backend->selected_box_guid));
+	backend->selected_box_generation = box->generation_sequence;
+	return 0;
+}
+
+static struct fts_backend *fts_backend_lucene_alloc(void)
+{
+	struct lucene_fts_backend *backend;
+
+	backend = i_new(struct lucene_fts_backend, 1);
+	backend->backend = fts_backend_lucene;
+	return &backend->backend;
+}
+
+static int
+fts_backend_lucene_init(struct fts_backend *_backend, const char **error_r)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+	struct fts_lucene_user *fuser =
+		FTS_LUCENE_USER_CONTEXT(_backend->ns->user);
+	const char *path;
+
+	if (fuser == NULL) {
+		/* invalid settings */
+		*error_r = "Invalid fts_lucene settings";
+		return -1;
+	}
+	/* fts already checked that index exists */
+
+	if (fuser->set.use_libfts) {
+		/* change our flags so we get proper input */
+		_backend->flags &= ENUM_NEGATE(FTS_BACKEND_FLAG_FUZZY_SEARCH);
+		_backend->flags |= FTS_BACKEND_FLAG_TOKENIZED_INPUT;
+	}
+	path = mailbox_list_get_root_forced(_backend->ns->list,
+					    MAILBOX_LIST_PATH_TYPE_INDEX);
+
+	backend->dir_path = i_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL);
+	backend->index = lucene_index_init(backend->dir_path,
+					   _backend->ns->list,
+					   &fuser->set);
+
+	path = t_strconcat(backend->dir_path, "/"LUCENE_EXPUNGE_LOG_NAME, NULL);
+	backend->expunge_log = fts_expunge_log_init(path);
+	return 0;
+}
+
+static void fts_backend_lucene_deinit(struct fts_backend *_backend)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+
+	if (backend->index != NULL)
+		lucene_index_deinit(backend->index);
+	if (backend->expunge_log != NULL)
+		fts_expunge_log_deinit(&backend->expunge_log);
+	i_free(backend->dir_path);
+	i_free(backend);
+}
+
+static int
+fts_backend_lucene_get_last_uid(struct fts_backend *_backend,
+				struct mailbox *box, uint32_t *last_uid_r)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+	struct fts_lucene_user *fuser =
+		FTS_LUCENE_USER_CONTEXT_REQUIRE(_backend->ns->user);
+	struct fts_index_header hdr;
+	uint32_t set_checksum;
+	int ret;
+
+	if (fts_index_get_header(box, &hdr)) {
+		set_checksum = fts_lucene_settings_checksum(&fuser->set);
+		ret = fts_index_have_compatible_settings(_backend->ns->list,
+							 set_checksum);
+		if (ret < 0)
+			return -1;
+		if (ret == 0) {
+			/* need to rebuild the index */
+			*last_uid_r = 0;
+		} else {
+			*last_uid_r = hdr.last_indexed_uid;
+		}
+		return 0;
+	}
+
+	/* either nothing has been indexed, or the index was corrupted.
+	   do it the slow way. */
+	if (fts_backend_select(backend, box) < 0)
+		return -1;
+	if (lucene_index_get_last_uid(backend->index, last_uid_r) < 0)
+		return -1;
+
+	fts_index_set_last_uid(box, *last_uid_r);
+	return 0;
+}
+
+static struct fts_backend_update_context *
+fts_backend_lucene_update_init(struct fts_backend *_backend)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+	struct lucene_fts_backend_update_context *ctx;
+	struct fts_lucene_user *fuser =
+		FTS_LUCENE_USER_CONTEXT_REQUIRE(_backend->ns->user);
+
+	i_assert(!backend->updating);
+
+	ctx = i_new(struct lucene_fts_backend_update_context, 1);
+	ctx->ctx.backend = _backend;
+	ctx->mime_parts = fuser->set.mime_parts;
+	backend->updating = TRUE;
+	return &ctx->ctx;
+}
+
+static bool
+fts_backend_lucene_need_optimize(struct lucene_fts_backend_update_context *ctx)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)ctx->ctx.backend;
+	unsigned int expunges;
+	uint32_t numdocs;
+
+	if (ctx->added_msgs >= LUCENE_OPTIMIZE_BATCH_MSGS_COUNT)
+		return TRUE;
+	if (lucene_index_get_doc_count(backend->index, &numdocs) < 0)
+		return FALSE;
+
+	if (fts_expunge_log_uid_count(backend->expunge_log, &expunges) < 0)
+		return FALSE;
+	return expunges > 0 &&
+		numdocs / expunges <= 50; /* >2% of index has been expunged */
+}
+
+static int
+fts_backend_lucene_update_deinit(struct fts_backend_update_context *_ctx)
+{
+	struct lucene_fts_backend_update_context *ctx =
+		(struct lucene_fts_backend_update_context *)_ctx;
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_ctx->backend;
+	int ret = _ctx->failed ? -1 : 0;
+
+	i_assert(backend->updating);
+
+	backend->updating = FALSE;
+	if (ctx->lucene_opened) {
+		if (lucene_index_build_deinit(backend->index) < 0)
+			ret = -1;
+	}
+
+	if (ctx->expunge_ctx != NULL) {
+		if (fts_expunge_log_append_commit(&ctx->expunge_ctx) < 0) {
+			struct stat st;
+			ret = -1;
+
+			if (stat(backend->dir_path, &st) < 0 && errno == ENOENT) {
+				/* lucene-indexes directory doesn't even exist,
+				   so dovecot.index's last_index_uid is wrong.
+				   rescan to update them. */
+				(void)lucene_index_rescan(backend->index);
+				ret = 0;
+			}
+		}
+	}
+
+	if (fts_backend_lucene_need_optimize(ctx)) {
+		if (ctx->lucene_opened)
+			(void)fts_backend_optimize(_ctx->backend);
+		else if (ctx->first_box_vname != NULL) {
+			struct mail_user *user = backend->backend.ns->user;
+			const char *cmd, *path;
+			int fd;
+
+			/* the optimize affects all mailboxes within namespace,
+			   so just use any mailbox name in it */
+			cmd = t_strdup_printf("OPTIMIZE\t0\t%s\t%s\n",
+				str_tabescape(user->username),
+				str_tabescape(ctx->first_box_vname));
+			fd = fts_indexer_cmd(user, cmd, &path);
+			i_close_fd(&fd);
+		}
+	}
+
+	i_free(ctx->first_box_vname);
+	i_free(ctx);
+	return ret;
+}
+
+static void
+fts_backend_lucene_update_set_mailbox(struct fts_backend_update_context *_ctx,
+				      struct mailbox *box)
+{
+	struct lucene_fts_backend_update_context *ctx =
+		(struct lucene_fts_backend_update_context *)_ctx;
+
+	if (ctx->last_uid != 0) {
+		fts_index_set_last_uid(ctx->box, ctx->last_uid);
+		ctx->last_uid = 0;
+	}
+	if (ctx->first_box_vname == NULL && box != NULL)
+		ctx->first_box_vname = i_strdup(box->vname);
+	ctx->box = box;
+	ctx->last_indexed_uid_set = FALSE;
+}
+
+static void
+fts_backend_lucene_update_expunge(struct fts_backend_update_context *_ctx,
+				  uint32_t uid)
+{
+	struct lucene_fts_backend_update_context *ctx =
+		(struct lucene_fts_backend_update_context *)_ctx;
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_ctx->backend;
+	struct fts_index_header hdr;
+
+	if (!ctx->last_indexed_uid_set) {
+		if (!fts_index_get_header(ctx->box, &hdr))
+			ctx->last_indexed_uid = 0;
+		else
+			ctx->last_indexed_uid = hdr.last_indexed_uid;
+		ctx->last_indexed_uid_set = TRUE;
+	}
+	if (ctx->last_indexed_uid == 0 ||
+	    uid > ctx->last_indexed_uid + 100) {
+		/* don't waste time adding expunge to log for a message that
+		   isn't even indexed. this check is racy, because indexer may
+		   just be in the middle of indexing this message. we'll
+		   attempt to avoid that by skipping the expunging only if
+		   indexing hasn't been done for a while (100 msgs). */
+		return;
+	}
+
+	if (ctx->expunge_ctx == NULL) {
+		ctx->expunge_ctx =
+			fts_expunge_log_append_begin(backend->expunge_log);
+	}
+
+	if (fts_backend_select(backend, ctx->box) < 0)
+		_ctx->failed = TRUE;
+
+	fts_expunge_log_append_next(ctx->expunge_ctx,
+				    backend->selected_box_guid, uid);
+}
+
+static bool
+fts_backend_lucene_update_set_build_key(struct fts_backend_update_context *_ctx,
+					const struct fts_backend_build_key *key)
+{
+	struct lucene_fts_backend_update_context *ctx =
+		(struct lucene_fts_backend_update_context *)_ctx;
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_ctx->backend;
+
+	if (!ctx->lucene_opened) {
+		if (fts_backend_lucene_mkdir(backend) < 0)
+			ctx->ctx.failed = TRUE;
+		if (lucene_index_build_init(backend->index) < 0)
+			ctx->ctx.failed = TRUE;
+		ctx->lucene_opened = TRUE;
+	}
+
+	if (fts_backend_select(backend, ctx->box) < 0)
+		_ctx->failed = TRUE;
+
+	switch (key->type) {
+	case FTS_BACKEND_BUILD_KEY_HDR:
+	case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+		i_assert(key->hdr_name != NULL);
+
+		i_free(ctx->hdr_name);
+		ctx->hdr_name = i_strdup(key->hdr_name);
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART:
+		i_free_and_null(ctx->hdr_name);
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+		i_unreached();
+	}
+
+	if (key->uid != ctx->last_uid) {
+		i_assert(key->uid >= ctx->last_uid);
+		ctx->last_uid = key->uid;
+		ctx->added_msgs++;
+	}
+
+	ctx->uid = key->uid;
+	if (ctx->mime_parts)
+		ctx->part_num = message_part_to_idx(key->part);
+	return TRUE;
+}
+
+static void
+fts_backend_lucene_update_unset_build_key(struct fts_backend_update_context *_ctx)
+{
+	struct lucene_fts_backend_update_context *ctx =
+		(struct lucene_fts_backend_update_context *)_ctx;
+
+	ctx->uid = 0;
+	ctx->part_num = 0;
+	i_free_and_null(ctx->hdr_name);
+}
+
+static int
+fts_backend_lucene_update_build_more(struct fts_backend_update_context *_ctx,
+				     const unsigned char *data, size_t size)
+{
+	struct lucene_fts_backend_update_context *ctx =
+		(struct lucene_fts_backend_update_context *)_ctx;
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_ctx->backend;
+	int ret;
+
+	i_assert(ctx->uid != 0);
+
+	if (_ctx->failed)
+		return -1;
+
+	T_BEGIN {
+		ret = lucene_index_build_more(backend->index, ctx->uid,
+					      ctx->part_num, data, size,
+					      ctx->hdr_name);
+	} T_END;
+	return ret;
+}
+
+static int
+fts_backend_lucene_refresh(struct fts_backend *_backend)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+
+	if (backend->index != NULL)
+		lucene_index_close(backend->index);
+	return 0;
+}
+
+static int fts_backend_lucene_rescan(struct fts_backend *_backend)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+
+	if (lucene_index_rescan(backend->index) < 0)
+		return -1;
+	return lucene_index_optimize(backend->index);
+}
+
+static int fts_backend_lucene_optimize(struct fts_backend *_backend)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+	int ret;
+
+	ret = lucene_index_expunge_from_log(backend->index,
+					    backend->expunge_log);
+	if (ret == 0) {
+		/* log was corrupted, need to rescan */
+		ret = lucene_index_rescan(backend->index);
+	}
+	if (ret >= 0)
+		ret = lucene_index_optimize(backend->index);
+	return ret;
+}
+
+static int
+fts_backend_lucene_lookup(struct fts_backend *_backend, struct mailbox *box,
+			  struct mail_search_arg *args,
+			  enum fts_lookup_flags flags,
+			  struct fts_result *result)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+	int ret;
+
+	if (fts_backend_select(backend, box) < 0)
+		return -1;
+	T_BEGIN {
+		ret = lucene_index_lookup(backend->index, args, flags, result);
+	} T_END;
+	return ret;
+}
+
+/* a char* hash function from ASU -- from glib */
+static unsigned int wstr_hash(const wchar_t *s)
+{
+	unsigned int g, h = 0;
+
+	while (*s != '\0') {
+		h = (h << 4) + *s;
+		if ((g = h & 0xf0000000UL) != 0) {
+			h = h ^ (g >> 24);
+			h = h ^ g;
+		}
+		s++;
+	}
+
+	return h;
+}
+
+static int
+mailboxes_get_guids(struct mailbox *const boxes[],
+		    HASH_TABLE_TYPE(wguid_result) guids,
+		    struct fts_multi_result *result)
+{
+	ARRAY(struct fts_result) box_results;
+	struct fts_result *box_result;
+	const char *guid;
+	wchar_t *guid_dup;
+	unsigned int i, j;
+
+	p_array_init(&box_results, result->pool, 32);
+	/* first create the box_results - we'll be using pointers to them
+	   later on and appending to the array changes the pointers */
+	for (i = 0; boxes[i] != NULL; i++) {
+		box_result = array_append_space(&box_results);
+		box_result->box = boxes[i];
+	}
+	for (i = 0; boxes[i] != NULL; i++) {
+		if (fts_mailbox_get_guid(boxes[i], &guid) < 0)
+			return -1;
+
+		i_assert(strlen(guid) == MAILBOX_GUID_HEX_LENGTH);
+		guid_dup = t_new(wchar_t, MAILBOX_GUID_HEX_LENGTH + 1);
+		for (j = 0; j < MAILBOX_GUID_HEX_LENGTH; j++)
+			guid_dup[j] = guid[j];
+
+		box_result = array_idx_modifiable(&box_results, i);
+		hash_table_insert(guids, guid_dup, box_result);
+	}
+
+	array_append_zero(&box_results);
+	result->box_results = array_front_modifiable(&box_results);
+	return 0;
+}
+
+static int
+fts_backend_lucene_lookup_multi(struct fts_backend *_backend,
+				struct mailbox *const boxes[],
+				struct mail_search_arg *args,
+				enum fts_lookup_flags flags,
+				struct fts_multi_result *result)
+{
+	struct lucene_fts_backend *backend =
+		(struct lucene_fts_backend *)_backend;
+	int ret;
+
+	T_BEGIN {
+		HASH_TABLE_TYPE(wguid_result) guids;
+
+		hash_table_create(&guids, default_pool, 0, wstr_hash, wcscmp);
+		ret = mailboxes_get_guids(boxes, guids, result);
+		if (ret == 0) {
+			ret = lucene_index_lookup_multi(backend->index,
+							guids, args, flags,
+							result);
+		}
+		hash_table_destroy(&guids);
+	} T_END;
+	return ret;
+}
+
+static void fts_backend_lucene_lookup_done(struct fts_backend *_backend)
+{
+	/* the next refresh is going to close the index anyway, so we might as
+	   well do it now */
+	(void)fts_backend_lucene_refresh(_backend);
+}
+
+struct fts_backend fts_backend_lucene = {
+	.name = "lucene",
+	.flags = FTS_BACKEND_FLAG_BUILD_FULL_WORDS |
+		FTS_BACKEND_FLAG_FUZZY_SEARCH,
+
+	{
+		fts_backend_lucene_alloc,
+		fts_backend_lucene_init,
+		fts_backend_lucene_deinit,
+		fts_backend_lucene_get_last_uid,
+		fts_backend_lucene_update_init,
+		fts_backend_lucene_update_deinit,
+		fts_backend_lucene_update_set_mailbox,
+		fts_backend_lucene_update_expunge,
+		fts_backend_lucene_update_set_build_key,
+		fts_backend_lucene_update_unset_build_key,
+		fts_backend_lucene_update_build_more,
+		fts_backend_lucene_refresh,
+		fts_backend_lucene_rescan,
+		fts_backend_lucene_optimize,
+		fts_backend_default_can_lookup,
+		fts_backend_lucene_lookup,
+		fts_backend_lucene_lookup_multi,
+		fts_backend_lucene_lookup_done
+	}
+};
diff --git a/src/plugins/fts-lucene/fts-lucene-plugin.c b/src/plugins/fts-lucene/fts-lucene-plugin.c
new file mode 100644
index 0000000..7c58fa7
--- /dev/null
+++ b/src/plugins/fts-lucene/fts-lucene-plugin.c
@@ -0,0 +1,146 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "crc32.h"
+#include "mail-storage-hooks.h"
+#include "lucene-wrapper.h"
+#include "fts-user.h"
+#include "fts-lucene-plugin.h"
+
+const char *fts_lucene_plugin_version = DOVECOT_ABI_VERSION;
+
+struct fts_lucene_user_module fts_lucene_user_module =
+	MODULE_CONTEXT_INIT(&mail_user_module_register);
+
+static int
+fts_lucene_plugin_init_settings(struct mail_user *user,
+				struct fts_lucene_settings *set,
+				const char *str)
+{
+	const char *const *tmp;
+
+	for (tmp = t_strsplit_spaces(str, " "); *tmp != NULL; tmp++) {
+		if (str_begins(*tmp, "default_language=")) {
+			set->default_language =
+				p_strdup(user->pool, *tmp + 17);
+		} else if (str_begins(*tmp, "textcat_conf=")) {
+			set->textcat_conf = p_strdup(user->pool, *tmp + 13);
+		} else if (str_begins(*tmp, "textcat_dir=")) {
+			set->textcat_dir = p_strdup(user->pool, *tmp + 12);
+		} else if (str_begins(*tmp, "whitespace_chars=")) {
+			set->whitespace_chars = p_strdup(user->pool, *tmp + 17);
+		} else if (strcmp(*tmp, "normalize") == 0) {
+			set->normalize = TRUE;
+		} else if (strcmp(*tmp, "no_snowball") == 0) {
+			set->no_snowball = TRUE;
+		} else if (strcmp(*tmp, "mime_parts") == 0) {
+			set->mime_parts = TRUE;
+		} else if (strcmp(*tmp, "use_libfts") == 0) {
+			set->use_libfts = TRUE;
+		} else {
+			i_error("fts_lucene: Invalid setting: %s", *tmp);
+			return -1;
+		}
+	}
+	if (set->textcat_conf != NULL && set->textcat_dir == NULL) {
+		i_error("fts_lucene: textcat_conf set, but textcat_dir unset");
+		return -1;
+	}
+	if (set->textcat_conf == NULL && set->textcat_dir != NULL) {
+		i_error("fts_lucene: textcat_dir set, but textcat_conf unset");
+		return -1;
+	}
+	if (set->whitespace_chars == NULL)
+		set->whitespace_chars = "";
+#ifndef HAVE_FTS_STEMMER
+	if (set->default_language != NULL) {
+		i_error("fts_lucene: default_language set, "
+			"but Dovecot built without stemmer support");
+		return -1;
+	}
+#else
+	if (set->default_language == NULL)
+		set->default_language = "english";
+#endif
+#ifndef HAVE_FTS_TEXTCAT
+	if (set->textcat_conf != NULL) {
+		i_error("fts_lucene: textcat_dir set, "
+			"but Dovecot built without textcat support");
+		return -1;
+	}
+#endif
+	return 0;
+}
+
+uint32_t fts_lucene_settings_checksum(const struct fts_lucene_settings *set)
+{
+	uint32_t crc;
+
+	if (set->use_libfts)
+		return crc32_str("l");
+
+	/* checksum is always different when compiling with/without stemmer */
+	crc = set->default_language == NULL ? 0 :
+		crc32_str(set->default_language);
+	crc = crc32_str_more(crc, set->whitespace_chars);
+	if (set->normalize)
+		crc = crc32_str_more(crc, "n");
+	if (set->no_snowball)
+		crc = crc32_str_more(crc, "s");
+	/* don't include mime_parts here, since changing it doesn't
+	   necessarily need the index to be rebuilt */
+	return crc;
+}
+
+static void fts_lucene_mail_user_deinit(struct mail_user *user)
+{
+	struct fts_lucene_user *fuser = FTS_LUCENE_USER_CONTEXT_REQUIRE(user);
+
+	fts_mail_user_deinit(user);
+	fuser->module_ctx.super.deinit(user);
+}
+
+static void fts_lucene_mail_user_created(struct mail_user *user)
+{
+	struct mail_user_vfuncs *v = user->vlast;
+	struct fts_lucene_user *fuser;
+	const char *env, *error;
+
+	fuser = p_new(user->pool, struct fts_lucene_user, 1);
+	env = mail_user_plugin_getenv(user, "fts_lucene");
+	if (env == NULL)
+		env = "";
+
+	if (fts_lucene_plugin_init_settings(user, &fuser->set, env) < 0) {
+		/* invalid settings, disabling */
+		return;
+	}
+	if (fts_mail_user_init(user, fuser->set.use_libfts, &error) < 0) {
+		i_error("fts_lucene: %s", error);
+		return;
+	}
+
+	fuser->module_ctx.super = *v;
+	user->vlast = &fuser->module_ctx.super;
+	v->deinit = fts_lucene_mail_user_deinit;
+	MODULE_CONTEXT_SET(user, fts_lucene_user_module, fuser);
+}
+
+static struct mail_storage_hooks fts_lucene_mail_storage_hooks = {
+	.mail_user_created = fts_lucene_mail_user_created
+};
+
+void fts_lucene_plugin_init(struct module *module ATTR_UNUSED)
+{
+	fts_backend_register(&fts_backend_lucene);
+	mail_storage_hooks_add(module, &fts_lucene_mail_storage_hooks);
+}
+
+void fts_lucene_plugin_deinit(void)
+{
+	fts_backend_unregister(fts_backend_lucene.name);
+	mail_storage_hooks_remove(&fts_lucene_mail_storage_hooks);
+	lucene_shutdown();
+}
+
+const char *fts_lucene_plugin_dependencies[] = { "fts", NULL };
diff --git a/src/plugins/fts-lucene/fts-lucene-plugin.h b/src/plugins/fts-lucene/fts-lucene-plugin.h
new file mode 100644
index 0000000..69440fb
--- /dev/null
+++ b/src/plugins/fts-lucene/fts-lucene-plugin.h
@@ -0,0 +1,36 @@
+#ifndef FTS_LUCENE_PLUGIN_H
+#define FTS_LUCENE_PLUGIN_H
+
+#include "module-context.h"
+#include "mail-user.h"
+#include "fts-api-private.h"
+
+#define FTS_LUCENE_USER_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, fts_lucene_user_module)
+#define FTS_LUCENE_USER_CONTEXT_REQUIRE(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, fts_lucene_user_module)
+
+struct fts_lucene_settings {
+	const char *default_language;
+	const char *textcat_conf, *textcat_dir;
+	const char *whitespace_chars;
+	bool normalize;
+	bool no_snowball;
+	bool mime_parts;
+	bool use_libfts;
+};
+
+struct fts_lucene_user {
+	union mail_user_module_context module_ctx;
+	struct fts_lucene_settings set;
+};
+
+extern struct fts_backend fts_backend_lucene;
+extern MODULE_CONTEXT_DEFINE(fts_lucene_user_module, &mail_user_module_register);
+
+uint32_t fts_lucene_settings_checksum(const struct fts_lucene_settings *set);
+
+void fts_lucene_plugin_init(struct module *module);
+void fts_lucene_plugin_deinit(void);
+
+#endif
diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
new file mode 100644
index 0000000..7446693
--- /dev/null
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -0,0 +1,1639 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+extern "C" {
+#include "lib.h"
+#include "array.h"
+#include "unichar.h"
+#include "hash.h"
+#include "hex-binary.h"
+#include "ioloop.h"
+#include "unlink-directory.h"
+#include "ioloop.h"
+#include "mail-index.h"
+#include "mail-search.h"
+#include "mail-namespace.h"
+#include "mailbox-list-private.h"
+#include "mail-storage.h"
+#include "fts-expunge-log.h"
+#include "fts-lucene-plugin.h"
+#include "lucene-wrapper.h"
+
+#include <sys/stat.h>
+#ifdef HAVE_LIBEXTTEXTCAT_TEXTCAT_H
+#  include <libexttextcat/textcat.h>
+#elif defined (HAVE_LIBTEXTCAT_TEXTCAT_H)
+#  include <libtextcat/textcat.h>
+#elif defined (HAVE_FTS_TEXTCAT)
+#  include <textcat.h>
+#endif
+};
+#include <CLucene.h>
+#include <CLucene/util/CLStreams.h>
+#include <CLucene/search/MultiPhraseQuery.h>
+#include "SnowballAnalyzer.h"
+
+/* Lucene's default is 10000. Use it here also.. */
+#define MAX_TERMS_PER_DOCUMENT 10000
+#define FTS_LUCENE_MAX_SEARCH_TERMS 1000
+
+#define LUCENE_LOCK_OVERRIDE_SECS 60
+#define LUCENE_INDEX_CLOSE_TIMEOUT_MSECS (120*1000)
+
+using namespace lucene::document;
+using namespace lucene::index;
+using namespace lucene::search;
+using namespace lucene::queryParser;
+using namespace lucene::analysis;
+using namespace lucene::analysis;
+using namespace lucene::util;
+
+struct lucene_query {
+	Query *query;
+	BooleanClause::Occur occur;
+};
+ARRAY_DEFINE_TYPE(lucene_query, struct lucene_query);
+
+struct lucene_analyzer {
+	char *lang;
+	Analyzer *analyzer;
+};
+
+struct lucene_index {
+	char *path;
+	struct mailbox_list *list;
+	struct fts_lucene_settings set;
+	normalizer_func_t *normalizer;
+
+	wchar_t mailbox_guid[MAILBOX_GUID_HEX_LENGTH + 1];
+
+	IndexReader *reader;
+	IndexWriter *writer;
+	IndexSearcher *searcher;
+	struct timeout *to_close;
+
+	buffer_t *normalizer_buf;
+	Analyzer *default_analyzer, *cur_analyzer;
+	ARRAY(struct lucene_analyzer) analyzers;
+
+	Document *doc;
+	uint32_t prev_uid, prev_part_idx;
+	bool no_analyzer;
+};
+
+struct rescan_context {
+	struct lucene_index *index;
+
+	struct mailbox *box;
+	guid_128_t box_guid;
+	int box_ret;
+
+	pool_t pool;
+	HASH_TABLE(uint8_t *, uint8_t *) seen_mailbox_guids;
+
+	ARRAY_TYPE(seq_range) uids;
+	struct seq_range_iter uids_iter;
+	unsigned int uids_iter_n;
+
+	uint32_t last_existing_uid;
+	bool warned;
+};
+
+static void *textcat = NULL;
+#ifdef HAVE_FTS_TEXTCAT
+static bool textcat_broken = FALSE;
+#endif
+static int textcat_refcount = 0;
+
+static void lucene_handle_error(struct lucene_index *index, CLuceneError &err,
+				const char *msg);
+static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
+					  struct rescan_context *rescan_ctx);
+
+struct lucene_index *lucene_index_init(const char *path,
+				       struct mailbox_list *list,
+				       const struct fts_lucene_settings *set)
+{
+	struct lucene_index *index;
+
+	index = i_new(struct lucene_index, 1);
+	index->path = i_strdup(path);
+	index->list = list;
+	if (set != NULL) {
+		index->set = *set;
+		index->normalizer = !set->normalize ? NULL :
+			mailbox_list_get_namespace(list)->user->default_normalizer;
+	} else {
+		/* this is valid only for doveadm dump, so it doesn't matter */
+		index->set.default_language = "";
+	}
+	if (index->set.use_libfts) {
+		index->default_analyzer = _CLNEW KeywordAnalyzer();
+	} else
+#ifdef HAVE_FTS_STEMMER
+	if (set == NULL || !set->no_snowball) {
+		index->default_analyzer =
+			_CLNEW snowball::SnowballAnalyzer(index->normalizer,
+							  index->set.default_language);
+	} else
+#endif
+	{
+		index->default_analyzer = _CLNEW standard::StandardAnalyzer();
+		if (index->normalizer != NULL) {
+			index->normalizer_buf =
+				buffer_create_dynamic(default_pool, 1024);
+		}
+	}
+
+	i_array_init(&index->analyzers, 32);
+	textcat_refcount++;
+
+	return index;
+}
+
+void lucene_index_close(struct lucene_index *index)
+{
+	timeout_remove(&index->to_close);
+
+	_CLDELETE(index->searcher);
+	if (index->writer != NULL) {
+		try {
+			index->writer->close();
+		} catch (CLuceneError &err) {
+			lucene_handle_error(index, err, "IndexWriter::close");
+		}
+		_CLDELETE(index->writer);
+	}
+	if (index->reader != NULL) {
+		try {
+			index->reader->close();
+		} catch (CLuceneError &err) {
+			lucene_handle_error(index, err, "IndexReader::close");
+		}
+		_CLDELETE(index->reader);
+	}
+}
+
+void lucene_index_deinit(struct lucene_index *index)
+{
+	struct lucene_analyzer *a;
+
+	lucene_index_close(index);
+	array_foreach_modifiable(&index->analyzers, a) {
+		i_free(a->lang);
+		_CLDELETE(a->analyzer);
+	}
+	array_free(&index->analyzers);
+	if (--textcat_refcount == 0 && textcat != NULL) {
+#ifdef HAVE_FTS_TEXTCAT
+		textcat_Done(textcat);
+#endif
+		textcat = NULL;
+	}
+	_CLDELETE(index->default_analyzer);
+	if (index->normalizer_buf != NULL)
+		buffer_free(&index->normalizer_buf);
+	i_free(index->path);
+	i_free(index);
+}
+
+static void lucene_data_translate(struct lucene_index *index,
+				  wchar_t *data, unsigned int len)
+{
+	const char *whitespace_chars = index->set.whitespace_chars;
+	unsigned int i;
+
+	if (*whitespace_chars == '\0' || index->set.use_libfts)
+		return;
+
+	for (i = 0; i < len; i++) {
+		if (strchr(whitespace_chars, data[i]) != NULL)
+			data[i] = ' ';
+	}
+}
+
+void lucene_utf8_n_to_tchar(const unsigned char *src, size_t srcsize,
+			    wchar_t *dest, size_t destsize)
+{
+	ARRAY_TYPE(unichars) dest_arr;
+	buffer_t buf = { { 0, 0 } };
+
+	i_assert(sizeof(wchar_t) == sizeof(unichar_t));
+
+	buffer_create_from_data(&buf, dest, sizeof(wchar_t) * destsize);
+	array_create_from_buffer(&dest_arr, &buf, sizeof(wchar_t));
+	if (uni_utf8_to_ucs4_n(src, srcsize, &dest_arr) < 0)
+		i_unreached();
+	i_assert(array_count(&dest_arr)+1 == destsize);
+	dest[destsize-1] = 0;
+}
+
+static const wchar_t *
+t_lucene_utf8_to_tchar(struct lucene_index *index, const char *str)
+{
+	ARRAY_TYPE(unichars) dest_arr;
+	const unichar_t *chars;
+	wchar_t *ret;
+	unsigned int len;
+
+	i_assert(sizeof(wchar_t) == sizeof(unichar_t));
+
+	t_array_init(&dest_arr, strlen(str) + 1);
+	if (uni_utf8_to_ucs4(str, &dest_arr) < 0)
+		i_unreached();
+	(void)array_append_space(&dest_arr);
+
+	chars = array_get_modifiable(&dest_arr, &len);
+	ret = (wchar_t *)chars;
+	lucene_data_translate(index, ret, len - 1);
+	return ret;
+}
+
+void lucene_index_select_mailbox(struct lucene_index *index,
+				 const wchar_t guid[MAILBOX_GUID_HEX_LENGTH])
+{
+	memcpy(index->mailbox_guid, guid,
+	       MAILBOX_GUID_HEX_LENGTH * sizeof(wchar_t));
+	index->mailbox_guid[MAILBOX_GUID_HEX_LENGTH] = '\0';
+}
+
+void lucene_index_unselect_mailbox(struct lucene_index *index)
+{
+	memset(index->mailbox_guid, 0, sizeof(index->mailbox_guid));
+}
+
+static void lucene_handle_error(struct lucene_index *index, CLuceneError &err,
+				const char *msg)
+{
+	const char *error, *what = err.what();
+
+	i_error("lucene index %s: %s failed (#%d): %s",
+		index->path, msg, err.number(), what);
+
+	if (index->list != NULL &&
+	    (err.number() == CL_ERR_CorruptIndex ||
+	     err.number() == CL_ERR_IO)) {
+		/* delete corrupted index. most IO errors are also about
+		   missing files and other such corruption.. */
+		if (unlink_directory(index->path, (enum unlink_directory_flags)0, &error) < 0)
+			i_error("unlink_directory(%s) failed: %s", index->path, error);
+		rescan_clear_unseen_mailboxes(index, NULL);
+	}
+}
+
+static int lucene_index_open(struct lucene_index *index)
+{
+	if (index->reader != NULL) {
+		i_assert(index->to_close != NULL);
+		timeout_reset(index->to_close);
+		return 1;
+	}
+
+	if (!IndexReader::indexExists(index->path))
+		return 0;
+
+	try {
+		index->reader = IndexReader::open(index->path);
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "IndexReader::open()");
+		return -1;
+	}
+	i_assert(index->to_close == NULL);
+	index->to_close = timeout_add(LUCENE_INDEX_CLOSE_TIMEOUT_MSECS,
+				      lucene_index_close, index);
+	return 1;
+}
+
+static int lucene_index_open_search(struct lucene_index *index)
+{
+	int ret;
+
+	if (index->searcher != NULL)
+		return 1;
+
+	if ((ret = lucene_index_open(index)) <= 0)
+		return ret;
+
+	index->searcher = _CLNEW IndexSearcher(index->reader);
+	return 1;
+}
+
+static int
+lucene_doc_get_uid(struct lucene_index *index, Document *doc, uint32_t *uid_r)
+{
+	Field *field = doc->getField(_T("uid"));
+	const TCHAR *uid = field == NULL ? NULL : field->stringValue();
+	if (uid == NULL) {
+		i_error("lucene: Corrupted FTS index %s: No UID for document",
+			index->path);
+		return -1;
+	}
+
+	uint32_t num = 0;
+	while (*uid != 0) {
+		num = num*10 + (*uid - '0');
+		uid++;
+	}
+	*uid_r = num;
+	return 0;
+}
+
+static uint32_t
+lucene_doc_get_part(struct lucene_index *index, Document *doc)
+{
+	Field *field = doc->getField(_T("part"));
+	const TCHAR *part = field == NULL ? NULL : field->stringValue();
+	if (part == NULL)
+		return 0;
+
+	uint32_t num = 0;
+	while (*part != 0) {
+		num = num*10 + (*part - '0');
+		part++;
+	}
+	return num;
+}
+
+int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r)
+{
+	int ret = 0;
+
+	*last_uid_r = 0;
+
+	if ((ret = lucene_index_open_search(index)) <= 0)
+		return ret;
+
+	Term mailbox_term(_T("box"), index->mailbox_guid);
+	TermQuery query(&mailbox_term);
+
+	uint32_t last_uid = 0;
+	try {
+		Hits *hits = index->searcher->search(&query);
+
+		for (size_t  i = 0; i < hits->length(); i++) {
+			uint32_t uid;
+
+			if (lucene_doc_get_uid(index, &hits->doc(i),
+					       &uid) < 0) {
+				ret = -1;
+				break;
+			}
+
+			if (uid > last_uid)
+				last_uid = uid;
+		}
+		_CLDELETE(hits);
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "last_uid search");
+		ret = -1;
+	}
+	*last_uid_r = last_uid;
+	return ret;
+}
+
+int lucene_index_get_doc_count(struct lucene_index *index, uint32_t *count_r)
+{
+	int ret;
+
+	if (index->reader == NULL) {
+		lucene_index_close(index);
+		if ((ret = lucene_index_open(index)) < 0)
+			return -1;
+		if (ret == 0) {
+			*count_r = 0;
+			return 0;
+		}
+	}
+	*count_r = index->reader->numDocs();
+	return 0;
+}
+
+static int lucene_settings_check(struct lucene_index *index)
+{
+	uint32_t set_checksum;
+	const char *error;
+	int ret = 0;
+
+	set_checksum = fts_lucene_settings_checksum(&index->set);
+	ret = fts_index_have_compatible_settings(index->list, set_checksum);
+	if (ret != 0)
+		return ret;
+
+	i_warning("fts-lucene: Settings have changed, rebuilding index for mailbox");
+
+	/* settings changed, rebuild index */
+	if (unlink_directory(index->path, (enum unlink_directory_flags)0, &error) < 0) {
+		i_error("unlink_directory(%s) failed: %s", index->path, error);
+		ret = -1;
+	} else {
+		rescan_clear_unseen_mailboxes(index, NULL);
+	}
+	return ret;
+}
+
+int lucene_index_build_init(struct lucene_index *index)
+{
+	const char *lock_path;
+	struct stat st;
+
+	lucene_index_close(index);
+
+	lock_path = t_strdup_printf("%s/write.lock", index->path);
+	if (stat(lock_path, &st) == 0 &&
+	    st.st_mtime < time(NULL) - LUCENE_LOCK_OVERRIDE_SECS) {
+		if (unlink(lock_path) < 0)
+			i_error("unlink(%s) failed: %m", lock_path);
+	}
+
+	if (lucene_settings_check(index) < 0)
+		return -1;
+
+	bool exists = IndexReader::indexExists(index->path);
+	try {
+		index->writer = _CLNEW IndexWriter(index->path,
+						   index->default_analyzer,
+						   !exists);
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "IndexWriter()");
+		return -1;
+	}
+	index->writer->setMaxFieldLength(MAX_TERMS_PER_DOCUMENT);
+	return 0;
+}
+
+#ifdef HAVE_FTS_TEXTCAT
+static Analyzer *get_analyzer(struct lucene_index *index, const char *lang)
+{
+	normalizer_func_t *normalizer = index->normalizer;
+	const struct lucene_analyzer *a;
+	struct lucene_analyzer new_analyzer;
+	Analyzer *analyzer;
+
+	array_foreach(&index->analyzers, a) {
+		if (strcmp(a->lang, lang) == 0)
+			return a->analyzer;
+	}
+
+	memset(&new_analyzer, 0, sizeof(new_analyzer));
+	new_analyzer.lang = i_strdup(lang);
+	new_analyzer.analyzer =
+		_CLNEW snowball::SnowballAnalyzer(normalizer, lang);
+	array_append_i(&index->analyzers.arr, &new_analyzer, 1);
+	return new_analyzer.analyzer;
+}
+
+static void *textcat_init(struct lucene_index *index)
+{
+	const char *textcat_dir = index->set.textcat_dir;
+	unsigned int len;
+
+	if (textcat_dir == NULL)
+		return NULL;
+
+	/* textcat really wants the '/' suffix */
+	len = strlen(textcat_dir);
+	if (len > 0 && textcat_dir[len-1] != '/')
+		textcat_dir = t_strconcat(textcat_dir, "/", NULL);
+
+	return special_textcat_Init(index->set.textcat_conf, textcat_dir);
+}
+
+static Analyzer *
+guess_analyzer(struct lucene_index *index, const void *data, size_t size)
+{
+	const char *lang;
+
+	if (textcat_broken)
+		return NULL;
+
+	if (textcat == NULL) {
+		textcat = textcat_init(index);
+		if (textcat == NULL) {
+			textcat_broken = TRUE;
+			return NULL;
+		}
+	}
+
+	/* try to guess the language */
+	lang = textcat_Classify(textcat, (const char *)data,
+				I_MIN(size, 500));
+	const char *p = strchr(lang, ']');
+	if (lang[0] != '[' || p == NULL)
+		return NULL;
+	lang = t_strdup_until(lang+1, p);
+	if (strcmp(lang, index->set.default_language) == 0)
+		return index->default_analyzer;
+
+	return get_analyzer(index, lang);
+}
+#else
+static Analyzer *
+guess_analyzer(struct lucene_index *index ATTR_UNUSED,
+	       const void *data ATTR_UNUSED, size_t size ATTR_UNUSED)
+{
+	return NULL;
+}
+#endif
+
+static int lucene_index_build_flush(struct lucene_index *index)
+{
+	int ret = 0;
+
+	if (index->doc == NULL)
+		return 0;
+
+	try {
+		CL_NS(analysis)::Analyzer *analyzer = NULL;
+
+		if (!index->set.use_libfts) {
+			analyzer = index->cur_analyzer != NULL ?
+				index->cur_analyzer : index->default_analyzer;
+		}
+		index->writer->addDocument(index->doc, analyzer);
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "IndexWriter::addDocument()");
+		ret = -1;
+	}
+
+	_CLDELETE(index->doc);
+	index->doc = NULL;
+	index->cur_analyzer = NULL;
+	return ret;
+}
+
+int lucene_index_build_more(struct lucene_index *index, uint32_t uid,
+			    uint32_t part_idx, const unsigned char *data,
+			    size_t size, const char *hdr_name)
+{
+	wchar_t id[MAX_INT_STRLEN];
+	size_t namesize, datasize;
+
+	if (uid != index->prev_uid || part_idx != index->prev_part_idx) {
+		if (lucene_index_build_flush(index) < 0)
+			return -1;
+		index->prev_uid = uid;
+		index->prev_part_idx = part_idx;
+
+		index->doc = _CLNEW Document();
+		swprintf(id, N_ELEMENTS(id), L"%u", uid);
+		index->doc->add(*_CLNEW Field(_T("uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
+		if (part_idx != 0) {
+			swprintf(id, N_ELEMENTS(id), L"%u", part_idx);
+			index->doc->add(*_CLNEW Field(_T("part"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
+		}
+		index->doc->add(*_CLNEW Field(_T("box"), index->mailbox_guid, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
+	}
+
+	if (index->normalizer_buf != NULL && !index->set.use_libfts) {
+		buffer_set_used_size(index->normalizer_buf, 0);
+		index->normalizer(data, size, index->normalizer_buf);
+		data = (const unsigned char *)index->normalizer_buf->data;
+		size = index->normalizer_buf->used;
+	}
+
+	datasize = uni_utf8_strlen_n(data, size) + 1;
+	wchar_t *dest, *dest_free = NULL;
+	if (datasize < 4096)
+		dest = t_new(wchar_t, datasize);
+	else
+		dest = dest_free = i_new(wchar_t, datasize);
+	lucene_utf8_n_to_tchar(data, size, dest, datasize);
+	lucene_data_translate(index, dest, datasize-1);
+
+	int token_flag = index->set.use_libfts ?
+		Field::INDEX_UNTOKENIZED : Field::INDEX_TOKENIZED;
+	if (hdr_name != NULL) {
+		/* hdr_name should be ASCII, but don't break in case it isn't */
+		hdr_name = t_str_lcase(hdr_name);
+		namesize = uni_utf8_strlen(hdr_name) + 1;
+		wchar_t wname[namesize];
+		lucene_utf8_n_to_tchar((const unsigned char *)hdr_name,
+				       strlen(hdr_name), wname, namesize);
+		if (!index->set.use_libfts)
+			index->doc->add(*_CLNEW Field(_T("hdr"), wname, Field::STORE_NO | token_flag));
+		index->doc->add(*_CLNEW Field(_T("hdr"), dest, Field::STORE_NO | token_flag));
+
+		if (fts_header_want_indexed(hdr_name))
+			index->doc->add(*_CLNEW Field(wname, dest, Field::STORE_NO | token_flag));
+	} else if (size > 0) {
+		if (index->cur_analyzer == NULL && !index->set.use_libfts)
+			index->cur_analyzer = guess_analyzer(index, data, size);
+		index->doc->add(*_CLNEW Field(_T("body"), dest, Field::STORE_NO | token_flag));
+	}
+	i_free(dest_free);
+	return 0;
+}
+
+int lucene_index_build_deinit(struct lucene_index *index)
+{
+	int ret = 0;
+
+	if (index->prev_uid == 0) {
+		/* no changes. */
+		return 0;
+	}
+	index->prev_uid = 0;
+	index->prev_part_idx = 0;
+
+	if (index->writer == NULL) {
+		lucene_index_close(index);
+		return -1;
+	}
+
+	if (lucene_index_build_flush(index) < 0)
+		ret = -1;
+
+	try {
+		index->writer->close();
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "IndexWriter::close()");
+		ret = -1;
+	}
+
+	lucene_index_close(index);
+	return ret;
+}
+
+static int
+wcharguid_to_guid(guid_128_t dest, const wchar_t *src)
+{
+	buffer_t buf = { { 0, 0 } };
+	char src_chars[GUID_128_SIZE*2 + 1];
+	unsigned int i;
+
+	for (i = 0; i < sizeof(src_chars)-1; i++) {
+		if ((src[i] >= '0' && src[i] <= '9') ||
+		    (src[i] >= 'a' && src[i] <= 'f'))
+			src_chars[i] = src[i];
+		else
+			return -1;
+	}
+	if (src[i] != '\0')
+		return -1;
+	src_chars[i] = '\0';
+
+	buffer_create_from_data(&buf, dest, GUID_128_SIZE);
+	return hex_to_binary(src_chars, &buf);
+}
+
+static int
+rescan_get_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids)
+{
+	struct mailbox_status status;
+
+	if (mailbox_get_status(box, STATUS_MESSAGES, &status) < 0)
+		return -1;
+
+	if (status.messages > 0) T_BEGIN {
+		ARRAY_TYPE(seq_range) seqs;
+
+		t_array_init(&seqs, 2);
+		seq_range_array_add_range(&seqs, 1, status.messages);
+		mailbox_get_uid_range(box, &seqs, uids);
+	} T_END;
+	return 0;
+}
+
+static int rescan_finish(struct rescan_context *ctx)
+{
+	int ret;
+
+	ret = fts_index_set_last_uid(ctx->box, ctx->last_existing_uid);
+	mailbox_free(&ctx->box);
+	return ret;
+}
+
+static int
+fts_lucene_get_mailbox_guid(struct lucene_index *index, Document *doc,
+			    guid_128_t guid_r)
+{
+	Field *field = doc->getField(_T("box"));
+	const TCHAR *box_guid = field == NULL ? NULL : field->stringValue();
+	if (box_guid == NULL) {
+		i_error("lucene: Corrupted FTS index %s: No mailbox for document",
+			index->path);
+		return -1;
+	}
+
+	if (wcharguid_to_guid(guid_r, box_guid) < 0) {
+		i_error("lucene: Corrupted FTS index %s: "
+			"box field not in expected format", index->path);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+rescan_open_mailbox(struct rescan_context *ctx, Document *doc)
+{
+	guid_128_t guid, *guidp;
+	int ret;
+
+	if (fts_lucene_get_mailbox_guid(ctx->index, doc, guid) < 0)
+		return 0;
+
+	if (memcmp(guid, ctx->box_guid, sizeof(guid)) == 0) {
+		/* same as last one */
+		return ctx->box_ret;
+	}
+	memcpy(ctx->box_guid, guid, sizeof(ctx->box_guid));
+
+	guidp = p_new(ctx->pool, guid_128_t, 1);
+	memcpy(guidp, guid, sizeof(*guidp));
+	hash_table_insert(ctx->seen_mailbox_guids, guidp, guidp);
+
+	if (ctx->box != NULL)
+		rescan_finish(ctx);
+	ctx->box = mailbox_alloc_guid(ctx->index->list, guid,
+				      (enum mailbox_flags)0);
+	if (mailbox_open(ctx->box) < 0) {
+		enum mail_error error;
+		const char *errstr;
+
+		errstr = mailbox_get_last_internal_error(ctx->box, &error);
+		if (error == MAIL_ERROR_NOTFOUND)
+			ret = 0;
+		else {
+			i_error("lucene: Couldn't open mailbox %s: %s",
+				mailbox_get_vname(ctx->box), errstr);
+			ret = -1;
+		}
+		mailbox_free(&ctx->box);
+		ctx->box_ret = ret;
+		return ret;
+	}
+	if (mailbox_sync(ctx->box, (enum mailbox_sync_flags)0) < 0) {
+		i_error("lucene: Failed to sync mailbox %s: %s",
+			mailbox_get_vname(ctx->box),
+			mailbox_get_last_internal_error(ctx->box, NULL));
+		mailbox_free(&ctx->box);
+		ctx->box_ret = -1;
+		return -1;
+	}
+
+	array_clear(&ctx->uids);
+	rescan_get_uids(ctx->box, &ctx->uids);
+
+	ctx->warned = FALSE;
+	ctx->last_existing_uid = 0;
+	ctx->uids_iter_n = 0;
+	seq_range_array_iter_init(&ctx->uids_iter, &ctx->uids);
+
+	ctx->box_ret = 1;
+	return 1;
+}
+
+static int
+rescan_next(struct rescan_context *ctx, Document *doc)
+{
+	uint32_t lucene_uid, idx_uid;
+
+	if (lucene_doc_get_uid(ctx->index, doc, &lucene_uid) < 0)
+		return 0;
+
+	if (seq_range_array_iter_nth(&ctx->uids_iter, ctx->uids_iter_n,
+				     &idx_uid)) {
+		if (idx_uid == lucene_uid) {
+			ctx->uids_iter_n++;
+			ctx->last_existing_uid = idx_uid;
+			return 1;
+		}
+		if (idx_uid < lucene_uid) {
+			/* lucene is missing an UID from the middle. delete
+			   the rest of the messages from this mailbox and
+			   reindex. */
+			if (!ctx->warned) {
+				i_warning("lucene: Mailbox %s "
+					  "missing UIDs in the middle",
+					  mailbox_get_vname(ctx->box));
+				ctx->warned = TRUE;
+			}
+		} else {
+			/* UID has been expunged from index. delete from
+			   lucene as well. */
+		}
+		return 0;
+	} else {
+		/* the rest of the messages have been expunged from index */
+		return 0;
+	}
+}
+
+static void
+rescan_clear_unseen_mailbox(struct lucene_index *index,
+			    struct rescan_context *rescan_ctx,
+			    const char *vname,
+			    const struct fts_index_header *hdr)
+{
+	struct mailbox *box;
+	struct mailbox_metadata metadata;
+
+	box = mailbox_alloc(index->list, vname,
+			    (enum mailbox_flags)0);
+	if (mailbox_open(box) == 0 &&
+	    mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
+				 &metadata) == 0 &&
+	    (rescan_ctx == NULL ||
+	     hash_table_lookup(rescan_ctx->seen_mailbox_guids,
+			       metadata.guid) == NULL)) {
+		/* this mailbox had no records in lucene index.
+		   make sure its last indexed uid is 0 */
+		(void)fts_index_set_header(box, hdr);
+	}
+	mailbox_free(&box);
+}
+
+static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
+					  struct rescan_context *rescan_ctx)
+{
+	const enum mailbox_list_iter_flags iter_flags =
+		(enum mailbox_list_iter_flags)
+		(MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+		 MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
+	struct mailbox_list_iterate_context *iter;
+	const struct mailbox_info *info;
+	struct fts_index_header hdr;
+	struct mail_namespace *ns = index->list->ns;
+	const char *vname;
+
+	memset(&hdr, 0, sizeof(hdr));
+	hdr.settings_checksum = fts_lucene_settings_checksum(&index->set);
+
+	iter = mailbox_list_iter_init(index->list, "*", iter_flags);
+	while ((info = mailbox_list_iter_next(iter)) != NULL)
+		rescan_clear_unseen_mailbox(index, rescan_ctx, info->vname, &hdr);
+	(void)mailbox_list_iter_deinit(&iter);
+
+	if (ns->prefix_len > 0 &&
+	    ns->prefix[ns->prefix_len-1] == mail_namespace_get_sep(ns)) {
+		/* namespace prefix itself isn't returned by the listing */
+		vname = t_strndup(index->list->ns->prefix,
+				  index->list->ns->prefix_len-1);
+		rescan_clear_unseen_mailbox(index, rescan_ctx, vname, &hdr);
+	}
+}
+
+int lucene_index_rescan(struct lucene_index *index)
+{
+	static const TCHAR *sort_fields[] = { _T("box"), _T("uid"), NULL };
+	struct rescan_context ctx;
+	bool failed = false;
+	int ret;
+
+	i_assert(index->list != NULL);
+
+	if ((ret = lucene_index_open_search(index)) < 0)
+		return ret;
+
+	Term term(_T("box"), _T("*"));
+	WildcardQuery query(&term);
+	Sort sort(sort_fields);
+
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.index = index;
+	ctx.pool = pool_alloconly_create("guids", 1024);
+	hash_table_create(&ctx.seen_mailbox_guids, ctx.pool, 0,
+			  guid_128_hash, guid_128_cmp);
+	i_array_init(&ctx.uids, 128);
+
+	if (ret > 0) try {
+		Hits *hits = index->searcher->search(&query, &sort);
+
+		for (size_t i = 0; i < hits->length(); i++) {
+			ret = rescan_open_mailbox(&ctx, &hits->doc(i));
+			if (ret > 0)
+				ret = rescan_next(&ctx, &hits->doc(i));
+			if (ret < 0)
+				failed = true;
+			else if (ret == 0)
+				index->reader->deleteDocument(hits->id(i));
+		}
+		_CLDELETE(hits);
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "rescan search");
+		failed = true;
+	}
+	lucene_index_close(index);
+	if (ctx.box != NULL)
+		rescan_finish(&ctx);
+	array_free(&ctx.uids);
+
+	rescan_clear_unseen_mailboxes(index, &ctx);
+	hash_table_destroy(&ctx.seen_mailbox_guids);
+	pool_unref(&ctx.pool);
+	return failed ? -1 : 0;
+}
+
+static void guid128_to_wguid(const guid_128_t guid,
+			     wchar_t wguid_hex[MAILBOX_GUID_HEX_LENGTH + 1])
+{
+	buffer_t buf = { { 0, 0 } };
+	unsigned char guid_hex[MAILBOX_GUID_HEX_LENGTH];
+	unsigned int i;
+
+	buffer_create_from_data(&buf, guid_hex, MAILBOX_GUID_HEX_LENGTH);
+	binary_to_hex_append(&buf, guid, GUID_128_SIZE);
+	for (i = 0; i < MAILBOX_GUID_HEX_LENGTH; i++)
+		wguid_hex[i] = guid_hex[i];
+	wguid_hex[i] = '\0';
+}
+
+static bool
+lucene_index_add_uid_filter(BooleanQuery *query,
+			    const struct fts_expunge_log_read_record *rec)
+{
+	struct seq_range_iter iter;
+	wchar_t wuid[MAX_INT_STRLEN];
+	unsigned int n;
+	uint32_t uid;
+
+	/* RangeQuery and WildcardQuery work by enumerating through all terms
+	   that match them, and then adding TermQueries for them. So we can
+	   simply do the same directly, and if it looks like there are too
+	   many terms just go through everything. */
+
+	if (seq_range_count(&rec->uids) > FTS_LUCENE_MAX_SEARCH_TERMS)
+		return false;
+
+	seq_range_array_iter_init(&iter, &rec->uids); n = 0;
+	while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+		swprintf(wuid, N_ELEMENTS(wuid), L"%u", uid);
+
+		Term *term = _CLNEW Term(_T("uid"), wuid);
+		query->add(_CLNEW TermQuery(term), true, BooleanClause::SHOULD);
+		_CLDECDELETE(term);
+	}
+	return true;
+}
+
+static int
+lucene_index_expunge_record(struct lucene_index *index,
+			    const struct fts_expunge_log_read_record *rec)
+{
+	int ret;
+
+	if ((ret = lucene_index_open_search(index)) <= 0)
+		return ret;
+
+	BooleanQuery query;
+	BooleanQuery uids_query;
+
+	if (lucene_index_add_uid_filter(&uids_query, rec))
+		query.add(&uids_query, BooleanClause::MUST);
+
+	wchar_t wguid[MAILBOX_GUID_HEX_LENGTH + 1];
+	guid128_to_wguid(rec->mailbox_guid, wguid);
+	Term term(_T("box"), wguid);
+	TermQuery mailbox_query(&term);
+	query.add(&mailbox_query, BooleanClause::MUST);
+
+	try {
+		Hits *hits = index->searcher->search(&query);
+
+		for (size_t i = 0; i < hits->length(); i++) {
+			uint32_t uid;
+
+			if (lucene_doc_get_uid(index, &hits->doc(i),
+					       &uid) < 0 ||
+			    seq_range_exists(&rec->uids, uid))
+				index->reader->deleteDocument(hits->id(i));
+		}
+		_CLDELETE(hits);
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "expunge search");
+		ret = -1;
+	}
+	return ret < 0 ? -1 : 0;
+}
+
+int lucene_index_expunge_from_log(struct lucene_index *index,
+				  struct fts_expunge_log *log)
+{
+	struct fts_expunge_log_read_ctx *ctx;
+	const struct fts_expunge_log_read_record *rec;
+	int ret = 0, ret2;
+
+	ctx = fts_expunge_log_read_begin(log);
+	while ((rec = fts_expunge_log_read_next(ctx)) != NULL) {
+		if (lucene_index_expunge_record(index, rec) < 0) {
+			ret = -1;
+			break;
+		}
+	}
+
+	lucene_index_close(index);
+
+	ret2 = fts_expunge_log_read_end(&ctx);
+	if (ret < 0 || ret2 < 0)
+		return -1;
+	return ret2;
+}
+
+int lucene_index_optimize(struct lucene_index *index)
+{
+	int ret = 0;
+
+	if (!IndexReader::indexExists(index->path))
+		return 0;
+	if (IndexReader::isLocked(index->path))
+		IndexReader::unlock(index->path);
+
+	IndexWriter *writer = NULL;
+	try {
+		writer = _CLNEW IndexWriter(index->path, index->default_analyzer, false);
+		writer->optimize();
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "IndexWriter::optimize()");
+		ret = -1;
+	}
+	try {
+		writer->close();
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "IndexWriter::close()");
+		ret = -1;
+	}
+	if (writer != NULL)
+		_CLDELETE(writer);
+	return ret;
+}
+
+// Mostly copy&pasted from CLucene's QueryParser
+static Query* getFieldQuery(Analyzer *analyzer, const TCHAR* _field, const TCHAR* queryText, bool fuzzy) {
+  // Use the analyzer to get all the tokens, and then build a TermQuery,
+  // PhraseQuery, or nothing based on the term count
+
+  StringReader reader(queryText);
+  TokenStream* source = analyzer->tokenStream(_field, &reader);
+
+  CLVector<CL_NS(analysis)::Token*, Deletor::Object<CL_NS(analysis)::Token> > v;
+  CL_NS(analysis)::Token* t = NULL;
+  int32_t positionCount = 0;
+  bool severalTokensAtSamePosition = false;
+
+  while (true) {
+    t = _CLNEW Token();
+    try {
+      Token* _t = source->next(t);
+      if (_t == NULL) _CLDELETE(t);
+    }_CLCATCH_ERR(CL_ERR_IO, _CLLDELETE(source);_CLLDELETE(t);,{
+      t = NULL;
+    });
+    if (t == NULL)
+      break;
+    v.push_back(t);
+    if (t->getPositionIncrement() != 0)
+      positionCount += t->getPositionIncrement();
+    else
+      severalTokensAtSamePosition = true;
+  }
+  try {
+    source->close();
+  }
+  _CLCATCH_ERR_CLEANUP(CL_ERR_IO, {_CLLDELETE(source);_CLLDELETE(t);} ); /* cleanup */
+  _CLLDELETE(source);
+
+  if (v.size() == 0)
+    return NULL;
+  else if (v.size() == 1) {
+    Term* tm = _CLNEW Term(_field, v.at(0)->termBuffer());
+    Query* ret;
+    if (fuzzy)
+      ret = _CLNEW FuzzyQuery( tm );
+    else
+      ret = _CLNEW TermQuery( tm );
+    _CLDECDELETE(tm);
+    return ret;
+  } else {
+    if (severalTokensAtSamePosition) {
+      if (positionCount == 1) {
+        // no phrase query:
+        BooleanQuery* q = _CLNEW BooleanQuery(true);
+        for(size_t i=0; i<v.size(); i++ ){
+          Term* tm = _CLNEW Term(_field, v.at(i)->termBuffer());
+          q->add(_CLNEW TermQuery(tm), true, BooleanClause::SHOULD);
+          _CLDECDELETE(tm);
+        }
+        return q;
+      }else {
+		    MultiPhraseQuery* mpq = _CLNEW MultiPhraseQuery();
+		    CLArrayList<Term*> multiTerms;
+		    int32_t position = -1;
+		    for (size_t i = 0; i < v.size(); i++) {
+			    t = v.at(i);
+			    if (t->getPositionIncrement() > 0 && multiTerms.size() > 0) {
+            ValueArray<Term*> termsArray(multiTerms.size());
+            multiTerms.toArray(termsArray.values);
+	    mpq->add(&termsArray,position);
+				    multiTerms.clear();
+			    }
+			    position += t->getPositionIncrement();
+			    multiTerms.push_back(_CLNEW Term(_field, t->termBuffer()));
+		    }
+        ValueArray<Term*> termsArray(multiTerms.size());
+        multiTerms.toArray(termsArray.values);
+	mpq->add(&termsArray,position);
+		    return mpq;
+      }
+    }else {
+      PhraseQuery* pq = _CLNEW PhraseQuery();
+      int32_t position = -1;
+
+      for (size_t i = 0; i < v.size(); i++) {
+        t = v.at(i);
+        Term* tm = _CLNEW Term(_field, t->termBuffer());
+	position += t->getPositionIncrement();
+	pq->add(tm,position);
+        _CLDECDELETE(tm);
+      }
+      return pq;
+    }
+  }
+}
+
+static Query *
+lucene_get_query_str(struct lucene_index *index,
+		     const TCHAR *key, const char *str, bool fuzzy)
+{
+	const TCHAR *wvalue;
+	Analyzer *analyzer;
+
+	if (index->set.use_libfts) {
+		const wchar_t *wstr = t_lucene_utf8_to_tchar(index, str);
+		Term* tm = _CLNEW Term(key, wstr);
+		Query* ret;
+		if (fuzzy)
+			ret = _CLNEW FuzzyQuery( tm );
+		else
+			ret = _CLNEW TermQuery( tm );
+		_CLDECDELETE(tm);
+		return ret;
+	}
+
+	if (index->normalizer_buf != NULL) {
+		buffer_set_used_size(index->normalizer_buf, 0);
+		index->normalizer(str, strlen(str), index->normalizer_buf);
+		buffer_append_c(index->normalizer_buf, '\0');
+		str = (const char *)index->normalizer_buf->data;
+	}
+
+	wvalue = t_lucene_utf8_to_tchar(index, str);
+	analyzer = guess_analyzer(index, str, strlen(str));
+	if (analyzer == NULL) {
+		analyzer = index->default_analyzer;
+		i_assert(analyzer != NULL);
+	}
+
+	return getFieldQuery(analyzer, key, wvalue, fuzzy);
+}
+
+static Query *
+lucene_get_query(struct lucene_index *index,
+		 const TCHAR *key, const struct mail_search_arg *arg)
+{
+	return lucene_get_query_str(index, key, arg->value.str, arg->fuzzy);
+}
+
+static bool
+lucene_add_definite_query(struct lucene_index *index,
+			  ARRAY_TYPE(lucene_query) &queries,
+			  struct mail_search_arg *arg,
+			  enum fts_lookup_flags flags)
+{
+	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
+	Query *q;
+
+	if (arg->no_fts)
+		return false;
+
+	if (arg->match_not && !and_args) {
+		/* FIXME: we could handle this by doing multiple queries.. */
+		return false;
+	}
+
+	switch (arg->type) {
+	case SEARCH_TEXT: {
+		Query *q1 = lucene_get_query(index, _T("hdr"), arg);
+		Query *q2 = lucene_get_query(index, _T("body"), arg);
+
+		if (q1 == NULL && q2 == NULL)
+			q = NULL;
+		else {
+			BooleanQuery *bq = _CLNEW BooleanQuery();
+			if (q1 != NULL)
+				bq->add(q1, true, BooleanClause::SHOULD);
+			if (q2 != NULL)
+				bq->add(q2, true, BooleanClause::SHOULD);
+			q = bq;
+		}
+		break;
+	}
+	case SEARCH_BODY:
+		q = lucene_get_query(index, _T("body"), arg);
+		break;
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+		if (!fts_header_want_indexed(arg->hdr_field_name) ||
+		    *arg->value.str == '\0')
+			return false;
+
+		q = lucene_get_query(index,
+				     t_lucene_utf8_to_tchar(index, t_str_lcase(arg->hdr_field_name)),
+				     arg);
+		break;
+	default:
+		return false;
+	}
+
+	if (q == NULL) {
+		/* couldn't handle this search after all (e.g. trying to search
+		   a stop word) */
+		return false;
+	}
+
+	struct lucene_query *lq = array_append_space(&queries);
+	lq->query = q;
+	if (!and_args)
+		lq->occur = BooleanClause::SHOULD;
+	else if (!arg->match_not)
+		lq->occur = BooleanClause::MUST;
+	else
+		lq->occur = BooleanClause::MUST_NOT;
+	return true;
+}
+
+static bool
+lucene_add_maybe_query(struct lucene_index *index,
+		       ARRAY_TYPE(lucene_query) &queries,
+		       struct mail_search_arg *arg,
+		       enum fts_lookup_flags flags)
+{
+	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
+	Query *q = NULL;
+
+	if (arg->no_fts)
+		return false;
+
+	if (arg->match_not) {
+		/* FIXME: we could handle this by doing multiple queries.. */
+		return false;
+	}
+
+	switch (arg->type) {
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+		if (*arg->value.str == '\0' && !index->set.use_libfts) {
+			/* checking potential existence of the header name */
+			q = lucene_get_query_str(index, _T("hdr"),
+				t_str_lcase(arg->hdr_field_name), FALSE);
+			break;
+		}
+
+		if (fts_header_want_indexed(arg->hdr_field_name))
+			return false;
+
+		/* we can check if the search key exists in some header and
+		   filter out the messages that have no chance of matching */
+		q = lucene_get_query(index, _T("hdr"), arg);
+		break;
+	default:
+		return false;
+	}
+
+	if (q == NULL) {
+		/* couldn't handle this search after all (e.g. trying to search
+		   a stop word) */
+		return false;
+	}
+	struct lucene_query *lq = array_append_space(&queries);
+	lq->query = q;
+	if (!and_args)
+		lq->occur = BooleanClause::SHOULD;
+	else if (!arg->match_not)
+		lq->occur = BooleanClause::MUST;
+	else
+		lq->occur = BooleanClause::MUST_NOT;
+	return true;
+}
+
+static bool queries_have_non_must_nots(ARRAY_TYPE(lucene_query) &queries)
+{
+	const struct lucene_query *lq;
+
+	array_foreach(&queries, lq) {
+		if (lq->occur != BooleanClause::MUST_NOT)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static void search_query_add(BooleanQuery &query,
+			     ARRAY_TYPE(lucene_query) &queries)
+{
+	BooleanQuery *search_query = _CLNEW BooleanQuery();
+	const struct lucene_query *lq;
+
+	if (queries_have_non_must_nots(queries)) {
+		array_foreach(&queries, lq)
+			search_query->add(lq->query, true, lq->occur);
+		query.add(search_query, true, BooleanClause::MUST);
+	} else {
+		array_foreach(&queries, lq)
+			search_query->add(lq->query, true, BooleanClause::SHOULD);
+		query.add(search_query, true, BooleanClause::MUST_NOT);
+	}
+}
+
+static int
+lucene_index_search(struct lucene_index *index,
+		    ARRAY_TYPE(lucene_query) &queries,
+		    struct fts_result *result, ARRAY_TYPE(seq_range) *uids_r)
+{
+	struct fts_score_map *score;
+	int ret = 0;
+
+	BooleanQuery query;
+	search_query_add(query, queries);
+
+	Term mailbox_term(_T("box"), index->mailbox_guid);
+	TermQuery mailbox_query(&mailbox_term);
+	query.add(&mailbox_query, BooleanClause::MUST);
+
+	try {
+		Hits *hits = index->searcher->search(&query);
+
+		uint32_t last_uid = 0;
+		if (result != NULL)
+			result->scores_sorted = true;
+
+		for (size_t i = 0; i < hits->length(); i++) {
+			uint32_t uid;
+
+			if (lucene_doc_get_uid(index, &hits->doc(i),
+					       &uid) < 0) {
+				ret = -1;
+				break;
+			}
+
+			if (seq_range_array_add(uids_r, uid)) {
+				/* duplicate result */
+			} else if (result != NULL) {
+				if (uid < last_uid)
+					result->scores_sorted = false;
+				last_uid = uid;
+
+				score = array_append_space(&result->scores);
+				score->uid = uid;
+				score->score = hits->score(i);
+			}
+		}
+		_CLDELETE(hits);
+		return ret;
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "search");
+		return -1;
+	}
+}
+
+int lucene_index_lookup(struct lucene_index *index,
+			struct mail_search_arg *args,
+			enum fts_lookup_flags flags,
+			struct fts_result *result)
+{
+	struct mail_search_arg *arg;
+
+	if (lucene_index_open_search(index) <= 0)
+		return -1;
+
+	ARRAY_TYPE(lucene_query) def_queries;
+	t_array_init(&def_queries, 16);
+	bool have_definites = false;
+
+	for (arg = args; arg != NULL; arg = arg->next) {
+		if (lucene_add_definite_query(index, def_queries, arg, flags)) {
+			arg->match_always = true;
+			have_definites = true;
+		}
+	}
+
+	if (have_definites) {
+		ARRAY_TYPE(seq_range) *uids_arr =
+			(flags & FTS_LOOKUP_FLAG_NO_AUTO_FUZZY) == 0 ?
+			&result->definite_uids : &result->maybe_uids;
+		if (lucene_index_search(index, def_queries, result,
+					uids_arr) < 0)
+			return -1;
+	}
+
+	if (have_definites) {
+		/* FIXME: mixing up definite + maybe queries is broken. if the
+		   definite query matched, it'll just assume that the maybe
+		   queries matched as well */
+		return 0;
+	}
+
+	ARRAY_TYPE(lucene_query) maybe_queries;
+	t_array_init(&maybe_queries, 16);
+	bool have_maybies = false;
+
+	for (arg = args; arg != NULL; arg = arg->next) {
+		if (lucene_add_maybe_query(index, maybe_queries, arg, flags)) {
+			arg->match_always = true;
+			have_maybies = true;
+		}
+	}
+
+	if (have_maybies) {
+		if (lucene_index_search(index, maybe_queries, NULL,
+					&result->maybe_uids) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int
+lucene_index_search_multi(struct lucene_index *index,
+			  HASH_TABLE_TYPE(wguid_result) guids,
+			  ARRAY_TYPE(lucene_query) &queries,
+			  enum fts_lookup_flags flags,
+			  struct fts_multi_result *result)
+{
+	struct fts_score_map *score;
+	int ret = 0;
+
+	BooleanQuery query;
+	search_query_add(query, queries);
+
+	BooleanQuery mailbox_query;
+	struct hash_iterate_context *iter;
+	void *key, *value;
+	iter = hash_table_iterate_init(guids);
+	while (hash_table_iterate(iter, guids, &key, &value)) {
+		Term *term = _CLNEW Term(_T("box"), (wchar_t *)key);
+		TermQuery *q = _CLNEW TermQuery(term);
+		mailbox_query.add(q, true, BooleanClause::SHOULD);
+	}
+	hash_table_iterate_deinit(&iter);
+
+	query.add(&mailbox_query, BooleanClause::MUST);
+	try {
+		Hits *hits = index->searcher->search(&query);
+
+		for (size_t i = 0; i < hits->length(); i++) {
+			uint32_t uid;
+
+			Field *field = hits->doc(i).getField(_T("box"));
+			const TCHAR *box_guid = field == NULL ? NULL : field->stringValue();
+			if (box_guid == NULL) {
+				i_error("lucene: Corrupted FTS index %s: No mailbox for document",
+					index->path);
+				ret = -1;
+				break;
+			}
+			struct fts_result *br =
+				hash_table_lookup(guids, box_guid);
+			if (br == NULL) {
+				i_warning("lucene: Returned unexpected mailbox with GUID %ls", box_guid);
+				continue;
+			}
+
+			if (lucene_doc_get_uid(index, &hits->doc(i),
+					       &uid) < 0) {
+				ret = -1;
+				break;
+			}
+
+			ARRAY_TYPE(seq_range) *uids_arr =
+				(flags & FTS_LOOKUP_FLAG_NO_AUTO_FUZZY) == 0 ?
+				&br->maybe_uids : &br->definite_uids;
+			if (!array_is_created(uids_arr)) {
+				p_array_init(uids_arr, result->pool, 32);
+				p_array_init(&br->scores, result->pool, 32);
+			}
+			if (seq_range_array_add(uids_arr, uid)) {
+				/* duplicate result */
+			} else {
+				score = array_append_space(&br->scores);
+				score->uid = uid;
+				score->score = hits->score(i);
+			}
+		}
+		_CLDELETE(hits);
+		return ret;
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "multi search");
+		return -1;
+	}
+}
+
+int lucene_index_lookup_multi(struct lucene_index *index,
+			      HASH_TABLE_TYPE(wguid_result) guids,
+			      struct mail_search_arg *args,
+			      enum fts_lookup_flags flags,
+			      struct fts_multi_result *result)
+{
+	struct mail_search_arg *arg;
+
+	if (lucene_index_open_search(index) <= 0)
+		return -1;
+
+	ARRAY_TYPE(lucene_query) def_queries;
+	t_array_init(&def_queries, 16);
+	bool have_definites = false;
+
+	for (arg = args; arg != NULL; arg = arg->next) {
+		if (lucene_add_definite_query(index, def_queries, arg, flags)) {
+			arg->match_always = true;
+			have_definites = true;
+		}
+	}
+
+	if (have_definites) {
+		if (lucene_index_search_multi(index, guids, def_queries, flags,
+					      result) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+struct lucene_index_iter {
+	struct lucene_index *index;
+	struct lucene_index_record rec;
+
+	Term *term;
+	WildcardQuery *query;
+	Sort *sort;
+
+	Hits *hits;
+	size_t i;
+	bool failed;
+};
+
+struct lucene_index_iter *
+lucene_index_iter_init(struct lucene_index *index)
+{
+	static const TCHAR *sort_fields[] = { _T("box"), _T("uid"), NULL };
+	struct lucene_index_iter *iter;
+	int ret;
+
+	iter = i_new(struct lucene_index_iter, 1);
+	iter->index = index;
+	if ((ret = lucene_index_open_search(index)) <= 0) {
+		if (ret < 0)
+			iter->failed = true;
+		return iter;
+	}
+
+	iter->term = _CLNEW Term(_T("box"), _T("*"));
+	iter->query = _CLNEW WildcardQuery(iter->term);
+	iter->sort = _CLNEW Sort(sort_fields);
+
+	try {
+		iter->hits = index->searcher->search(iter->query, iter->sort);
+	} catch (CLuceneError &err) {
+		lucene_handle_error(index, err, "rescan search");
+		iter->failed = true;
+	}
+	return iter;
+}
+
+const struct lucene_index_record *
+lucene_index_iter_next(struct lucene_index_iter *iter)
+{
+	if (iter->hits == NULL)
+		return NULL;
+	if (iter->i == iter->hits->length())
+		return NULL;
+
+	Document *doc = &iter->hits->doc(iter->i);
+	iter->i++;
+
+	memset(&iter->rec, 0, sizeof(iter->rec));
+	(void)fts_lucene_get_mailbox_guid(iter->index, doc,
+					  iter->rec.mailbox_guid);
+	(void)lucene_doc_get_uid(iter->index, doc, &iter->rec.uid);
+	iter->rec.part_num = lucene_doc_get_part(iter->index, doc);
+	return &iter->rec;
+}
+
+int lucene_index_iter_deinit(struct lucene_index_iter **_iter)
+{
+	struct lucene_index_iter *iter = *_iter;
+	int ret = iter->failed ? -1 : 0;
+
+	*_iter = NULL;
+	if (iter->hits != NULL)
+		_CLDELETE(iter->hits);
+	if (iter->query != NULL) {
+		_CLDELETE(iter->query);
+		_CLDELETE(iter->sort);
+		_CLDELETE(iter->term);
+	}
+	i_free(iter);
+	return ret;
+}
+
+void lucene_shutdown(void)
+{
+	_lucene_shutdown();
+}
diff --git a/src/plugins/fts-lucene/lucene-wrapper.h b/src/plugins/fts-lucene/lucene-wrapper.h
new file mode 100644
index 0000000..270e902
--- /dev/null
+++ b/src/plugins/fts-lucene/lucene-wrapper.h
@@ -0,0 +1,67 @@
+#ifndef LUCENE_WRAPPER_H
+#define LUCENE_WRAPPER_H
+
+#include "fts-api-private.h"
+#include "guid.h"
+
+struct mailbox_list;
+struct fts_expunge_log;
+struct fts_lucene_settings;
+
+#define MAILBOX_GUID_HEX_LENGTH (GUID_128_SIZE*2)
+
+struct lucene_index_record {
+	guid_128_t mailbox_guid;
+	uint32_t uid, part_num;
+};
+
+HASH_TABLE_DEFINE_TYPE(wguid_result, wchar_t *, struct fts_result *);
+
+struct lucene_index *
+lucene_index_init(const char *path, struct mailbox_list *list,
+		  const struct fts_lucene_settings *set)
+	ATTR_NULL(2, 3);
+void lucene_index_deinit(struct lucene_index *index);
+
+void lucene_index_select_mailbox(struct lucene_index *index,
+				 const wchar_t guid[MAILBOX_GUID_HEX_LENGTH]);
+void lucene_index_unselect_mailbox(struct lucene_index *index);
+int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r);
+int lucene_index_get_doc_count(struct lucene_index *index, uint32_t *count_r);
+
+int lucene_index_build_init(struct lucene_index *index);
+int lucene_index_build_more(struct lucene_index *index, uint32_t uid,
+			    uint32_t part_num, const unsigned char *data,
+			    size_t size, const char *hdr_name);
+int lucene_index_build_deinit(struct lucene_index *index);
+
+void lucene_index_close(struct lucene_index *index);
+int lucene_index_rescan(struct lucene_index *index);
+int lucene_index_expunge_from_log(struct lucene_index *index,
+				  struct fts_expunge_log *log);
+int lucene_index_optimize(struct lucene_index *index);
+
+int lucene_index_lookup(struct lucene_index *index, 
+			struct mail_search_arg *args,
+			enum fts_lookup_flags flags,
+			struct fts_result *result);
+
+int lucene_index_lookup_multi(struct lucene_index *index,
+			      HASH_TABLE_TYPE(wguid_result) guids,
+			      struct mail_search_arg *args,
+			      enum fts_lookup_flags flags,
+			      struct fts_multi_result *result);
+
+struct lucene_index_iter *
+lucene_index_iter_init(struct lucene_index *index);
+const struct lucene_index_record *
+lucene_index_iter_next(struct lucene_index_iter *iter);
+int lucene_index_iter_deinit(struct lucene_index_iter **iter);
+
+/* internal: */
+void lucene_utf8_n_to_tchar(const unsigned char *src, size_t srcsize,
+			    wchar_t *dest, size_t destsize);
+
+void lucene_shutdown(void);
+
+#endif
diff --git a/src/plugins/fts-lucene/textcat.conf b/src/plugins/fts-lucene/textcat.conf
new file mode 100644
index 0000000..d75c4fe
--- /dev/null
+++ b/src/plugins/fts-lucene/textcat.conf
@@ -0,0 +1,25 @@
+#
+# A sample config file for the language models 
+# provided with Gertjan van Noords language guesser
+# (http://odur.let.rug.nl/~vannoord/TextCat/)
+#
+# Notes: 
+# - You may consider eliminating a couple of small languages from this
+# list because they cause false positives with big languages and are
+# bad for performance. (Do you really want to recognize Drents?)
+# - Putting the most probable languages at the top of the list
+# improves performance, because this will raise the threshold for
+# likely candidates more quickly.
+#
+LM/english.lm			english
+LM/italian.lm			italian
+LM/danish.lm			danish
+LM/dutch.lm			dutch
+LM/finnish.lm			finnish
+LM/french.lm			french
+LM/german.lm			german
+LM/norwegian.lm			norwegian
+LM/portuguese.lm		portuguese
+LM/russian.lm			russian
+LM/spanish.lm			spanish
+LM/swedish.lm			swedish
diff --git a/src/plugins/fts-solr/Makefile.am b/src/plugins/fts-solr/Makefile.am
new file mode 100644
index 0000000..cd1e17b
--- /dev/null
+++ b/src/plugins/fts-solr/Makefile.am
@@ -0,0 +1,64 @@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-http \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/plugins/fts
+
+NOPLUGIN_LDFLAGS =
+lib21_fts_solr_plugin_la_LDFLAGS = -module -avoid-version
+
+module_LTLIBRARIES = \
+	lib21_fts_solr_plugin.la
+
+if DOVECOT_PLUGIN_DEPS
+fts_plugin_dep = ../fts/lib20_fts_plugin.la
+endif
+
+lib21_fts_solr_plugin_la_LIBADD = \
+	$(fts_plugin_dep) \
+	-lexpat
+
+lib21_fts_solr_plugin_la_SOURCES = \
+	fts-backend-solr.c \
+	fts-backend-solr-old.c \
+	fts-solr-plugin.c \
+	solr-response.c \
+	solr-connection.c
+
+noinst_HEADERS = \
+	fts-solr-plugin.h \
+	solr-response.h \
+	solr-connection.h
+
+test_programs = \
+	test-solr-response
+
+test_libs = \
+	../../lib-test/libtest.la \
+	../../lib-charset/libcharset.la \
+	../../lib/liblib.la \
+	$(MODULE_LIBS)
+
+noinst_PROGRAMS = test-solr-response
+
+test_solr_response_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	-I$(top_srcdir)/src/lib-test
+test_solr_response_SOURCES = \
+	solr-response.c \
+	test-solr-response.c
+test_solr_response_LDADD = \
+	$(test_libs) -lexpat
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+
+check: check-am check-test
+check-test: all-am
+	for bin in $(test_programs); do \
+	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+	done
diff --git a/src/plugins/fts-solr/Makefile.in b/src/plugins/fts-solr/Makefile.in
new file mode 100644
index 0000000..ee248f8
--- /dev/null
+++ b/src/plugins/fts-solr/Makefile.in
@@ -0,0 +1,965 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+noinst_PROGRAMS = test-solr-response$(EXEEXT)
+subdir = src/plugins/fts-solr
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+	$(pkginc_lib_HEADERS) $(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+PROGRAMS = $(noinst_PROGRAMS)
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(moduledir)" \
+	"$(DESTDIR)$(pkginc_libdir)"
+LTLIBRARIES = $(module_LTLIBRARIES)
+lib21_fts_solr_plugin_la_DEPENDENCIES = $(fts_plugin_dep)
+am_lib21_fts_solr_plugin_la_OBJECTS = fts-backend-solr.lo \
+	fts-backend-solr-old.lo fts-solr-plugin.lo solr-response.lo \
+	solr-connection.lo
+lib21_fts_solr_plugin_la_OBJECTS =  \
+	$(am_lib21_fts_solr_plugin_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+lib21_fts_solr_plugin_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+	$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+	$(AM_CFLAGS) $(CFLAGS) $(lib21_fts_solr_plugin_la_LDFLAGS) \
+	$(LDFLAGS) -o $@
+am_test_solr_response_OBJECTS =  \
+	test_solr_response-solr-response.$(OBJEXT) \
+	test_solr_response-test-solr-response.$(OBJEXT)
+test_solr_response_OBJECTS = $(am_test_solr_response_OBJECTS)
+am__DEPENDENCIES_1 =
+am__DEPENDENCIES_2 = ../../lib-test/libtest.la \
+	../../lib-charset/libcharset.la ../../lib/liblib.la \
+	$(am__DEPENDENCIES_1)
+test_solr_response_DEPENDENCIES = $(am__DEPENDENCIES_2)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/fts-backend-solr-old.Plo \
+	./$(DEPDIR)/fts-backend-solr.Plo \
+	./$(DEPDIR)/fts-solr-plugin.Plo \
+	./$(DEPDIR)/solr-connection.Plo ./$(DEPDIR)/solr-response.Plo \
+	./$(DEPDIR)/test_solr_response-solr-response.Po \
+	./$(DEPDIR)/test_solr_response-test-solr-response.Po
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(lib21_fts_solr_plugin_la_SOURCES) \
+	$(test_solr_response_SOURCES)
+DIST_SOURCES = $(lib21_fts_solr_plugin_la_SOURCES) \
+	$(test_solr_response_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = 
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-http \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/plugins/fts
+
+lib21_fts_solr_plugin_la_LDFLAGS = -module -avoid-version
+module_LTLIBRARIES = \
+	lib21_fts_solr_plugin.la
+
+@DOVECOT_PLUGIN_DEPS_TRUE@fts_plugin_dep = ../fts/lib20_fts_plugin.la
+lib21_fts_solr_plugin_la_LIBADD = \
+	$(fts_plugin_dep) \
+	-lexpat
+
+lib21_fts_solr_plugin_la_SOURCES = \
+	fts-backend-solr.c \
+	fts-backend-solr-old.c \
+	fts-solr-plugin.c \
+	solr-response.c \
+	solr-connection.c
+
+noinst_HEADERS = \
+	fts-solr-plugin.h \
+	solr-response.h \
+	solr-connection.h
+
+test_programs = \
+	test-solr-response
+
+test_libs = \
+	../../lib-test/libtest.la \
+	../../lib-charset/libcharset.la \
+	../../lib/liblib.la \
+	$(MODULE_LIBS)
+
+test_solr_response_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	-I$(top_srcdir)/src/lib-test
+
+test_solr_response_SOURCES = \
+	solr-response.c \
+	test-solr-response.c
+
+test_solr_response_LDADD = \
+	$(test_libs) -lexpat
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/plugins/fts-solr/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/plugins/fts-solr/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstPROGRAMS:
+	@list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \
+	echo " rm -f" $$list; \
+	rm -f $$list || exit $$?; \
+	test -n "$(EXEEXT)" || exit 0; \
+	list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+	echo " rm -f" $$list; \
+	rm -f $$list
+
+install-moduleLTLIBRARIES: $(module_LTLIBRARIES)
+	@$(NORMAL_INSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	list2=; for p in $$list; do \
+	  if test -f $$p; then \
+	    list2="$$list2 $$p"; \
+	  else :; fi; \
+	done; \
+	test -z "$$list2" || { \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(moduledir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(moduledir)" || exit 1; \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(moduledir)'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(moduledir)"; \
+	}
+
+uninstall-moduleLTLIBRARIES:
+	@$(NORMAL_UNINSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	for p in $$list; do \
+	  $(am__strip_dir) \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(moduledir)/$$f'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(moduledir)/$$f"; \
+	done
+
+clean-moduleLTLIBRARIES:
+	-test -z "$(module_LTLIBRARIES)" || rm -f $(module_LTLIBRARIES)
+	@list='$(module_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+lib21_fts_solr_plugin.la: $(lib21_fts_solr_plugin_la_OBJECTS) $(lib21_fts_solr_plugin_la_DEPENDENCIES) $(EXTRA_lib21_fts_solr_plugin_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(lib21_fts_solr_plugin_la_LINK) -rpath $(moduledir) $(lib21_fts_solr_plugin_la_OBJECTS) $(lib21_fts_solr_plugin_la_LIBADD) $(LIBS)
+
+test-solr-response$(EXEEXT): $(test_solr_response_OBJECTS) $(test_solr_response_DEPENDENCIES) $(EXTRA_test_solr_response_DEPENDENCIES) 
+	@rm -f test-solr-response$(EXEEXT)
+	$(AM_V_CCLD)$(LINK) $(test_solr_response_OBJECTS) $(test_solr_response_LDADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-backend-solr-old.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-backend-solr.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-solr-plugin.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/solr-connection.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/solr-response.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_solr_response-solr-response.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_solr_response-test-solr-response.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+test_solr_response-solr-response.o: solr-response.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT test_solr_response-solr-response.o -MD -MP -MF $(DEPDIR)/test_solr_response-solr-response.Tpo -c -o test_solr_response-solr-response.o `test -f 'solr-response.c' || echo '$(srcdir)/'`solr-response.c
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/test_solr_response-solr-response.Tpo $(DEPDIR)/test_solr_response-solr-response.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='solr-response.c' object='test_solr_response-solr-response.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o test_solr_response-solr-response.o `test -f 'solr-response.c' || echo '$(srcdir)/'`solr-response.c
+
+test_solr_response-solr-response.obj: solr-response.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT test_solr_response-solr-response.obj -MD -MP -MF $(DEPDIR)/test_solr_response-solr-response.Tpo -c -o test_solr_response-solr-response.obj `if test -f 'solr-response.c'; then $(CYGPATH_W) 'solr-response.c'; else $(CYGPATH_W) '$(srcdir)/solr-response.c'; fi`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/test_solr_response-solr-response.Tpo $(DEPDIR)/test_solr_response-solr-response.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='solr-response.c' object='test_solr_response-solr-response.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o test_solr_response-solr-response.obj `if test -f 'solr-response.c'; then $(CYGPATH_W) 'solr-response.c'; else $(CYGPATH_W) '$(srcdir)/solr-response.c'; fi`
+
+test_solr_response-test-solr-response.o: test-solr-response.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT test_solr_response-test-solr-response.o -MD -MP -MF $(DEPDIR)/test_solr_response-test-solr-response.Tpo -c -o test_solr_response-test-solr-response.o `test -f 'test-solr-response.c' || echo '$(srcdir)/'`test-solr-response.c
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/test_solr_response-test-solr-response.Tpo $(DEPDIR)/test_solr_response-test-solr-response.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='test-solr-response.c' object='test_solr_response-test-solr-response.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o test_solr_response-test-solr-response.o `test -f 'test-solr-response.c' || echo '$(srcdir)/'`test-solr-response.c
+
+test_solr_response-test-solr-response.obj: test-solr-response.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT test_solr_response-test-solr-response.obj -MD -MP -MF $(DEPDIR)/test_solr_response-test-solr-response.Tpo -c -o test_solr_response-test-solr-response.obj `if test -f 'test-solr-response.c'; then $(CYGPATH_W) 'test-solr-response.c'; else $(CYGPATH_W) '$(srcdir)/test-solr-response.c'; fi`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/test_solr_response-test-solr-response.Tpo $(DEPDIR)/test_solr_response-test-solr-response.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='test-solr-response.c' object='test_solr_response-test-solr-response.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_solr_response_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o test_solr_response-test-solr-response.obj `if test -f 'test-solr-response.c'; then $(CYGPATH_W) 'test-solr-response.c'; else $(CYGPATH_W) '$(srcdir)/test-solr-response.c'; fi`
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(moduledir)" "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-moduleLTLIBRARIES \
+	clean-noinstPROGRAMS mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/fts-backend-solr-old.Plo
+	-rm -f ./$(DEPDIR)/fts-backend-solr.Plo
+	-rm -f ./$(DEPDIR)/fts-solr-plugin.Plo
+	-rm -f ./$(DEPDIR)/solr-connection.Plo
+	-rm -f ./$(DEPDIR)/solr-response.Plo
+	-rm -f ./$(DEPDIR)/test_solr_response-solr-response.Po
+	-rm -f ./$(DEPDIR)/test_solr_response-test-solr-response.Po
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-moduleLTLIBRARIES install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/fts-backend-solr-old.Plo
+	-rm -f ./$(DEPDIR)/fts-backend-solr.Plo
+	-rm -f ./$(DEPDIR)/fts-solr-plugin.Plo
+	-rm -f ./$(DEPDIR)/solr-connection.Plo
+	-rm -f ./$(DEPDIR)/solr-response.Plo
+	-rm -f ./$(DEPDIR)/test_solr_response-solr-response.Po
+	-rm -f ./$(DEPDIR)/test_solr_response-test-solr-response.Po
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-moduleLTLIBRARIES uninstall-pkginc_libHEADERS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-moduleLTLIBRARIES \
+	clean-noinstPROGRAMS cscopelist-am ctags ctags-am distclean \
+	distclean-compile distclean-generic distclean-libtool \
+	distclean-tags distdir dvi dvi-am html html-am info info-am \
+	install install-am install-data install-data-am install-dvi \
+	install-dvi-am install-exec install-exec-am install-html \
+	install-html-am install-info install-info-am install-man \
+	install-moduleLTLIBRARIES install-pdf install-pdf-am \
+	install-pkginc_libHEADERS install-ps install-ps-am \
+	install-strip installcheck installcheck-am installdirs \
+	maintainer-clean maintainer-clean-generic mostlyclean \
+	mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
+	pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \
+	uninstall-moduleLTLIBRARIES uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+check: check-am check-test
+check-test: all-am
+	for bin in $(test_programs); do \
+	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+	done
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/plugins/fts-solr/fts-backend-solr-old.c b/src/plugins/fts-solr/fts-backend-solr-old.c
new file mode 100644
index 0000000..20b891b
--- /dev/null
+++ b/src/plugins/fts-solr/fts-backend-solr-old.c
@@ -0,0 +1,879 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "hash.h"
+#include "strescape.h"
+#include "unichar.h"
+#include "iostream-ssl.h"
+#include "http-url.h"
+#include "imap-utf7.h"
+#include "mail-storage-private.h"
+#include "mailbox-list-private.h"
+#include "mail-search.h"
+#include "fts-api.h"
+#include "solr-connection.h"
+#include "fts-solr-plugin.h"
+
+#include <ctype.h>
+
+#define SOLR_CMDBUF_SIZE (1024*64)
+#define SOLR_MAX_MULTI_ROWS 100000
+
+struct solr_fts_backend {
+	struct fts_backend backend;
+	struct solr_connection *solr_conn;
+	char *id_username, *id_namespace;
+	struct mail_namespace *default_ns;
+};
+
+struct solr_fts_backend_update_context {
+	struct fts_backend_update_context ctx;
+
+	struct mailbox *cur_box;
+	char *id_box_name;
+
+	struct solr_connection_post *post;
+	uint32_t prev_uid, uid_validity;
+	string_t *cmd, *hdr;
+
+	bool headers_open;
+	bool body_open;
+	bool documents_added;
+};
+
+static const char *solr_escape_chars = "+-&|!(){}[]^\"~*?:\\/ ";
+
+static bool is_valid_xml_char(unichar_t chr)
+{
+	/* Valid characters in XML:
+
+	   #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] |
+	   [#x10000-#x10FFFF]
+
+	   This function gets called only for #x80 and higher */
+	if (chr > 0xd7ff && chr < 0xe000)
+		return FALSE;
+	if (chr > 0xfffd && chr < 0x10000)
+		return FALSE;
+	return chr < 0x10ffff;
+}
+
+static void
+xml_encode_data(string_t *dest, const unsigned char *data, size_t len)
+{
+	unichar_t chr;
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		switch (data[i]) {
+		case '&':
+			str_append(dest, "&amp;");
+			break;
+		case '<':
+			str_append(dest, "&lt;");
+			break;
+		case '>':
+			str_append(dest, "&gt;");
+			break;
+		case '\t':
+		case '\n':
+		case '\r':
+			/* exceptions to the following control char check */
+			str_append_c(dest, data[i]);
+			break;
+		default:
+			if (data[i] < 32) {
+				/* SOLR doesn't like control characters.
+				   replace them with spaces. */
+				str_append_c(dest, ' ');
+			} else if (data[i] >= 0x80) {
+				/* make sure the character is valid for XML
+				   so we don't get XML parser errors */
+				int char_len =
+					uni_utf8_get_char_n(data + i, len - i, &chr);
+				i_assert(char_len > 0); /* input is valid UTF8 */
+				if (is_valid_xml_char(chr))
+					str_append_data(dest, data + i, char_len);
+				else {
+					str_append_data(dest, utf8_replacement_char,
+							UTF8_REPLACEMENT_CHAR_LEN);
+				}
+				i += char_len - 1;
+			} else {
+				str_append_c(dest, data[i]);
+			}
+			break;
+		}
+	}
+}
+
+static void xml_encode(string_t *dest, const char *str)
+{
+	xml_encode_data(dest, (const unsigned char *)str, strlen(str));
+}
+
+static const char *solr_escape_id_str(const char *str)
+{
+	string_t *tmp;
+	const char *p;
+
+	for (p = str; *p != '\0'; p++) {
+		if (*p == '/' || *p == '!')
+			break;
+	}
+	if (*p == '\0')
+		return str;
+
+	tmp = t_str_new(64);
+	for (p = str; *p != '\0'; p++) {
+		switch (*p) {
+		case '/':
+			str_append(tmp, "!\\");
+			break;
+		case '!':
+			str_append(tmp, "!!");
+			break;
+		default:
+			str_append_c(tmp, *p);
+			break;
+		}
+	}
+	return str_c(tmp);
+}
+
+static const char *solr_escape(const char *str)
+{
+	string_t *ret;
+	unsigned int i;
+
+	if (str[0] == '\0')
+		return "\"\"";
+
+	ret = t_str_new(strlen(str) + 16);
+	for (i = 0; str[i] != '\0'; i++) {
+		if (strchr(solr_escape_chars, str[i]) != NULL)
+			str_append_c(ret, '\\');
+		str_append_c(ret, str[i]);
+	}
+	return str_c(ret);
+}
+
+static void solr_quote(string_t *dest, const char *str)
+{
+	str_append(dest, solr_escape(str));
+}
+
+static void solr_quote_http(string_t *dest, const char *str)
+{
+	http_url_escape_param(dest, solr_escape(str));
+}
+
+static void fts_solr_set_default_ns(struct solr_fts_backend *backend)
+{
+	struct mail_namespace *ns = backend->backend.ns;
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT_REQUIRE(ns->user);
+	const struct fts_solr_settings *set = &fuser->set;
+	const char *str;
+
+	if (backend->default_ns != NULL)
+		return;
+
+	if (set->default_ns_prefix != NULL) {
+		backend->default_ns =
+			mail_namespace_find_prefix(ns->user->namespaces,
+						   set->default_ns_prefix);
+		if (backend->default_ns == NULL) {
+			i_error("fts_solr: default_ns setting points to "
+				"nonexistent namespace");
+		}
+	}
+	if (backend->default_ns == NULL) {
+		backend->default_ns =
+			mail_namespace_find_inbox(ns->user->namespaces);
+	}
+	while (backend->default_ns->alias_for != NULL)
+		backend->default_ns = backend->default_ns->alias_for;
+
+	if (ns != backend->default_ns) {
+		str = solr_escape_id_str(ns->prefix);
+		backend->id_namespace = i_strdup(str);
+	}
+}
+
+static void fts_box_name_get_root(struct mail_namespace **ns, const char **name)
+{
+	struct mail_namespace *orig_ns = *ns;
+
+	while ((*ns)->alias_for != NULL)
+		*ns = (*ns)->alias_for;
+
+	if (**name == '\0' && *ns != orig_ns &&
+	    ((*ns)->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
+		/* ugly workaround to allow selecting INBOX from a Maildir/
+		   when it's not in the inbox=yes namespace. */
+		*name = "INBOX";
+	}
+}
+
+static const char *
+fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r)
+{
+	struct mail_namespace *ns = mailbox_get_namespace(box);
+	const char *name;
+
+	if (t_imap_utf8_to_utf7(box->name, &name) < 0)
+		i_unreached();
+
+	fts_box_name_get_root(&ns, &name);
+	*ns_r = ns;
+	return name;
+}
+
+static struct fts_backend *fts_backend_solr_alloc(void)
+{
+	struct solr_fts_backend *backend;
+
+	backend = i_new(struct solr_fts_backend, 1);
+	backend->backend = fts_backend_solr_old;
+	return &backend->backend;
+}
+
+static int
+fts_backend_solr_init(struct fts_backend *_backend, const char **error_r)
+{
+	struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user);
+	struct ssl_iostream_settings ssl_set;
+	const char *str;
+
+	if (fuser == NULL) {
+		*error_r = "Invalid fts_solr setting";
+		return -1;
+	}
+
+	mail_user_init_ssl_client_settings(_backend->ns->user, &ssl_set);
+	if (solr_connection_init(&fuser->set, &ssl_set,
+				 _backend->ns->user->event,
+				 &backend->solr_conn, error_r) < 0)
+		return -1;
+
+	str = solr_escape_id_str(_backend->ns->user->username);
+	backend->id_username = i_strdup(str);
+	return 0;
+}
+
+static void fts_backend_solr_deinit(struct fts_backend *_backend)
+{
+	struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+
+	solr_connection_deinit(&backend->solr_conn);
+	i_free(backend->id_namespace);
+	i_free(backend->id_username);
+	i_free(backend);
+}
+
+static void
+solr_add_ns_query(string_t *str, struct solr_fts_backend *backend,
+		  struct mail_namespace *ns, bool neg)
+{
+	while (ns->alias_for != NULL)
+		ns = ns->alias_for;
+
+	if (ns == backend->default_ns || *ns->prefix == '\0') {
+		if (!neg)
+			str_append(str, " -ns:[* TO *]");
+		else
+			str_append(str, " +ns:[* TO *]");
+	} else {
+		if (!neg)
+			str_append(str, " +ns:");
+		else
+			str_append(str, " -ns:");
+		solr_quote(str, ns->prefix);
+	}
+}
+
+static void
+solr_add_ns_query_http(string_t *str, struct solr_fts_backend *backend,
+		       struct mail_namespace *ns)
+{
+	string_t *tmp;
+
+	tmp = t_str_new(64);
+	solr_add_ns_query(tmp, backend, ns, FALSE);
+	http_url_escape_param(str, str_c(tmp));
+}
+
+static int
+fts_backend_solr_get_last_uid_fallback(struct solr_fts_backend *backend,
+				       struct mailbox *box,
+				       uint32_t *last_uid_r)
+{
+	struct mail_namespace *ns;
+	struct mailbox_status status;
+	struct solr_result **results;
+	const struct seq_range *uidvals;
+	const char *box_name;
+	unsigned int count;
+	string_t *str;
+	pool_t pool;
+	int ret = 0;
+
+	str = t_str_new(256);
+	str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
+
+	box_name = fts_box_get_root(box, &ns);
+
+	mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
+	str_printfa(str, "uidv:%u+AND+box:", status.uidvalidity);
+	solr_quote_http(str, box_name);
+	solr_add_ns_query_http(str, backend, ns);
+	str_append(str, "+AND+user:");
+	solr_quote_http(str, ns->user->username);
+
+	pool = pool_alloconly_create("solr last uid lookup", 1024);
+	if (solr_connection_select(backend->solr_conn, str_c(str),
+				   pool, &results) < 0)
+		ret = -1;
+	else if (results[0] == NULL) {
+		/* no UIDs */
+		*last_uid_r = 0;
+	} else {
+		uidvals = array_get(&results[0]->uids, &count);
+		i_assert(count > 0);
+		if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
+			*last_uid_r = uidvals[0].seq1;
+		} else {
+			i_error("fts_solr: Last UID lookup returned multiple rows");
+			ret = -1;
+		}
+	}
+	pool_unref(&pool);
+	return ret;
+}
+
+static int
+fts_backend_solr_get_last_uid(struct fts_backend *_backend,
+			      struct mailbox *box, uint32_t *last_uid_r)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)_backend;
+	struct fts_index_header hdr;
+
+	if (fts_index_get_header(box, &hdr)) {
+		*last_uid_r = hdr.last_indexed_uid;
+		return 0;
+	}
+
+	/* either nothing has been indexed, or the index was corrupted.
+	   do it the slow way. */
+	if (fts_backend_solr_get_last_uid_fallback(backend, box, last_uid_r) < 0)
+		return -1;
+
+	fts_index_set_last_uid(box, *last_uid_r);
+	return 0;
+}
+
+static struct fts_backend_update_context *
+fts_backend_solr_update_init(struct fts_backend *_backend)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)_backend;
+	struct solr_fts_backend_update_context *ctx;
+
+	ctx = i_new(struct solr_fts_backend_update_context, 1);
+	ctx->ctx.backend = _backend;
+	ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
+	ctx->hdr = str_new(default_pool, 4096);
+	fts_solr_set_default_ns(backend);
+	return &ctx->ctx;
+}
+
+static void xml_encode_id(struct solr_fts_backend_update_context *ctx,
+			  string_t *str, uint32_t uid)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)ctx->ctx.backend;
+
+	if (uid != 0)
+		str_printfa(str, "%u/", uid);
+	else
+		str_append(str, "L/");
+
+	if (backend->id_namespace != NULL) {
+		xml_encode(str, backend->id_namespace);
+		str_append_c(str, '/');
+	}
+	str_printfa(str, "%u/", ctx->uid_validity);
+	xml_encode(str, backend->id_username);
+	str_append_c(str, '/');
+	xml_encode(str, ctx->id_box_name);
+}
+
+static void
+fts_backend_solr_add_doc_prefix(struct solr_fts_backend_update_context *ctx,
+				uint32_t uid)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)ctx->ctx.backend;
+	struct mailbox *box = ctx->cur_box;
+	struct mail_namespace *ns;
+	const char *box_name;
+
+	ctx->documents_added = TRUE;
+
+	str_printfa(ctx->cmd, "<doc>"
+		    "<field name=\"uid\">%u</field>"
+		    "<field name=\"uidv\">%u</field>",
+		    uid, ctx->uid_validity);
+
+	box_name = fts_box_get_root(box, &ns);
+
+	if (ns != backend->default_ns) {
+		str_append(ctx->cmd, "<field name=\"ns\">");
+		xml_encode(ctx->cmd, ns->prefix);
+		str_append(ctx->cmd, "</field>");
+	}
+	str_append(ctx->cmd, "<field name=\"box\">");
+	xml_encode(ctx->cmd, box_name);
+	str_append(ctx->cmd, "</field><field name=\"user\">");
+	xml_encode(ctx->cmd, ns->user->username);
+	str_append(ctx->cmd, "</field>");
+}
+
+static int
+fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx)
+{
+	if (ctx->post == NULL)
+		return 0;
+
+	str_append(ctx->cmd, "</doc></add>");
+
+	solr_connection_post_more(ctx->post, str_data(ctx->cmd),
+				  str_len(ctx->cmd));
+	return solr_connection_post_end(&ctx->post);
+}
+
+static int
+fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)_ctx->backend;
+	const char *str;
+	int ret;
+
+	ret = fts_backed_solr_build_commit(ctx);
+
+	/* commit and wait until the documents we just indexed are
+	   visible to the following search */
+	str = t_strdup_printf("<commit waitFlush=\"false\" "
+			      "waitSearcher=\"%s\"/>",
+			      ctx->documents_added ? "true" : "false");
+	if (solr_connection_post(backend->solr_conn, str) < 0)
+		ret = -1;
+
+	str_free(&ctx->cmd);
+	str_free(&ctx->hdr);
+	i_free(ctx->id_box_name);
+	i_free(ctx);
+	return ret;
+}
+
+static void
+fts_backend_solr_update_set_mailbox(struct fts_backend_update_context *_ctx,
+				    struct mailbox *box)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+	struct mailbox_status status;
+	struct mail_namespace *ns;
+
+	if (ctx->prev_uid != 0) {
+		fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid);
+		ctx->prev_uid = 0;
+	}
+
+	ctx->cur_box = box;
+	ctx->uid_validity = 0;
+	i_free_and_null(ctx->id_box_name);
+
+	if (box != NULL) {
+		ctx->id_box_name = i_strdup(fts_box_get_root(box, &ns));
+
+		mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
+		ctx->uid_validity = status.uidvalidity;
+	}
+}
+
+static void
+fts_backend_solr_update_expunge(struct fts_backend_update_context *_ctx,
+				uint32_t uid)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)_ctx->backend;
+
+	T_BEGIN {
+		string_t *cmd;
+
+		cmd = t_str_new(256);
+		str_append(cmd, "<delete><id>");
+		xml_encode_id(ctx, cmd, uid);
+		str_append(cmd, "</id></delete>");
+
+		(void)solr_connection_post(backend->solr_conn, str_c(cmd));
+	} T_END;
+}
+
+static void
+fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx,
+			     uint32_t uid)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)ctx->ctx.backend;
+
+	if (ctx->post == NULL) {
+		i_assert(ctx->prev_uid == 0);
+
+		ctx->post = solr_connection_post_begin(backend->solr_conn);
+		str_append(ctx->cmd, "<add>");
+	} else {
+		ctx->headers_open = FALSE;
+		if (ctx->body_open) {
+			ctx->body_open = FALSE;
+			str_append(ctx->cmd, "</field>");
+		}
+		str_append(ctx->cmd, "<field name=\"hdr\">");
+		str_append_str(ctx->cmd, ctx->hdr);
+		str_append(ctx->cmd, "</field>");
+		str_truncate(ctx->hdr, 0);
+
+		str_append(ctx->cmd, "</doc>");
+	}
+	ctx->prev_uid = uid;
+
+	fts_backend_solr_add_doc_prefix(ctx, uid);
+	str_printfa(ctx->cmd, "<field name=\"id\">");
+	xml_encode_id(ctx, ctx->cmd, uid);
+	str_append(ctx->cmd, "</field>");
+}
+
+static bool
+fts_backend_solr_update_set_build_key(struct fts_backend_update_context *_ctx,
+				      const struct fts_backend_build_key *key)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+
+	if (key->uid != ctx->prev_uid)
+		fts_backend_solr_uid_changed(ctx, key->uid);
+
+	switch (key->type) {
+	case FTS_BACKEND_BUILD_KEY_HDR:
+	case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+		xml_encode(ctx->hdr, key->hdr_name);
+		str_append(ctx->hdr, ": ");
+		ctx->headers_open = TRUE;
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART:
+		ctx->headers_open = FALSE;
+		if (!ctx->body_open) {
+			ctx->body_open = TRUE;
+			str_append(ctx->cmd, "<field name=\"body\">");
+		}
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+		i_unreached();
+	}
+	return TRUE;
+}
+
+static void
+fts_backend_solr_update_unset_build_key(struct fts_backend_update_context *_ctx)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+
+	if (ctx->headers_open)
+		str_append_c(ctx->hdr, '\n');
+	else {
+		i_assert(ctx->body_open);
+		str_append_c(ctx->cmd, '\n');
+	}
+}
+
+static int
+fts_backend_solr_update_build_more(struct fts_backend_update_context *_ctx,
+				   const unsigned char *data, size_t size)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+
+	xml_encode_data(ctx->cmd, data, size);
+	if (str_len(ctx->cmd) > SOLR_CMDBUF_SIZE-128) {
+		solr_connection_post_more(ctx->post, str_data(ctx->cmd),
+					  str_len(ctx->cmd));
+		str_truncate(ctx->cmd, 0);
+	}
+	return 0;
+}
+
+static int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED)
+{
+	return 0;
+}
+
+static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED)
+{
+	return 0;
+}
+
+static bool
+solr_add_definite_query(string_t *str, struct mail_search_arg *arg)
+{
+	if (arg->no_fts)
+		return FALSE;
+	switch (arg->type) {
+	case SEARCH_TEXT: {
+		if (arg->match_not)
+			str_append_c(str, '-');
+		str_append(str, "(hdr:");
+		solr_quote_http(str, arg->value.str);
+		str_append(str, "+OR+body:");
+		solr_quote_http(str, arg->value.str);
+		str_append(str, ")");
+		break;
+	}
+	case SEARCH_BODY:
+		if (arg->match_not)
+			str_append_c(str, '-');
+		str_append(str, "body:");
+		solr_quote_http(str, arg->value.str);
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static bool
+solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg,
+			     bool and_args)
+{
+	size_t last_len;
+
+	last_len = str_len(str);
+	for (; arg != NULL; arg = arg->next) {
+		if (solr_add_definite_query(str, arg)) {
+			arg->match_always = TRUE;
+			last_len = str_len(str);
+			if (and_args)
+				str_append(str, "+AND+");
+			else
+				str_append(str, "+OR+");
+		}
+	}
+	if (str_len(str) == last_len)
+		return FALSE;
+
+	str_truncate(str, last_len);
+	return TRUE;
+}
+
+static int
+fts_backend_solr_lookup(struct fts_backend *_backend, struct mailbox *box,
+			struct mail_search_arg *args,
+			enum fts_lookup_flags flags,
+			struct fts_result *result)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)_backend;
+	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
+	struct mail_namespace *ns;
+	struct mailbox_status status;
+	string_t *str;
+	const char *box_name;
+	pool_t pool;
+	struct solr_result **results;
+	int ret;
+
+	fts_solr_set_default_ns(backend);
+	mailbox_get_open_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT,
+				&status);
+
+	str = t_str_new(256);
+	str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=%%7b!lucene+q.op%%3dAND%%7d",
+		    status.uidnext);
+
+	if (!solr_add_definite_query_args(str, args, and_args)) {
+		/* can't search this query */
+		return 0;
+	}
+
+	/* use a separate filter query for selecting the mailbox. it shouldn't
+	   affect the score and there could be some caching benefits too. */
+	str_append(str, "&fq=%2Buser:");
+	solr_quote_http(str, box->storage->user->username);
+	box_name = fts_box_get_root(box, &ns);
+	str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
+	solr_quote_http(str, box_name);
+	solr_add_ns_query_http(str, backend, ns);
+
+	pool = pool_alloconly_create("fts solr search", 1024);
+	ret = solr_connection_select(backend->solr_conn, str_c(str),
+				     pool, &results);
+	if (ret == 0 && results[0] != NULL) {
+		if ((flags & FTS_LOOKUP_FLAG_NO_AUTO_FUZZY) == 0)
+			array_append_array(&result->definite_uids, &results[0]->uids);
+		else
+			array_append_array(&result->maybe_uids, &results[0]->uids);
+		array_append_array(&result->scores, &results[0]->scores);
+	}
+	result->scores_sorted = TRUE;
+	pool_unref(&pool);
+	return ret;
+}
+
+static char *
+mailbox_get_id(struct solr_fts_backend *backend, struct mail_namespace *ns,
+	       const char *mailbox, uint32_t uidvalidity)
+{
+	string_t *str = t_str_new(64);
+
+	str_printfa(str, "%u\001", uidvalidity);
+	str_append(str, mailbox);
+	if (ns != backend->default_ns)
+		str_printfa(str, "\001%s", ns->prefix);
+	return str_c_modifiable(str);
+}
+
+static int
+solr_search_multi(struct solr_fts_backend *backend, string_t *str,
+		  struct mailbox *const boxes[],
+		  enum fts_lookup_flags flags,
+		  struct fts_multi_result *result)
+{
+	struct solr_result **solr_results;
+	struct fts_result *fts_result;
+	ARRAY(struct fts_result) fts_results;
+	struct mail_namespace *ns;
+	struct mailbox_status status;
+	HASH_TABLE(char *, struct mailbox *) mailboxes;
+	struct mailbox *box;
+	const char *box_name;
+	char *box_id;
+	unsigned int i;
+	size_t len;
+
+	/* use a separate filter query for selecting the mailbox. it shouldn't
+	   affect the score and there could be some caching benefits too. */
+	str_append(str, "&fq=%2Buser:");
+	if (backend->backend.ns->owner != NULL)
+		solr_quote_http(str, backend->backend.ns->owner->username);
+	else
+		str_append(str, "%22%22");
+
+	hash_table_create(&mailboxes, default_pool, 0, str_hash, strcmp);
+	str_append(str, "%2B(");
+	len = str_len(str);
+	for (i = 0; boxes[i] != NULL; i++) {
+		if (str_len(str) != len)
+			str_append(str, "+OR+");
+
+		box_name = fts_box_get_root(boxes[i], &ns);
+		mailbox_get_open_status(boxes[i], STATUS_UIDVALIDITY, &status);
+		str_printfa(str, "%%2B(%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
+		solr_quote_http(str, box_name);
+		solr_add_ns_query_http(str, backend, ns);
+		str_append_c(str, ')');
+
+		box_id = mailbox_get_id(backend, ns, box_name, status.uidvalidity);
+		hash_table_insert(mailboxes, box_id, boxes[i]);
+	}
+	str_append_c(str, ')');
+
+	if (solr_connection_select(backend->solr_conn, str_c(str),
+				   result->pool, &solr_results) < 0) {
+		hash_table_destroy(&mailboxes);
+		return -1;
+	}
+
+	p_array_init(&fts_results, result->pool, 32);
+	for (i = 0; solr_results[i] != NULL; i++) {
+		box = hash_table_lookup(mailboxes, solr_results[i]->box_id);
+		if (box == NULL) {
+			i_warning("fts_solr: Lookup returned unexpected mailbox "
+				  "with id=%s", solr_results[i]->box_id);
+			continue;
+		}
+		fts_result = array_append_space(&fts_results);
+		fts_result->box = box;
+		if ((flags & FTS_LOOKUP_FLAG_NO_AUTO_FUZZY) == 0)
+			fts_result->definite_uids = solr_results[i]->uids;
+		else
+			fts_result->maybe_uids = solr_results[i]->uids;
+		fts_result->scores = solr_results[i]->scores;
+		fts_result->scores_sorted = TRUE;
+	}
+	array_append_zero(&fts_results);
+	result->box_results = array_front_modifiable(&fts_results);
+	hash_table_destroy(&mailboxes);
+	return 0;
+}
+
+static int
+fts_backend_solr_lookup_multi(struct fts_backend *_backend,
+			      struct mailbox *const boxes[],
+			      struct mail_search_arg *args,
+			      enum fts_lookup_flags flags,
+			      struct fts_multi_result *result)
+{
+	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)_backend;
+	string_t *str;
+
+	fts_solr_set_default_ns(backend);
+
+	str = t_str_new(256);
+	str_printfa(str, "fl=ns,box,uidv,uid,score&rows=%u&sort=box+asc,uid+asc&q=%%7b!lucene+q.op%%3dAND%%7d",
+		    SOLR_MAX_MULTI_ROWS);
+
+	if (solr_add_definite_query_args(str, args, and_args)) {
+		if (solr_search_multi(backend, str, boxes, flags, result) < 0)
+			return -1;
+	}
+	/* FIXME: maybe_uids could be handled also with some more work.. */
+	return 0;
+}
+
+struct fts_backend fts_backend_solr_old = {
+	.name = "solr_old",
+	.flags = 0,
+
+	{
+		fts_backend_solr_alloc,
+		fts_backend_solr_init,
+		fts_backend_solr_deinit,
+		fts_backend_solr_get_last_uid,
+		fts_backend_solr_update_init,
+		fts_backend_solr_update_deinit,
+		fts_backend_solr_update_set_mailbox,
+		fts_backend_solr_update_expunge,
+		fts_backend_solr_update_set_build_key,
+		fts_backend_solr_update_unset_build_key,
+		fts_backend_solr_update_build_more,
+		fts_backend_solr_refresh,
+		NULL,
+		fts_backend_solr_optimize,
+		fts_backend_default_can_lookup,
+		fts_backend_solr_lookup,
+		fts_backend_solr_lookup_multi,
+		NULL
+	}
+};
diff --git a/src/plugins/fts-solr/fts-backend-solr.c b/src/plugins/fts-solr/fts-backend-solr.c
new file mode 100644
index 0000000..0ac0f18
--- /dev/null
+++ b/src/plugins/fts-solr/fts-backend-solr.c
@@ -0,0 +1,984 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "hash.h"
+#include "strescape.h"
+#include "unichar.h"
+#include "iostream-ssl.h"
+#include "http-url.h"
+#include "mail-storage-private.h"
+#include "mailbox-list-private.h"
+#include "mail-search.h"
+#include "fts-api.h"
+#include "solr-connection.h"
+#include "fts-solr-plugin.h"
+
+#include <ctype.h>
+
+#define SOLR_CMDBUF_SIZE (1024*64)
+#define SOLR_CMDBUF_FLUSH_SIZE (SOLR_CMDBUF_SIZE-128)
+#define SOLR_MAX_MULTI_ROWS 100000
+
+/* If header is larger than this, truncate it. */
+#define SOLR_HEADER_MAX_SIZE (1024*1024)
+/* If SOLR_HEADER_MAX_SIZE was already reached, write still to individual
+   header fields as long as they're smaller than this */
+#define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024
+
+#define SOLR_QUERY_MAX_MAILBOX_COUNT 10
+
+struct solr_fts_backend {
+	struct fts_backend backend;
+	struct solr_connection *solr_conn;
+};
+
+struct solr_fts_field {
+	char *key;
+	string_t *value;
+};
+
+struct solr_fts_backend_update_context {
+	struct fts_backend_update_context ctx;
+
+	struct mailbox *cur_box;
+	char box_guid[MAILBOX_GUID_HEX_LENGTH+1];
+
+	struct solr_connection_post *post;
+	uint32_t prev_uid;
+	string_t *cmd, *cur_value, *cur_value2;
+	string_t *cmd_expunge;
+	ARRAY(struct solr_fts_field) fields;
+
+	uint32_t last_indexed_uid;
+	unsigned int mails_since_flush;
+
+	bool tokenized_input:1;
+	bool last_indexed_uid_set:1;
+	bool body_open:1;
+	bool documents_added:1;
+	bool expunges:1;
+	bool truncate_header:1;
+};
+
+static const char *solr_escape_chars = "+-&|!(){}[]^\"~*?:\\/ ";
+
+static bool is_valid_xml_char(unichar_t chr)
+{
+	/* Valid characters in XML:
+
+	   #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] |
+	   [#x10000-#x10FFFF]
+
+	   This function gets called only for #x80 and higher */
+	if (chr > 0xd7ff && chr < 0xe000)
+		return FALSE;
+	if (chr > 0xfffd && chr < 0x10000)
+		return FALSE;
+	return chr < 0x10ffff;
+}
+
+static size_t
+xml_encode_data_max(string_t *dest, const unsigned char *data, size_t len,
+		    unsigned int max_len)
+{
+	unichar_t chr;
+	size_t i;
+
+	i_assert(max_len > 0 || len == 0);
+
+	if (max_len > len)
+		max_len = len;
+	for (i = 0; i < max_len; i++) {
+		switch (data[i]) {
+		case '&':
+			str_append(dest, "&amp;");
+			break;
+		case '<':
+			str_append(dest, "&lt;");
+			break;
+		case '>':
+			str_append(dest, "&gt;");
+			break;
+		case '\t':
+		case '\n':
+		case '\r':
+			/* exceptions to the following control char check */
+			str_append_c(dest, data[i]);
+			break;
+		default:
+			if (data[i] < 32) {
+				/* SOLR doesn't like control characters.
+				   replace them with spaces. */
+				str_append_c(dest, ' ');
+			} else if (data[i] >= 0x80) {
+				/* make sure the character is valid for XML
+				   so we don't get XML parser errors */
+				int char_len =
+					uni_utf8_get_char_n(data + i, len - i, &chr);
+				i_assert(char_len > 0); /* input is valid UTF8 */
+				if (is_valid_xml_char(chr))
+					str_append_data(dest, data + i, char_len);
+				else {
+					str_append_data(dest, utf8_replacement_char,
+							UTF8_REPLACEMENT_CHAR_LEN);
+				}
+				i += char_len - 1;
+			} else {
+				str_append_c(dest, data[i]);
+			}
+			break;
+		}
+	}
+	return i;
+}
+
+static void
+xml_encode_data(string_t *dest, const unsigned char *data, size_t len)
+{
+	(void)xml_encode_data_max(dest, data, len, len);
+}
+
+static void xml_encode(string_t *dest, const char *str)
+{
+	xml_encode_data(dest, (const unsigned char *)str, strlen(str));
+}
+
+static const char *solr_escape(const char *str)
+{
+	string_t *ret;
+	unsigned int i;
+
+	ret = t_str_new(strlen(str) + 16);
+	for (i = 0; str[i] != '\0'; i++) {
+		if (strchr(solr_escape_chars, str[i]) != NULL)
+			str_append_c(ret, '\\');
+		str_append_c(ret, str[i]);
+	}
+	return str_c(ret);
+}
+
+static void solr_quote_http(string_t *dest, const char *str)
+{
+	if (str[0] != '\0')
+		http_url_escape_param(dest, solr_escape(str));
+	else
+		str_append(dest, "%22%22");
+}
+
+static struct fts_backend *fts_backend_solr_alloc(void)
+{
+	struct solr_fts_backend *backend;
+
+	backend = i_new(struct solr_fts_backend, 1);
+	backend->backend = fts_backend_solr;
+	return &backend->backend;
+}
+
+static int
+fts_backend_solr_init(struct fts_backend *_backend, const char **error_r)
+{
+	struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user);
+	struct ssl_iostream_settings ssl_set;
+
+	if (fuser == NULL) {
+		*error_r = "Invalid fts_solr setting";
+		return -1;
+	}
+	if (fuser->set.use_libfts) {
+		/* change our flags so we get proper input */
+		_backend->flags &= ENUM_NEGATE(FTS_BACKEND_FLAG_FUZZY_SEARCH);
+		_backend->flags |= FTS_BACKEND_FLAG_TOKENIZED_INPUT;
+	}
+
+	mail_user_init_ssl_client_settings(_backend->ns->user, &ssl_set);
+	return solr_connection_init(&fuser->set, &ssl_set,
+				    _backend->ns->user->event,
+				    &backend->solr_conn, error_r);
+}
+
+static void fts_backend_solr_deinit(struct fts_backend *_backend)
+{
+	struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+
+	solr_connection_deinit(&backend->solr_conn);
+	i_free(backend);
+}
+
+static int
+get_last_uid_fallback(struct fts_backend *_backend, struct mailbox *box,
+		      uint32_t *last_uid_r)
+{
+	struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+	const struct seq_range *uidvals;
+	const char *box_guid;
+	unsigned int count;
+	struct solr_result **results;
+	string_t *str;
+	pool_t pool;
+	int ret = 0;
+
+	str = t_str_new(256);
+	str_append(str, "wt=xml&fl=uid&rows=1&sort=uid+desc&q=");
+
+	if (fts_mailbox_get_guid(box, &box_guid) < 0)
+		return -1;
+
+	str_printfa(str, "box:%s+AND+user:", box_guid);
+	if (_backend->ns->owner != NULL)
+		solr_quote_http(str, _backend->ns->owner->username);
+	else
+		str_append(str, "%22%22");
+
+	pool = pool_alloconly_create("solr last uid lookup", 1024);
+	if (solr_connection_select(backend->solr_conn, str_c(str),
+				   pool, &results) < 0)
+		ret = -1;
+	else if (results[0] == NULL) {
+		/* no UIDs */
+		*last_uid_r = 0;
+	} else {
+		uidvals = array_get(&results[0]->uids, &count);
+		i_assert(count > 0);
+		if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
+			*last_uid_r = uidvals[0].seq1;
+		} else {
+			i_error("fts_solr: Last UID lookup returned multiple rows");
+			ret = -1;
+		}
+	}
+	pool_unref(&pool);
+	return ret;
+}
+
+static int
+fts_backend_solr_get_last_uid(struct fts_backend *_backend,
+			      struct mailbox *box, uint32_t *last_uid_r)
+{
+	struct fts_index_header hdr;
+
+	if (fts_index_get_header(box, &hdr)) {
+		*last_uid_r = hdr.last_indexed_uid;
+		return 0;
+	}
+
+	/* either nothing has been indexed, or the index was corrupted.
+	   do it the slow way. */
+	if (get_last_uid_fallback(_backend, box, last_uid_r) < 0)
+		return -1;
+
+	fts_index_set_last_uid(box, *last_uid_r);
+	return 0;
+}
+
+static struct fts_backend_update_context *
+fts_backend_solr_update_init(struct fts_backend *_backend)
+{
+	struct solr_fts_backend_update_context *ctx;
+
+	ctx = i_new(struct solr_fts_backend_update_context, 1);
+	ctx->ctx.backend = _backend;
+	ctx->tokenized_input =
+		(_backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0;
+	i_array_init(&ctx->fields, 16);
+	return &ctx->ctx;
+}
+
+static void xml_encode_id(struct solr_fts_backend_update_context *ctx,
+			  string_t *str, uint32_t uid)
+{
+	str_printfa(str, "%u/%s", uid, ctx->box_guid);
+	if (ctx->ctx.backend->ns->owner != NULL) {
+		str_append_c(str, '/');
+		xml_encode(str, ctx->ctx.backend->ns->owner->username);
+	}
+}
+
+static void
+fts_backend_solr_doc_open(struct solr_fts_backend_update_context *ctx,
+			  uint32_t uid)
+{
+	ctx->documents_added = TRUE;
+
+	str_printfa(ctx->cmd, "<doc>"
+		    "<field name=\"uid\">%u</field>"
+		    "<field name=\"box\">%s</field>",
+		    uid, ctx->box_guid);
+	str_append(ctx->cmd, "<field name=\"user\">");
+	if (ctx->ctx.backend->ns->owner != NULL)
+		xml_encode(ctx->cmd, ctx->ctx.backend->ns->owner->username);
+	str_append(ctx->cmd, "</field>");
+
+	str_printfa(ctx->cmd, "<field name=\"id\">");
+	xml_encode_id(ctx, ctx->cmd, uid);
+	str_append(ctx->cmd, "</field>");
+}
+
+static string_t *
+fts_solr_field_get(struct solr_fts_backend_update_context *ctx, const char *key)
+{
+	const struct solr_fts_field *field;
+	struct solr_fts_field new_field;
+
+	/* there are only a few fields. this lookup is fast enough. */
+	array_foreach(&ctx->fields, field) {
+		if (strcasecmp(field->key, key) == 0)
+			return field->value;
+	}
+
+	i_zero(&new_field);
+	new_field.key = str_lcase(i_strdup(key));
+	new_field.value = str_new(default_pool, 128);
+	array_push_back(&ctx->fields, &new_field);
+	return new_field.value;
+}
+
+static void
+fts_backend_solr_doc_close(struct solr_fts_backend_update_context *ctx)
+{
+	struct solr_fts_field *field;
+
+	if (ctx->body_open) {
+		ctx->body_open = FALSE;
+		str_append(ctx->cmd, "</field>");
+	}
+	array_foreach_modifiable(&ctx->fields, field) {
+		str_printfa(ctx->cmd, "<field name=\"%s\">", field->key);
+		/* the values are already xml-escaped */
+		str_append_str(ctx->cmd, field->value);
+		str_append(ctx->cmd, "</field>");
+		str_truncate(field->value, 0);
+	}
+	str_append(ctx->cmd, "</doc>");
+}
+
+static int
+fts_backed_solr_build_flush(struct solr_fts_backend_update_context *ctx)
+{
+	if (ctx->post == NULL)
+		return 0;
+
+	fts_backend_solr_doc_close(ctx);
+	str_append(ctx->cmd, "</add>");
+	ctx->mails_since_flush = 0;
+
+	solr_connection_post_more(ctx->post, str_data(ctx->cmd),
+				  str_len(ctx->cmd));
+	str_truncate(ctx->cmd, 0);
+	return solr_connection_post_end(&ctx->post);
+}
+
+static void
+fts_backend_solr_expunge_flush(struct solr_fts_backend_update_context *ctx)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)ctx->ctx.backend;
+
+	str_append(ctx->cmd_expunge, "</delete>");
+	(void)solr_connection_post(backend->solr_conn, str_c(ctx->cmd_expunge));
+	str_truncate(ctx->cmd_expunge, 0);
+	str_append(ctx->cmd_expunge, "<delete>");
+}
+
+static int
+fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)_ctx->backend;
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_ctx->backend->ns->user);
+	struct solr_fts_field *field;
+	const char *str;
+	int ret = _ctx->failed ? -1 : 0;
+
+	if (fts_backed_solr_build_flush(ctx) < 0)
+		ret = -1;
+
+	if (ctx->documents_added || ctx->expunges) {
+		/* commit and wait until the documents we just indexed are
+		   visible to the following search */
+		if (ctx->expunges)
+			fts_backend_solr_expunge_flush(ctx);
+		if (fuser->set.soft_commit) {
+			str = t_strdup_printf("<commit softCommit=\"true\" waitSearcher=\"%s\"/>",
+					      ctx->documents_added ? "true" : "false");
+			if (solr_connection_post(backend->solr_conn, str) < 0)
+				ret = -1;
+		}
+	}
+
+	str_free(&ctx->cmd);
+	str_free(&ctx->cmd_expunge);
+	array_foreach_modifiable(&ctx->fields, field) {
+		str_free(&field->value);
+		i_free(field->key);
+	}
+	array_free(&ctx->fields);
+	i_free(ctx);
+	return ret;
+}
+
+static void
+fts_backend_solr_update_set_mailbox(struct fts_backend_update_context *_ctx,
+				    struct mailbox *box)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+	const char *box_guid;
+
+	if (ctx->prev_uid != 0) {
+		i_assert(ctx->cur_box != NULL);
+
+		/* flush solr between mailboxes, so we don't wrongly update
+		   last_uid before we know it has succeeded */
+		if (fts_backed_solr_build_flush(ctx) < 0)
+			_ctx->failed = TRUE;
+		else if (!_ctx->failed)
+			fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid);
+		ctx->prev_uid = 0;
+	}
+
+	if (box != NULL) {
+		if (fts_mailbox_get_guid(box, &box_guid) < 0)
+			_ctx->failed = TRUE;
+
+		i_assert(strlen(box_guid) == sizeof(ctx->box_guid)-1);
+		memcpy(ctx->box_guid, box_guid, sizeof(ctx->box_guid)-1);
+	} else {
+		memset(ctx->box_guid, 0, sizeof(ctx->box_guid));
+	}
+	ctx->cur_box = box;
+}
+
+static void
+fts_backend_solr_update_expunge(struct fts_backend_update_context *_ctx,
+				uint32_t uid)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+	struct fts_index_header hdr;
+
+	if (!ctx->last_indexed_uid_set) {
+		if (!fts_index_get_header(ctx->cur_box, &hdr))
+			ctx->last_indexed_uid = 0;
+		else
+			ctx->last_indexed_uid = hdr.last_indexed_uid;
+		ctx->last_indexed_uid_set = TRUE;
+	}
+	if (ctx->last_indexed_uid == 0 ||
+	    uid > ctx->last_indexed_uid + 100) {
+		/* don't waste time asking Solr to expunge a message that is
+		   highly unlikely to be indexed at this time. */
+		return;
+	}
+	if (!ctx->expunges) {
+		ctx->expunges = TRUE;
+		ctx->cmd_expunge = str_new(default_pool, 1024);
+		str_append(ctx->cmd_expunge, "<delete>");
+	}
+
+	if (str_len(ctx->cmd_expunge) >= SOLR_CMDBUF_FLUSH_SIZE)
+		fts_backend_solr_expunge_flush(ctx);
+
+	str_append(ctx->cmd_expunge, "<id>");
+	xml_encode_id(ctx, ctx->cmd_expunge, uid);
+	str_append(ctx->cmd_expunge, "</id>");
+}
+
+static void
+fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx,
+			     uint32_t uid)
+{
+	struct solr_fts_backend *backend =
+		(struct solr_fts_backend *)ctx->ctx.backend;
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(ctx->ctx.backend->ns->user);
+
+	if (ctx->mails_since_flush >= fuser->set.batch_size) {
+		if (fts_backed_solr_build_flush(ctx) < 0)
+			ctx->ctx.failed = TRUE;
+	}
+	ctx->mails_since_flush++;
+	if (ctx->post == NULL) {
+		if (ctx->cmd == NULL)
+			ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
+		ctx->post = solr_connection_post_begin(backend->solr_conn);
+		str_append(ctx->cmd, "<add>");
+	} else {
+		fts_backend_solr_doc_close(ctx);
+	}
+	ctx->prev_uid = uid;
+	ctx->truncate_header = FALSE;
+	fts_backend_solr_doc_open(ctx, uid);
+}
+
+static bool
+fts_backend_solr_update_set_build_key(struct fts_backend_update_context *_ctx,
+				      const struct fts_backend_build_key *key)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+
+	if (key->uid != ctx->prev_uid)
+		fts_backend_solr_uid_changed(ctx, key->uid);
+
+	switch (key->type) {
+	case FTS_BACKEND_BUILD_KEY_HDR:
+		if (fts_header_want_indexed(key->hdr_name)) {
+			ctx->cur_value2 =
+				fts_solr_field_get(ctx, key->hdr_name);
+		}
+		/* fall through */
+	case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+		ctx->cur_value = fts_solr_field_get(ctx, "hdr");
+		xml_encode(ctx->cur_value, key->hdr_name);
+		str_append(ctx->cur_value, ": ");
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART:
+		if (!ctx->body_open) {
+			ctx->body_open = TRUE;
+			str_append(ctx->cmd, "<field name=\"body\">");
+		}
+		ctx->cur_value = ctx->cmd;
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+		i_unreached();
+	}
+	return TRUE;
+}
+
+static void
+fts_backend_solr_update_unset_build_key(struct fts_backend_update_context *_ctx)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+
+	/* There can be multiple duplicate keys (duplicate header lines,
+	   multiple MIME body parts). Make sure they are separated by
+	   whitespace. */
+	str_append_c(ctx->cur_value, '\n');
+	ctx->cur_value = NULL;
+	if (ctx->cur_value2 != NULL) {
+		str_append_c(ctx->cur_value2, '\n');
+		ctx->cur_value2 = NULL;
+	}
+}
+
+static int
+fts_backend_solr_update_build_more(struct fts_backend_update_context *_ctx,
+				   const unsigned char *data, size_t size)
+{
+	struct solr_fts_backend_update_context *ctx =
+		(struct solr_fts_backend_update_context *)_ctx;
+	size_t len;
+
+	if (_ctx->failed)
+		return -1;
+
+	if (ctx->cur_value2 == NULL && ctx->cur_value == ctx->cmd) {
+		/* we're writing to message body. if size is huge,
+		   flush it once in a while */
+		while (size >= SOLR_CMDBUF_FLUSH_SIZE) {
+			if (str_len(ctx->cmd) >= SOLR_CMDBUF_FLUSH_SIZE) {
+				solr_connection_post_more(ctx->post,
+							  str_data(ctx->cmd),
+							  str_len(ctx->cmd));
+				str_truncate(ctx->cmd, 0);
+			}
+			len = xml_encode_data_max(ctx->cmd, data, size,
+						  SOLR_CMDBUF_FLUSH_SIZE -
+						  str_len(ctx->cmd));
+			i_assert(len > 0);
+			i_assert(len <= size);
+			data += len;
+			size -= len;
+		}
+		xml_encode_data(ctx->cmd, data, size);
+		if (ctx->tokenized_input)
+			str_append_c(ctx->cmd, ' ');
+	} else {
+		if (!ctx->truncate_header) {
+			xml_encode_data(ctx->cur_value, data, size);
+			if (ctx->tokenized_input)
+				str_append_c(ctx->cur_value, ' ');
+		}
+		if (ctx->cur_value2 != NULL &&
+		    (!ctx->truncate_header ||
+		     str_len(ctx->cur_value2) < SOLR_HEADER_LINE_MAX_TRUNC_SIZE)) {
+			xml_encode_data(ctx->cur_value2, data, size);
+			if (ctx->tokenized_input)
+				str_append_c(ctx->cur_value2, ' ');
+		}
+	}
+
+	if (str_len(ctx->cmd) >= SOLR_CMDBUF_FLUSH_SIZE) {
+		solr_connection_post_more(ctx->post, str_data(ctx->cmd),
+					  str_len(ctx->cmd));
+		str_truncate(ctx->cmd, 0);
+	}
+	if (!ctx->truncate_header &&
+	    str_len(ctx->cur_value) >= SOLR_HEADER_MAX_SIZE) {
+		/* a large header */
+		i_assert(ctx->cur_value != ctx->cmd);
+
+		i_warning("fts-solr(%s): Mailbox %s UID=%u header size is huge, truncating",
+			  ctx->cur_box->storage->user->username,
+			  mailbox_get_vname(ctx->cur_box), ctx->prev_uid);
+		ctx->truncate_header = TRUE;
+	}
+	return 0;
+}
+
+static int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED)
+{
+	return 0;
+}
+
+static int fts_backend_solr_rescan(struct fts_backend *backend)
+{
+	/* FIXME: proper rescan needed. for now we'll just reset the
+	   last-uids */
+	return fts_backend_reset_last_uids(backend);
+}
+
+static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED)
+{
+	return 0;
+}
+
+static bool solr_need_escaping(const char *str)
+{
+	for (; *str != '\0'; str++) {
+		if (strchr(solr_escape_chars, *str) != NULL)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static void solr_add_str_arg(string_t *str, struct mail_search_arg *arg)
+{
+	/* currently we'll just disable fuzzy searching if there are any
+	   parameters that need escaping. solr doesn't seem to give good
+	   fuzzy results even if we did escape them.. */
+	if (!arg->fuzzy || arg->value.str[0] == '\0' ||
+	    solr_need_escaping(arg->value.str))
+		solr_quote_http(str, arg->value.str);
+	else {
+		http_url_escape_param(str, arg->value.str);
+		str_append_c(str, '~');
+	}
+}
+
+static bool
+solr_add_definite_query(string_t *str, struct mail_search_arg *arg)
+{
+	if (arg->no_fts)
+		return FALSE;
+	switch (arg->type) {
+	case SEARCH_TEXT: {
+		if (arg->match_not)
+			str_append_c(str, '-');
+		str_append(str, "(hdr:");
+		solr_add_str_arg(str, arg);
+		str_append(str, "+OR+body:");
+		solr_add_str_arg(str, arg);
+		str_append(str, ")");
+		break;
+	}
+	case SEARCH_BODY:
+		if (arg->match_not)
+			str_append_c(str, '-');
+		str_append(str, "body:");
+		solr_add_str_arg(str, arg);
+		break;
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+		if (!fts_header_want_indexed(arg->hdr_field_name))
+			return FALSE;
+
+		if (arg->match_not)
+			str_append_c(str, '-');
+		str_append(str, t_str_lcase(arg->hdr_field_name));
+		str_append_c(str, ':');
+		solr_add_str_arg(str, arg);
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static bool
+solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg,
+			     bool and_args)
+{
+	size_t last_len;
+
+	last_len = str_len(str);
+	for (; arg != NULL; arg = arg->next) {
+		if (solr_add_definite_query(str, arg)) {
+			arg->match_always = TRUE;
+			last_len = str_len(str);
+			if (and_args)
+				str_append(str, "+AND+");
+			else
+				str_append(str, "+OR+");
+		}
+	}
+	if (str_len(str) == last_len)
+		return FALSE;
+
+	str_truncate(str, last_len);
+	return TRUE;
+}
+
+static bool
+solr_add_maybe_query(string_t *str, struct mail_search_arg *arg)
+{
+	if (arg->no_fts)
+		return FALSE;
+	switch (arg->type) {
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+		if (fts_header_want_indexed(arg->hdr_field_name))
+			return FALSE;
+		if (arg->match_not) {
+			/* all matches would be definite, but all non-matches
+			   would be maybies. too much trouble to optimize. */
+			return FALSE;
+		}
+
+		/* we can check if the search key exists in some header and
+		   filter out the messages that have no chance of matching */
+		str_append(str, "hdr:");
+		if (*arg->value.str != '\0')
+			solr_quote_http(str, arg->value.str);
+		else {
+			/* checking potential existence of the header name */
+			solr_quote_http(str, t_str_lcase(arg->hdr_field_name));
+		}
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static bool
+solr_add_maybe_query_args(string_t *str, struct mail_search_arg *arg,
+			  bool and_args)
+{
+	size_t last_len;
+
+	last_len = str_len(str);
+	for (; arg != NULL; arg = arg->next) {
+		if (solr_add_maybe_query(str, arg)) {
+			arg->match_always = TRUE;
+			last_len = str_len(str);
+			if (and_args)
+				str_append(str, "+AND+");
+			else
+				str_append(str, "+OR+");
+		}
+	}
+	if (str_len(str) == last_len)
+		return FALSE;
+
+	str_truncate(str, last_len);
+	return TRUE;
+}
+
+static int solr_search(struct fts_backend *_backend, string_t *str,
+		       const char *box_guid, ARRAY_TYPE(seq_range) *uids_r,
+		       ARRAY_TYPE(fts_score_map) *scores_r)
+{
+	struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+	pool_t pool = pool_alloconly_create("fts solr search", 1024);
+	struct solr_result **results;
+	int ret;
+
+	/* use a separate filter query for selecting the mailbox. it shouldn't
+	   affect the score and there could be some caching benefits too. */
+	str_printfa(str, "&fq=%%2Bbox:%s+%%2Buser:", box_guid);
+	if (_backend->ns->owner != NULL)
+		solr_quote_http(str, _backend->ns->owner->username);
+	else
+		str_append(str, "%22%22");
+
+	ret = solr_connection_select(backend->solr_conn, str_c(str),
+				     pool, &results);
+	if (ret == 0 && results[0] != NULL) {
+		array_append_array(uids_r, &results[0]->uids);
+		array_append_array(scores_r, &results[0]->scores);
+	}
+	pool_unref(&pool);
+	return ret;
+}
+
+static int
+fts_backend_solr_lookup(struct fts_backend *_backend, struct mailbox *box,
+			struct mail_search_arg *args,
+			enum fts_lookup_flags flags,
+			struct fts_result *result)
+{
+	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
+	struct mailbox_status status;
+	string_t *str;
+	const char *box_guid;
+	size_t prefix_len;
+
+	if (fts_mailbox_get_guid(box, &box_guid) < 0)
+		return -1;
+	mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
+
+	str = t_str_new(256);
+	str_printfa(str, "wt=xml&fl=uid,score&rows=%u&sort=uid+asc&q=%%7b!lucene+q.op%%3dAND%%7d",
+		    status.uidnext);
+	prefix_len = str_len(str);
+
+	if (solr_add_definite_query_args(str, args, and_args)) {
+		ARRAY_TYPE(seq_range) *uids_arr =
+			(flags & FTS_LOOKUP_FLAG_NO_AUTO_FUZZY) == 0 ?
+			&result->definite_uids : &result->maybe_uids;
+		if (solr_search(_backend, str, box_guid,
+				uids_arr, &result->scores) < 0)
+			return -1;
+	}
+	str_truncate(str, prefix_len);
+	if (solr_add_maybe_query_args(str, args, and_args)) {
+		if (solr_search(_backend, str, box_guid,
+				&result->maybe_uids, &result->scores) < 0)
+			return -1;
+	}               
+	result->scores_sorted = TRUE;
+	return 0;
+}
+
+static int
+solr_search_multi(struct fts_backend *_backend, string_t *str,
+		  struct mailbox *const boxes[], enum fts_lookup_flags flags,
+		  struct fts_multi_result *result)
+{
+	struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+	struct solr_result **solr_results;
+	struct fts_result *fts_result;
+	ARRAY(struct fts_result) fts_results;
+	HASH_TABLE(char *, struct mailbox *) mailboxes;
+	struct mailbox *box;
+	const char *box_guid;
+	unsigned int i;
+	size_t len;
+	bool search_all_mailboxes;
+
+	/* use a separate filter query for selecting the mailbox. it shouldn't
+	   affect the score and there could be some caching benefits too. */
+	str_append(str, "&fq=%2Buser:");
+	if (_backend->ns->owner != NULL)
+		solr_quote_http(str, _backend->ns->owner->username);
+	else
+		str_append(str, "%22%22");
+
+	hash_table_create(&mailboxes, default_pool, 0, str_hash, strcmp);
+	for (i = 0; boxes[i] != NULL; i++) ;
+	search_all_mailboxes = i > SOLR_QUERY_MAX_MAILBOX_COUNT;
+	if (!search_all_mailboxes)
+		str_append(str, "+%2B(");
+	len = str_len(str);
+
+	for (i = 0; boxes[i] != NULL; i++) {
+		if (fts_mailbox_get_guid(boxes[i], &box_guid) < 0)
+			continue;
+
+		if (!search_all_mailboxes) {
+			if (str_len(str) != len)
+				str_append(str, "+OR+");
+			str_printfa(str, "box:%s", box_guid);
+		}
+		hash_table_insert(mailboxes, t_strdup_noconst(box_guid),
+				  boxes[i]);
+	}
+	if (!search_all_mailboxes)
+		str_append_c(str, ')');
+
+	if (solr_connection_select(backend->solr_conn, str_c(str),
+				   result->pool, &solr_results) < 0) {
+		hash_table_destroy(&mailboxes);
+		return -1;
+	}
+
+	p_array_init(&fts_results, result->pool, 32);
+	for (i = 0; solr_results[i] != NULL; i++) {
+		box = hash_table_lookup(mailboxes, solr_results[i]->box_id);
+		if (box == NULL) {
+			if (!search_all_mailboxes) {
+				i_warning("fts_solr: Lookup returned unexpected mailbox "
+					  "with guid=%s", solr_results[i]->box_id);
+			}
+			continue;
+		}
+		fts_result = array_append_space(&fts_results);
+		fts_result->box = box;
+		if ((flags & FTS_LOOKUP_FLAG_NO_AUTO_FUZZY) == 0)
+			fts_result->definite_uids = solr_results[i]->uids;
+		else
+			fts_result->maybe_uids = solr_results[i]->uids;
+		fts_result->scores = solr_results[i]->scores;
+		fts_result->scores_sorted = TRUE;
+	}
+	array_append_zero(&fts_results);
+	result->box_results = array_front_modifiable(&fts_results);
+	hash_table_destroy(&mailboxes);
+	return 0;
+}
+
+static int
+fts_backend_solr_lookup_multi(struct fts_backend *backend,
+			      struct mailbox *const boxes[],
+			      struct mail_search_arg *args,
+			      enum fts_lookup_flags flags,
+			      struct fts_multi_result *result)
+{
+	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
+	string_t *str;
+
+	str = t_str_new(256);
+	str_printfa(str, "wt=xml&fl=box,uid,score&rows=%u&sort=box+asc,uid+asc&q=%%7b!lucene+q.op%%3dAND%%7d",
+		    SOLR_MAX_MULTI_ROWS);
+
+	if (solr_add_definite_query_args(str, args, and_args)) {
+		if (solr_search_multi(backend, str, boxes, flags, result) < 0)
+			return -1;
+	}
+	/* FIXME: maybe_uids could be handled also with some more work.. */
+	return 0;
+}
+
+struct fts_backend fts_backend_solr = {
+	.name = "solr",
+	.flags = FTS_BACKEND_FLAG_FUZZY_SEARCH,
+
+	{
+		fts_backend_solr_alloc,
+		fts_backend_solr_init,
+		fts_backend_solr_deinit,
+		fts_backend_solr_get_last_uid,
+		fts_backend_solr_update_init,
+		fts_backend_solr_update_deinit,
+		fts_backend_solr_update_set_mailbox,
+		fts_backend_solr_update_expunge,
+		fts_backend_solr_update_set_build_key,
+		fts_backend_solr_update_unset_build_key,
+		fts_backend_solr_update_build_more,
+		fts_backend_solr_refresh,
+		fts_backend_solr_rescan,
+		fts_backend_solr_optimize,
+		fts_backend_default_can_lookup,
+		fts_backend_solr_lookup,
+		fts_backend_solr_lookup_multi,
+		NULL
+	}
+};
diff --git a/src/plugins/fts-solr/fts-solr-plugin.c b/src/plugins/fts-solr/fts-solr-plugin.c
new file mode 100644
index 0000000..5899784
--- /dev/null
+++ b/src/plugins/fts-solr/fts-solr-plugin.c
@@ -0,0 +1,131 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "http-client.h"
+#include "mail-user.h"
+#include "mail-storage-hooks.h"
+#include "solr-connection.h"
+#include "fts-user.h"
+#include "fts-solr-plugin.h"
+
+#define DEFAULT_SOLR_BATCH_SIZE 1000
+
+const char *fts_solr_plugin_version = DOVECOT_ABI_VERSION;
+struct http_client *solr_http_client = NULL;
+
+struct fts_solr_user_module fts_solr_user_module =
+	MODULE_CONTEXT_INIT(&mail_user_module_register);
+
+static int
+fts_solr_plugin_init_settings(struct mail_user *user,
+			      struct fts_solr_settings *set, const char *str)
+{
+	const char *const *tmp;
+
+	if (str == NULL)
+		str = "";
+
+	set->batch_size = DEFAULT_SOLR_BATCH_SIZE;
+	set->soft_commit = TRUE;
+
+	for (tmp = t_strsplit_spaces(str, " "); *tmp != NULL; tmp++) {
+		if (str_begins(*tmp, "url=")) {
+			set->url = p_strdup(user->pool, *tmp + 4);
+		} else if (strcmp(*tmp, "debug") == 0) {
+			set->debug = TRUE;
+		} else if (strcmp(*tmp, "use_libfts") == 0) {
+			set->use_libfts = TRUE;
+		} else if (str_begins(*tmp, "default_ns=")) {
+			set->default_ns_prefix =
+				p_strdup(user->pool, *tmp + 11);
+		} else if (str_begins(*tmp, "rawlog_dir=")) {
+			set->rawlog_dir = p_strdup(user->pool, *tmp + 11);
+		} else if (str_begins(*tmp, "batch_size=")) {
+			if (str_to_uint(*tmp+11, &set->batch_size) < 0 ||
+			    set->batch_size == 0) {
+				i_error("fts_solr: batch_size must be a positive integer");
+					return -1;
+			}
+		} else if (str_begins(*tmp, "soft_commit=")) {
+			if (strcmp(*tmp + 12, "yes") == 0) {
+				set->soft_commit = TRUE;
+			} else if (strcmp(*tmp + 12, "no") == 0) {
+				set->soft_commit = FALSE;
+			} else {
+				i_error("fts_solr: Invalid setting for soft_commit: %s", *tmp+12);
+				return -1;
+			}
+		} else {
+			i_error("fts_solr: Invalid setting: %s", *tmp);
+			return -1;
+		}
+	}
+	if (set->url == NULL) {
+		i_error("fts_solr: url setting missing");
+		return -1;
+	}
+	return 0;
+}
+
+static void fts_solr_mail_user_deinit(struct mail_user *user)
+{
+	struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT_REQUIRE(user);
+
+	fts_mail_user_deinit(user);
+	fuser->module_ctx.super.deinit(user);
+}
+
+static void fts_solr_mail_user_create(struct mail_user *user, const char *env)
+{
+	struct mail_user_vfuncs *v = user->vlast;
+	struct fts_solr_user *fuser;
+	const char *error;
+
+	fuser = p_new(user->pool, struct fts_solr_user, 1);
+	if (fts_solr_plugin_init_settings(user, &fuser->set, env) < 0) {
+		/* invalid settings, disabling */
+		return;
+	}
+	if (fts_mail_user_init(user, fuser->set.use_libfts, &error) < 0) {
+		i_error("fts-solr: %s", error);
+		return;
+	}
+
+	fuser->module_ctx.super = *v;
+	user->vlast = &fuser->module_ctx.super;
+	v->deinit = fts_solr_mail_user_deinit;
+	MODULE_CONTEXT_SET(user, fts_solr_user_module, fuser);
+}
+
+static void fts_solr_mail_user_created(struct mail_user *user)
+{
+	const char *env;
+
+	env = mail_user_plugin_getenv(user, "fts_solr");
+	if (env != NULL)
+		fts_solr_mail_user_create(user, env);
+}
+
+static struct mail_storage_hooks fts_solr_mail_storage_hooks = {
+	.mail_user_created = fts_solr_mail_user_created
+};
+
+void fts_solr_plugin_init(struct module *module)
+{
+	fts_backend_register(&fts_backend_solr);
+	fts_backend_register(&fts_backend_solr_old);
+	mail_storage_hooks_add(module, &fts_solr_mail_storage_hooks);
+}
+
+void fts_solr_plugin_deinit(void)
+{
+	fts_backend_unregister(fts_backend_solr.name);
+	fts_backend_unregister(fts_backend_solr_old.name);
+	mail_storage_hooks_remove(&fts_solr_mail_storage_hooks);
+	if (solr_http_client != NULL)
+		http_client_deinit(&solr_http_client);
+
+}
+
+const char *fts_solr_plugin_dependencies[] = { "fts", NULL };
diff --git a/src/plugins/fts-solr/fts-solr-plugin.h b/src/plugins/fts-solr/fts-solr-plugin.h
new file mode 100644
index 0000000..abc1b66
--- /dev/null
+++ b/src/plugins/fts-solr/fts-solr-plugin.h
@@ -0,0 +1,35 @@
+#ifndef FTS_SOLR_PLUGIN_H
+#define FTS_SOLR_PLUGIN_H
+
+#include "module-context.h"
+#include "mail-user.h"
+#include "fts-api-private.h"
+
+#define FTS_SOLR_USER_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, fts_solr_user_module)
+#define FTS_SOLR_USER_CONTEXT_REQUIRE(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, fts_solr_user_module)
+
+struct fts_solr_settings {
+	const char *url, *default_ns_prefix, *rawlog_dir;
+	unsigned int batch_size;
+	bool use_libfts;
+	bool debug;
+	bool soft_commit;
+};
+
+struct fts_solr_user {
+	union mail_user_module_context module_ctx;
+	struct fts_solr_settings set;
+};
+
+extern const char *fts_solr_plugin_dependencies[];
+extern struct fts_backend fts_backend_solr;
+extern struct fts_backend fts_backend_solr_old;
+extern MODULE_CONTEXT_DEFINE(fts_solr_user_module, &mail_user_module_register);
+extern struct http_client *solr_http_client;
+
+void fts_solr_plugin_init(struct module *module);
+void fts_solr_plugin_deinit(void);
+
+#endif
diff --git a/src/plugins/fts-solr/solr-connection.c b/src/plugins/fts-solr/solr-connection.c
new file mode 100644
index 0000000..41a4fee
--- /dev/null
+++ b/src/plugins/fts-solr/solr-connection.c
@@ -0,0 +1,327 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "strescape.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "http-url.h"
+#include "http-client.h"
+#include "fts-solr-plugin.h"
+#include "solr-connection.h"
+
+#include <expat.h>
+
+struct solr_lookup_context {
+	pool_t result_pool;
+	struct istream *payload;
+	struct io *io;
+
+	int request_status;
+
+	struct solr_response_parser *parser;
+	struct solr_result **results;
+};
+
+struct solr_connection_post {
+	struct solr_connection *conn;
+
+	struct http_client_request *http_req;
+	int request_status;
+
+	bool failed:1;
+};
+
+struct solr_connection {
+	struct event *event;
+	char *http_host;
+	in_port_t http_port;
+	char *http_base_url;
+	char *http_failure;
+	char *http_user;
+	char *http_password;
+
+	bool debug:1;
+	bool posting:1;
+	bool http_ssl:1;
+};
+
+/* Regardless of the specified URL, make sure path ends in '/' */
+static char *solr_connection_create_http_base_url(struct http_url *http_url)
+{
+	if (http_url->path == NULL)
+		return i_strconcat("/", http_url->enc_query, NULL);
+	size_t len = strlen(http_url->path);
+	if (len > 0 && http_url->path[len-1] != '/')
+		return i_strconcat(http_url->path, "/",
+				   http_url->enc_query, NULL);
+	/* http_url->path is NULL on empty path, so this is impossible. */
+	i_assert(len != 0);
+	return i_strconcat(http_url->path, http_url->enc_query, NULL);
+}
+
+int solr_connection_init(const struct fts_solr_settings *solr_set,
+			 const struct ssl_iostream_settings *ssl_client_set,
+			 struct event *event_parent,
+			 struct solr_connection **conn_r, const char **error_r)
+{
+	struct http_client_settings http_set;
+	struct solr_connection *conn;
+	struct http_url *http_url;
+	const char *error;
+
+	if (http_url_parse(solr_set->url, NULL, HTTP_URL_ALLOW_USERINFO_PART,
+			   pool_datastack_create(), &http_url, &error) < 0) {
+		*error_r = t_strdup_printf(
+			"fts_solr: Failed to parse HTTP url: %s", error);
+		return -1;
+	}
+
+	conn = i_new(struct solr_connection, 1);
+	conn->event = event_create(event_parent);
+	conn->http_host = i_strdup(http_url->host.name);
+	conn->http_port = http_url->port;
+	conn->http_base_url = solr_connection_create_http_base_url(http_url);
+	conn->http_ssl = http_url->have_ssl;
+	if (http_url->user != NULL) {
+		conn->http_user = i_strdup(http_url->user);
+		/* allow empty password */
+		conn->http_password = i_strdup(http_url->password != NULL ?
+					       http_url->password : "");
+	}
+
+	conn->debug = solr_set->debug;
+
+	if (solr_http_client == NULL) {
+		i_zero(&http_set);
+		http_set.max_idle_time_msecs = 5*1000;
+		http_set.max_parallel_connections = 1;
+		http_set.max_pipelined_requests = 1;
+		http_set.max_redirects = 1;
+		http_set.max_attempts = 3;
+		http_set.connect_timeout_msecs = 5*1000;
+		http_set.request_timeout_msecs = 60*1000;
+		http_set.ssl = ssl_client_set;
+		http_set.debug = solr_set->debug;
+		http_set.rawlog_dir = solr_set->rawlog_dir;
+		http_set.event_parent = conn->event;
+
+		/* FIXME: We should initialize a shared client instead. However,
+		          this is currently not possible due to an obscure bug
+		          in the blocking HTTP payload API, which causes
+		          conflicts with other HTTP applications like FTS Tika.
+		          Using a private client will provide a quick fix for
+		          now. */
+		solr_http_client = http_client_init_private(&http_set);
+	}
+
+	*conn_r = conn;
+	return 0;
+}
+
+void solr_connection_deinit(struct solr_connection **_conn)
+{
+	struct solr_connection *conn = *_conn;
+
+	*_conn = NULL;
+	event_unref(&conn->event);
+	i_free(conn->http_host);
+	i_free(conn->http_base_url);
+	i_free(conn->http_user);
+	i_free(conn->http_password);
+	i_free(conn);
+}
+
+static void solr_connection_payload_input(struct solr_lookup_context *lctx)
+{
+	int ret;
+
+	/* read payload */
+	ret = solr_response_parse(lctx->parser, &lctx->results);
+
+	if (ret == 0) {
+		/* we will be called again for more data */
+	} else {
+		if (lctx->payload->stream_errno != 0) {
+			i_assert(ret < 0);
+			i_error("fts_solr: "
+				"failed to read payload from HTTP server: %s",
+				i_stream_get_error(lctx->payload));
+		}
+		if (ret < 0)
+			lctx->request_status = -1;
+		solr_response_parser_deinit(&lctx->parser);
+		io_remove(&lctx->io);
+	}
+}
+
+static void
+solr_connection_select_response(const struct http_response *response,
+				struct solr_lookup_context *lctx)
+{
+	if (response->status / 100 != 2) {
+		i_error("fts_solr: Lookup failed: %s",
+			http_response_get_message(response));
+		lctx->request_status = -1;
+		return;
+	}
+
+	if (response->payload == NULL) {
+		i_error("fts_solr: Lookup failed: Empty response payload");
+		lctx->request_status = -1;
+		return;
+	}
+
+	lctx->parser = solr_response_parser_init(lctx->result_pool,
+						 response->payload);
+	lctx->payload = response->payload;
+	lctx->io = io_add_istream(response->payload,
+				  solr_connection_payload_input, lctx);
+	solr_connection_payload_input(lctx);
+}
+
+int solr_connection_select(struct solr_connection *conn, const char *query,
+			   pool_t pool, struct solr_result ***box_results_r)
+{
+	struct solr_lookup_context lctx;
+	struct http_client_request *http_req;
+	const char *url;
+
+	i_zero(&lctx);
+	lctx.result_pool = pool;
+
+	i_free_and_null(conn->http_failure);
+	url = t_strconcat(conn->http_base_url, "select?", query, NULL);
+
+	http_req = http_client_request(solr_http_client, "GET",
+				       conn->http_host, url,
+				       solr_connection_select_response,
+				       &lctx);
+	if (conn->http_user != NULL) {
+		http_client_request_set_auth_simple(
+			http_req, conn->http_user, conn->http_password);
+	}
+	http_client_request_set_port(http_req, conn->http_port);
+	http_client_request_set_ssl(http_req, conn->http_ssl);
+	http_client_request_submit(http_req);
+
+	lctx.request_status = 0;
+	http_client_wait(solr_http_client);
+
+	if (lctx.request_status < 0)
+		return -1;
+
+	*box_results_r = lctx.results;
+	return 0;
+}
+
+static void
+solr_connection_update_response(const struct http_response *response,
+				struct solr_connection_post *post)
+{
+	if (response->status / 100 != 2) {
+		i_error("fts_solr: Indexing failed: %s",
+			http_response_get_message(response));
+		post->request_status = -1;
+	}
+}
+
+static struct http_client_request *
+solr_connection_post_request(struct solr_connection_post *post)
+{
+	struct solr_connection *conn = post->conn;
+	struct http_client_request *http_req;
+	const char *url;
+
+	url = t_strconcat(conn->http_base_url, "update", NULL);
+
+	http_req = http_client_request(solr_http_client, "POST",
+				       conn->http_host, url,
+				       solr_connection_update_response, post);
+	if (conn->http_user != NULL) {
+		http_client_request_set_auth_simple(
+			http_req, conn->http_user, conn->http_password);
+	}
+	http_client_request_set_port(http_req, conn->http_port);
+	http_client_request_set_ssl(http_req, conn->http_ssl);
+	http_client_request_add_header(http_req, "Content-Type", "text/xml");
+	return http_req;
+}
+
+struct solr_connection_post *
+solr_connection_post_begin(struct solr_connection *conn)
+{
+	struct solr_connection_post *post;
+
+	i_assert(!conn->posting);
+	conn->posting = TRUE;
+
+	post = i_new(struct solr_connection_post, 1);
+	post->conn = conn;
+	post->http_req = solr_connection_post_request(post);
+	return post;
+}
+
+void solr_connection_post_more(struct solr_connection_post *post,
+			       const unsigned char *data, size_t size)
+{
+	i_assert(post->conn->posting);
+
+	if (post->failed)
+		return;
+
+	if (post->request_status == 0) {
+		(void)http_client_request_send_payload(
+			&post->http_req, data, size);
+	}
+	if (post->request_status < 0)
+		post->failed = TRUE;
+}
+
+int solr_connection_post_end(struct solr_connection_post **_post)
+{
+	struct solr_connection_post *post = *_post;
+	struct solr_connection *conn = post->conn;
+	int ret = post->failed ? -1 : 0;
+
+	i_assert(conn->posting);
+
+	*_post = NULL;
+
+	if (!post->failed) {
+		if (http_client_request_finish_payload(&post->http_req) < 0 ||
+		    post->request_status < 0) {
+			ret = -1;
+		}
+	} else {
+		http_client_request_abort(&post->http_req);
+	}
+	i_free(post);
+
+	conn->posting = FALSE;
+	return ret;
+}
+
+int solr_connection_post(struct solr_connection *conn, const char *cmd)
+{
+	struct istream *post_payload;
+	struct solr_connection_post post;
+
+	i_assert(!conn->posting);
+
+	i_zero(&post);
+	post.conn = conn;
+
+	post.http_req = solr_connection_post_request(&post);
+	post_payload = i_stream_create_from_data(cmd, strlen(cmd));
+	http_client_request_set_payload(post.http_req, post_payload, TRUE);
+	i_stream_unref(&post_payload);
+	http_client_request_submit(post.http_req);
+
+	post.request_status = 0;
+	http_client_wait(solr_http_client);
+
+	return post.request_status;
+}
diff --git a/src/plugins/fts-solr/solr-connection.h b/src/plugins/fts-solr/solr-connection.h
new file mode 100644
index 0000000..ebad8be
--- /dev/null
+++ b/src/plugins/fts-solr/solr-connection.h
@@ -0,0 +1,26 @@
+#ifndef SOLR_CONNECTION_H
+#define SOLR_CONNECTION_H
+
+#include "solr-response.h"
+
+struct solr_connection;
+struct fts_solr_settings;
+
+int solr_connection_init(const struct fts_solr_settings *solr_set,
+			 const struct ssl_iostream_settings *ssl_client_set,
+			 struct event *event_parent,
+			 struct solr_connection **conn_r,
+			 const char **error_r);
+void solr_connection_deinit(struct solr_connection **conn);
+
+int solr_connection_select(struct solr_connection *conn, const char *query,
+			   pool_t pool, struct solr_result ***box_results_r);
+int solr_connection_post(struct solr_connection *conn, const char *cmd);
+
+struct solr_connection_post *
+solr_connection_post_begin(struct solr_connection *conn);
+void solr_connection_post_more(struct solr_connection_post *post,
+			       const unsigned char *data, size_t size);
+int solr_connection_post_end(struct solr_connection_post **post);
+
+#endif
diff --git a/src/plugins/fts-solr/solr-response.c b/src/plugins/fts-solr/solr-response.c
new file mode 100644
index 0000000..65a6a1f
--- /dev/null
+++ b/src/plugins/fts-solr/solr-response.c
@@ -0,0 +1,372 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "istream.h"
+#include "solr-response.h"
+
+#include <expat.h>
+
+#define MAX_VALUE_LEN 2048
+
+enum solr_xml_response_state {
+	SOLR_XML_RESPONSE_STATE_ROOT,
+	SOLR_XML_RESPONSE_STATE_RESPONSE,
+	SOLR_XML_RESPONSE_STATE_RESULT,
+	SOLR_XML_RESPONSE_STATE_DOC,
+	SOLR_XML_RESPONSE_STATE_CONTENT
+};
+
+enum solr_xml_content_state {
+	SOLR_XML_CONTENT_STATE_NONE = 0,
+	SOLR_XML_CONTENT_STATE_UID,
+	SOLR_XML_CONTENT_STATE_SCORE,
+	SOLR_XML_CONTENT_STATE_MAILBOX,
+	SOLR_XML_CONTENT_STATE_NAMESPACE,
+	SOLR_XML_CONTENT_STATE_UIDVALIDITY,
+	SOLR_XML_CONTENT_STATE_ERROR
+};
+
+struct solr_response_parser {
+	XML_Parser xml_parser;
+	struct istream *input;
+
+	enum solr_xml_response_state state;
+	enum solr_xml_content_state content_state;
+	int depth;
+	string_t *buffer;
+
+	uint32_t uid, uidvalidity;
+	float score;
+	char *mailbox, *ns;
+
+	pool_t result_pool;
+	/* box_id -> solr_result */
+	HASH_TABLE(char *, struct solr_result *) mailboxes;
+	ARRAY(struct solr_result *) results;
+
+	bool xml_failed:1;
+};
+
+static int
+solr_xml_parse(struct solr_response_parser *parser,
+	       const void *data, size_t size, bool done)
+{
+	enum XML_Error err;
+	int line, col;
+
+	if (parser->xml_failed)
+		return -1;
+
+	if (XML_Parse(parser->xml_parser, data, size, done ? 1 : 0) != 0)
+		return 0;
+
+	err = XML_GetErrorCode(parser->xml_parser);
+	if (err != XML_ERROR_FINISHED) {
+		line = XML_GetCurrentLineNumber(parser->xml_parser);
+		col = XML_GetCurrentColumnNumber(parser->xml_parser);
+		i_error("fts_solr: Invalid XML input at %d:%d: %s "
+			"(near: %.*s)", line, col, XML_ErrorString(err),
+			(int)I_MIN(size, 128), (const char *)data);
+		parser->xml_failed = TRUE;
+		return -1;
+	}
+	return 0;
+}
+
+static const char *attrs_get_name(const char **attrs)
+{
+	for (; *attrs != NULL; attrs += 2) {
+		if (strcmp(attrs[0], "name") == 0)
+			return attrs[1];
+	}
+	return "";
+}
+
+static void
+solr_lookup_xml_start(void *context, const char *name, const char **attrs)
+{
+	struct solr_response_parser *parser = context;
+	const char *name_attr;
+
+	i_assert(parser->depth >= (int)parser->state);
+
+	parser->depth++;
+	if (parser->depth - 1 > (int)parser->state) {
+		/* skipping over unwanted elements */
+		return;
+	}
+
+	str_truncate(parser->buffer, 0);
+
+	/* response -> result -> doc */
+	switch (parser->state) {
+	case SOLR_XML_RESPONSE_STATE_ROOT:
+		if (strcmp(name, "response") == 0)
+			parser->state++;
+		break;
+	case SOLR_XML_RESPONSE_STATE_RESPONSE:
+		if (strcmp(name, "result") == 0)
+			parser->state++;
+		break;
+	case SOLR_XML_RESPONSE_STATE_RESULT:
+		if (strcmp(name, "doc") == 0) {
+			parser->state++;
+			parser->uid = 0;
+			parser->score = 0;
+			i_free_and_null(parser->mailbox);
+			i_free_and_null(parser->ns);
+			parser->uidvalidity = 0;
+		}
+		break;
+	case SOLR_XML_RESPONSE_STATE_DOC:
+		name_attr = attrs_get_name(attrs);
+		if (strcmp(name_attr, "uid") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_UID;
+		else if (strcmp(name_attr, "score") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_SCORE;
+		else if (strcmp(name_attr, "box") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_MAILBOX;
+		else if (strcmp(name_attr, "ns") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_NAMESPACE;
+		else if (strcmp(name_attr, "uidv") == 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_UIDVALIDITY;
+		else
+			break;
+		parser->state++;
+		break;
+	case SOLR_XML_RESPONSE_STATE_CONTENT:
+		break;
+	}
+}
+
+static struct solr_result *
+solr_result_get(struct solr_response_parser *parser, const char *box_id)
+{
+	struct solr_result *result;
+	char *box_id_dup;
+
+	result = hash_table_lookup(parser->mailboxes, box_id);
+	if (result != NULL)
+		return result;
+
+	box_id_dup = p_strdup(parser->result_pool, box_id);
+	result = p_new(parser->result_pool, struct solr_result, 1);
+	result->box_id = box_id_dup;
+	p_array_init(&result->uids, parser->result_pool, 32);
+	p_array_init(&result->scores, parser->result_pool, 32);
+	hash_table_insert(parser->mailboxes, box_id_dup, result);
+	array_push_back(&parser->results, &result);
+	return result;
+}
+
+static int solr_lookup_add_doc(struct solr_response_parser *parser)
+{
+	struct fts_score_map *score;
+	struct solr_result *result;
+	const char *box_id;
+
+	if (parser->uid == 0) {
+		i_error("fts_solr: uid missing from inside doc");
+		return -1;
+	}
+
+	if (parser->mailbox == NULL) {
+		/* looking up from a single mailbox only */
+		box_id = "";
+	} else if (parser->uidvalidity != 0) {
+		/* old style lookup */
+		string_t *str = t_str_new(64);
+		str_printfa(str, "%u\001", parser->uidvalidity);
+		str_append(str, parser->mailbox);
+		if (parser->ns != NULL)
+			str_printfa(str, "\001%s", parser->ns);
+		box_id = str_c(str);
+	} else {
+		/* new style lookup */
+		box_id = parser->mailbox;
+	}
+	result = solr_result_get(parser, box_id);
+
+	if (seq_range_array_add(&result->uids, parser->uid)) {
+		/* duplicate result */
+	} else if (parser->score != 0) {
+		score = array_append_space(&result->scores);
+		score->uid = parser->uid;
+		score->score = parser->score;
+	}
+	return 0;
+}
+
+static void solr_lookup_xml_end(void *context, const char *name ATTR_UNUSED)
+{
+	struct solr_response_parser *parser = context;
+	string_t *buf = parser->buffer;
+	int ret;
+
+	switch (parser->content_state) {
+	case SOLR_XML_CONTENT_STATE_NONE:
+		break;
+	case SOLR_XML_CONTENT_STATE_UID:
+		if (str_to_uint32(str_c(buf), &parser->uid) < 0 ||
+		    parser->uid == 0) {
+			i_error("fts_solr: received invalid uid '%s'",
+				str_c(buf));
+			parser->content_state = SOLR_XML_CONTENT_STATE_ERROR;
+		}
+		break;
+	case SOLR_XML_CONTENT_STATE_SCORE:
+		parser->score = strtod(str_c(buf), NULL);
+		break;
+	case SOLR_XML_CONTENT_STATE_MAILBOX:
+		parser->mailbox = i_strdup(str_c(buf));
+		break;
+	case SOLR_XML_CONTENT_STATE_NAMESPACE:
+		parser->ns = i_strdup(str_c(buf));
+		break;
+	case SOLR_XML_CONTENT_STATE_UIDVALIDITY:
+		if (str_to_uint32(str_c(buf), &parser->uidvalidity) < 0)
+			i_error("fts_solr: received invalid uidvalidity");
+		break;
+	case SOLR_XML_CONTENT_STATE_ERROR:
+		return;
+	}
+
+	i_assert(parser->depth >= (int)parser->state);
+
+	if (parser->state == SOLR_XML_RESPONSE_STATE_CONTENT &&
+	    parser->content_state == SOLR_XML_CONTENT_STATE_MAILBOX &&
+	    parser->mailbox == NULL) {
+		/* mailbox is namespace prefix */
+		parser->mailbox = i_strdup("");
+	}
+
+	if (parser->depth == (int)parser->state) {
+		ret = 0;
+		if (parser->state == SOLR_XML_RESPONSE_STATE_DOC) {
+			T_BEGIN {
+				ret = solr_lookup_add_doc(parser);
+			} T_END;
+		}
+		parser->state--;
+		if (ret < 0)
+			parser->content_state = SOLR_XML_CONTENT_STATE_ERROR;
+		else
+			parser->content_state = SOLR_XML_CONTENT_STATE_NONE;
+	}
+	parser->depth--;
+}
+
+static void solr_lookup_xml_data(void *context, const char *str, int len)
+{
+	struct solr_response_parser *parser = context;
+
+	switch (parser->content_state) {
+	case SOLR_XML_CONTENT_STATE_NONE:
+	case SOLR_XML_CONTENT_STATE_ERROR:
+		/* ignore element data */
+		return;
+	case SOLR_XML_CONTENT_STATE_UID:
+	case SOLR_XML_CONTENT_STATE_SCORE:
+	case SOLR_XML_CONTENT_STATE_MAILBOX:
+	case SOLR_XML_CONTENT_STATE_NAMESPACE:
+	case SOLR_XML_CONTENT_STATE_UIDVALIDITY:
+		break;
+	}
+
+	if (str_len(parser->buffer) + len > MAX_VALUE_LEN) {
+		i_error("fts_solr: XML element data length out of range");
+		parser->content_state = SOLR_XML_CONTENT_STATE_ERROR;
+		return;
+	}
+
+	str_append_data(parser->buffer, str, len);
+}
+
+struct solr_response_parser *
+solr_response_parser_init(pool_t result_pool, struct istream *input)
+{
+	struct solr_response_parser *parser;
+
+	parser = i_new(struct solr_response_parser, 1);
+
+	parser->xml_parser = XML_ParserCreate("UTF-8");
+	if (parser->xml_parser == NULL) {
+		i_fatal_status(FATAL_OUTOFMEM,
+			       "fts_solr: Failed to allocate XML parser");
+	}
+
+	parser->buffer = str_new(default_pool, 256);
+	hash_table_create(&parser->mailboxes, default_pool, 0,
+			  str_hash, strcmp);
+
+	parser->result_pool = result_pool;
+	pool_ref(result_pool);
+	p_array_init(&parser->results, result_pool, 32);
+
+	parser->input = input;
+	i_stream_ref(input);
+
+	parser->xml_failed = FALSE;
+	XML_SetElementHandler(parser->xml_parser,
+			      solr_lookup_xml_start, solr_lookup_xml_end);
+	XML_SetCharacterDataHandler(parser->xml_parser, solr_lookup_xml_data);
+	XML_SetUserData(parser->xml_parser, parser);
+
+	return parser;
+}
+
+void solr_response_parser_deinit(struct solr_response_parser **_parser)
+{
+	struct solr_response_parser *parser = *_parser;
+
+	*_parser = NULL;
+
+	if (parser == NULL)
+		return;
+
+	str_free(&parser->buffer);
+	hash_table_destroy(&parser->mailboxes);
+	XML_ParserFree(parser->xml_parser);
+	i_stream_unref(&parser->input);
+	pool_unref(&parser->result_pool);
+	i_free(parser);
+}
+
+int solr_response_parse(struct solr_response_parser *parser,
+			struct solr_result ***box_results_r)
+{
+	const unsigned char *data;
+	size_t size;
+	int stream_errno, ret;
+
+	i_assert(parser->input != NULL);
+	i_zero(box_results_r);
+
+	/* read payload */
+	while ((ret = i_stream_read_more(parser->input, &data, &size)) > 0) {
+		(void)solr_xml_parse(parser, data, size, FALSE);
+		i_stream_skip(parser->input, size);
+	}
+
+	if (ret == 0) {
+		/* we will be called again for more data */
+		return 0;
+	}
+
+	stream_errno = parser->input->stream_errno;
+	i_stream_unref(&parser->input);
+
+	if (parser->content_state == SOLR_XML_CONTENT_STATE_ERROR)
+		return -1;
+	if (stream_errno != 0)
+		return -1;
+
+	ret = solr_xml_parse(parser, "", 0, TRUE);
+
+	array_append_zero(&parser->results);
+	*box_results_r = array_front_modifiable(&parser->results);
+	return (ret == 0 ? 1 : -1);
+}
diff --git a/src/plugins/fts-solr/solr-response.h b/src/plugins/fts-solr/solr-response.h
new file mode 100644
index 0000000..1d5cdd5
--- /dev/null
+++ b/src/plugins/fts-solr/solr-response.h
@@ -0,0 +1,23 @@
+#ifndef SOLR_RESPONSE_H
+#define SOLR_RESPONSE_H
+
+#include "seq-range-array.h"
+#include "fts-api.h"
+
+struct solr_response_parser;
+
+struct solr_result {
+	const char *box_id;
+
+	ARRAY_TYPE(seq_range) uids;
+	ARRAY_TYPE(fts_score_map) scores;
+};
+
+struct solr_response_parser *
+solr_response_parser_init(pool_t result_pool, struct istream *input);
+void solr_response_parser_deinit(struct solr_response_parser **_parser);
+
+int solr_response_parse(struct solr_response_parser *parser,
+			struct solr_result ***box_results_r);
+
+#endif
diff --git a/src/plugins/fts-solr/test-solr-response.c b/src/plugins/fts-solr/test-solr-response.c
new file mode 100644
index 0000000..8add6db
--- /dev/null
+++ b/src/plugins/fts-solr/test-solr-response.c
@@ -0,0 +1,295 @@
+/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "array.h"
+#include "istream.h"
+#include "solr-response.h"
+#include "test-common.h"
+
+#include <unistd.h>
+
+static bool debug = FALSE;
+
+struct solr_response_test_result {
+	const char *box_id;
+	struct fts_score_map *scores;
+};
+
+struct solr_response_test {
+	const char *input;
+
+	struct solr_response_test_result *results;
+};
+
+struct fts_score_map test_results1_scores[] = {
+	{ .score = 0.042314477,	.uid = 1 },
+	{ .score = 0.06996078, .uid = 2, },
+	{ .score = 0.020381179, .uid = 3 },
+	{ .score = 0.020381179,	.uid = 4 },
+	{ .score = 5.510487E-4,	.uid = 6 },
+	{ .score = 0.0424253, .uid = 7 },
+	{ .score = 0.04215967, .uid = 8 },
+	{ .score = 0.02470572, .uid = 9 },
+	{ .score = 0.05936369, .uid = 10 },
+	{ .score = 0.048221838,	.uid = 11 },
+	{ .score = 7.793006E-4,	.uid = 12 },
+	{ .score = 2.7900032E-4, .uid = 13 },
+	{ .score = 0.02088323, .uid = 14 },
+	{ .score = 0.011646388, .uid = 15 },
+	{ .score = 1.3776218E-4, .uid = 17 },
+	{ .score = 2.386111E-4, .uid = 19 },
+	{ .score = 2.7552436E-4, .uid = 20 },
+	{ .score = 4.772222E-4, .uid = 23 },
+	{ .score = 4.772222E-4, .uid = 24 },
+	{ .score = 5.965277E-4, .uid = 25 },
+	{ .score = 0.0471366, .uid = 26 },
+	{ .score = 0.0471366, .uid = 50 },
+	{ .score = 0.047274362, .uid = 51 },
+	{ .score = 0.053303234, .uid = 56 },
+	{ .score = 5.445528E-4, .uid = 62 },
+	{ .score = 2.922377E-4, .uid = 66 },
+	{ .score = 0.02623833, .uid = 68 },
+	{ .score = 3.4440547E-4, .uid = 70 },
+	{ .score = 2.922377E-4, .uid = 74 },
+	{ .score = 2.7552436E-4, .uid = 76 },
+	{ .score = 1.3776218E-4, .uid = 77 },
+	{ .score = 0, .uid = 0 },
+};
+
+struct solr_response_test_result test_results1[] = {
+	{
+		.box_id = "",
+		.scores = test_results1_scores,
+	},
+	{
+		.box_id = NULL
+	}
+};
+
+static const struct solr_response_test tests[] = {
+	{
+		.input =
+			"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+			"<response>\n"
+			"<lst name=\"responseHeader\"><int name=\"status\""
+			">0</int><int name=\"QTime\">3</int><lst name=\"pa"
+			"rams\"><str name=\"wt\">xml</str><str name=\"fl\""
+			">uid,score</str><str name=\"rows\">4023</str><str"
+			" name=\"sort\">uid asc</str><str name=\"q\">{!luc"
+			"ene q.op=AND}subject:pierreserveur OR from:pierre"
+			"serveur OR to:pierreserveur OR cc:pierreserveur O"
+			"R bcc:pierreserveur OR body:pierreserveur</str><s"
+			"tr name=\"fq\">+box:fa74101044cb607d5f0900001de14"
+			"712 +user:jpierreserveur</str></lst></lst><result"
+			" name=\"response\" numFound=\"31\" start=\"0\" ma"
+			"xScore=\"0.06996078\"><doc><float name=\"score\">"
+			"0.042314477</float><long name=\"uid\">1</long></d"
+			"oc><doc><float name=\"score\">0.06996078</float><"
+			"long name=\"uid\">2</long></doc><doc><float name="
+			"\"score\">0.020381179</float><long name=\"uid\">3"
+			"</long></doc><doc><float name=\"score\">0.0203811"
+			"79</float><long name=\"uid\">4</long></doc><doc><"
+			"float name=\"score\">5.510487E-4</float><long nam"
+			"e=\"uid\">6</long></doc><doc><float name=\"score\""
+			">0.0424253</float><long name=\"uid\">7</long></do"
+			"c><doc><float name=\"score\">0.04215967</float><l"
+			"ong name=\"uid\">8</long></doc><doc><float name=\""
+			"score\">0.02470572</float><long name=\"uid\">9</l"
+			"ong></doc><doc><float name=\"score\">0.05936369</"
+			"float><long name=\"uid\">10</long></doc><doc><flo"
+			"at name=\"score\">0.048221838</float><long name=\""
+			"uid\">11</long></doc><doc><float name=\"score\">7"
+			".793006E-4</float><long name=\"uid\">12</long></d"
+			"oc><doc><float name=\"score\">2.7900032E-4</float"
+			"><long name=\"uid\">13</long></doc><doc><float na"
+			"me=\"score\">0.02088323</float><long name=\"uid\""
+			">14</long></doc><doc><float name=\"score\">0.0116"
+			"46388</float><long name=\"uid\">15</long></doc><d"
+			"oc><float name=\"score\">1.3776218E-4</float><lon"
+			"g name=\"uid\">17</long></doc><doc><float name=\""
+			"score\">2.386111E-4</float><long name=\"uid\">19<"
+			"/long></doc><doc><float name=\"score\">2.7552436E"
+			"-4</float><long name=\"uid\">20</long></doc><doc>"
+			"<float name=\"score\">4.772222E-4</float><long na"
+			"me=\"uid\">23</long></doc><doc><float name=\"scor"
+			"e\">4.772222E-4</float><long name=\"uid\">24</lon"
+			"g></doc><doc><float name=\"score\">5.965277E-4</f"
+			"loat><long name=\"uid\">25</long></doc><doc><floa"
+			"t name=\"score\">0.0471366</float><long name=\"ui"
+			"d\">26</long></doc><doc><float name=\"score\">0.0"
+			"471366</float><long name=\"uid\">50</long></doc><"
+			"doc><float name=\"score\">0.047274362</float><lon"
+			"g name=\"uid\">51</long></doc><doc><float name=\""
+			"score\">0.053303234</float><long name=\"uid\">56<"
+			"/long></doc><doc><float name=\"score\">5.445528E-"
+			"4</float><long name=\"uid\">62</long></doc><doc><"
+			"float name=\"score\">2.922377E-4</float><long nam"
+			"e=\"uid\">66</long></doc><doc><float name=\"score"
+			"\">0.02623833</float><long name=\"uid\">68</long>"
+			"</doc><doc><float name=\"score\">3.4440547E-4</fl"
+			"oat><long name=\"uid\">70</long></doc><doc><float"
+			" name=\"score\">2.922377E-4</float><long name=\"u"
+			"id\">74</long></doc><doc><float name=\"score\">2."
+			"7552436E-4</float><long name=\"uid\">76</long></d"
+			"oc><doc><float name=\"score\">1.3776218E-4</float"
+			"><long name=\"uid\">77</long></doc></result>\n"
+			"</response>\n",
+		.results = test_results1,
+	},
+};
+
+static const unsigned tests_count = N_ELEMENTS(tests);
+
+static void
+test_solr_result(const struct solr_response_test_result *test_results,
+		 struct solr_result **parse_results)
+{
+	unsigned int rcount, i;
+
+	for (i = 0; test_results[i].box_id != NULL; i++);
+	rcount = i;
+
+	for (i = 0; parse_results[i] != NULL; i++);
+
+	test_out_quiet("result count equal", i == rcount);
+	if (test_has_failed())
+		return;
+
+	for (i = 0; i < rcount && parse_results[i] != NULL; i++) {
+		unsigned int scount, j;
+		const struct fts_score_map *tscores = test_results[i].scores;
+		const struct fts_score_map *pscores =
+			array_get(&parse_results[i]->scores, &scount);
+
+		test_out_quiet(t_strdup_printf("box id equal[%u]", i),
+			       strcmp(test_results[i].box_id,
+				      parse_results[i]->box_id) == 0);
+
+		for (j = 0; tscores[j].uid != 0; j++);
+		test_out_quiet(t_strdup_printf("scores count equal[%u]", i),
+			       j == scount);
+		if (j != scount)
+			continue;
+
+		for (j = 0; j < scount; j++) {
+			test_out_quiet(
+				t_strdup_printf("score uid equal[%u/%u]", i, j),
+				pscores[j].uid == tscores[j].uid);
+			test_out_quiet(
+				t_strdup_printf("score value equal[%u/%u]", i, j),
+				pscores[j].score == tscores[j].score);
+		}
+	}
+}
+
+static void test_solr_response_parser(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < tests_count; i++) T_BEGIN {
+		const struct solr_response_test *test;
+		const char *text;
+		unsigned int pos, text_len;
+		struct istream *input;
+		struct solr_response_parser *parser;
+		struct solr_result **box_results;
+		const char *error = NULL;
+		pool_t pool;
+		int ret = 0;
+
+		test = &tests[i];
+		text = test->input;
+		text_len = strlen(text);
+
+		test_begin(t_strdup_printf("solr response [%d]", i));
+
+		input = test_istream_create_data(text, text_len);
+		pool = pool_alloconly_create("solr response", 4096);
+		parser = solr_response_parser_init(pool, input);
+
+		ret = solr_response_parse(parser, &box_results);
+
+		test_out_reason("parse ok (buffer)", ret > 0, error);
+		if (ret > 0)
+			test_solr_result(test->results, box_results);
+
+		solr_response_parser_deinit(&parser);
+		pool_unref(&pool);
+		i_stream_unref(&input);
+
+		input = test_istream_create_data(text, text_len);
+		pool = pool_alloconly_create("solr response", 4096);
+		parser = solr_response_parser_init(pool, input);
+
+		ret = 0;
+		for (pos = 0; pos <= text_len && ret == 0; pos++) {
+			test_istream_set_size(input, pos);
+			ret = solr_response_parse(parser, &box_results);
+		}
+
+		test_out_reason("parse ok (trickle)", ret > 0, error);
+		if (ret > 0)
+			test_solr_result(test->results, box_results);
+
+		solr_response_parser_deinit(&parser);
+		pool_unref(&pool);
+		i_stream_unref(&input);
+
+		test_end();
+
+	} T_END;
+}
+
+static void test_solr_response_file(const char *file)
+{
+	pool_t pool;
+	struct istream *input;
+	struct solr_response_parser *parser;
+	struct solr_result **box_results;
+	int ret = 0;
+
+	pool = pool_alloconly_create("solr response", 4096);
+	input = i_stream_create_file(file, 1024);
+	parser = solr_response_parser_init(pool, input);
+
+	while ((ret = solr_response_parse(parser, &box_results)) == 0);
+
+	if (ret < 0)
+		i_fatal("Failed to read response");
+
+	solr_response_parser_deinit(&parser);
+	i_stream_unref(&input);
+	pool_unref(&pool);
+}
+
+int main(int argc, char *argv[])
+{
+	int c;
+
+	static void (*test_functions[])(void) = {
+		test_solr_response_parser,
+		NULL
+	};
+
+	while ((c = getopt(argc, argv, "D")) > 0) {
+		switch (c) {
+		case 'D':
+			debug = TRUE;
+			break;
+		default:
+			i_fatal("Usage: %s [-D]", argv[0]);
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc > 0) {
+		test_solr_response_file(argv[0]);
+		return 0;
+	}
+
+	return test_run(test_functions);
+}
+
+
diff --git a/src/plugins/fts-squat/Makefile.am b/src/plugins/fts-squat/Makefile.am
new file mode 100644
index 0000000..6c8181c
--- /dev/null
+++ b/src/plugins/fts-squat/Makefile.am
@@ -0,0 +1,47 @@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/plugins/fts
+
+NOPLUGIN_LDFLAGS =
+lib21_fts_squat_plugin_la_LDFLAGS = -module -avoid-version
+
+module_LTLIBRARIES = \
+	lib21_fts_squat_plugin.la
+
+if DOVECOT_PLUGIN_DEPS
+lib21_fts_squat_plugin_la_LIBADD = \
+	../fts/lib20_fts_plugin.la
+endif
+
+lib21_fts_squat_plugin_la_SOURCES = \
+	fts-squat-plugin.c \
+	fts-backend-squat.c \
+	squat-trie.c \
+	squat-uidlist.c
+
+noinst_HEADERS = \
+	fts-squat-plugin.h \
+	squat-trie.h \
+	squat-trie-private.h \
+	squat-uidlist.h
+
+noinst_PROGRAMS = squat-test
+
+squat_test_SOURCES = \
+	squat-test.c
+
+common_objects = \
+	squat-trie.lo \
+	squat-uidlist.lo
+
+squat_test_LDADD = \
+	$(common_objects) \
+	$(LIBDOVECOT_STORAGE) \
+	$(LIBDOVECOT)
+squat_test_DEPENDENCIES = \
+	$(common_objects) \
+	$(LIBDOVECOT_STORAGE_DEPS) \
+	$(LIBDOVECOT_DEPS)
diff --git a/src/plugins/fts-squat/Makefile.in b/src/plugins/fts-squat/Makefile.in
new file mode 100644
index 0000000..443a7d0
--- /dev/null
+++ b/src/plugins/fts-squat/Makefile.in
@@ -0,0 +1,883 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+noinst_PROGRAMS = squat-test$(EXEEXT)
+subdir = src/plugins/fts-squat
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+	$(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+PROGRAMS = $(noinst_PROGRAMS)
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(moduledir)"
+LTLIBRARIES = $(module_LTLIBRARIES)
+@DOVECOT_PLUGIN_DEPS_TRUE@lib21_fts_squat_plugin_la_DEPENDENCIES =  \
+@DOVECOT_PLUGIN_DEPS_TRUE@	../fts/lib20_fts_plugin.la
+am_lib21_fts_squat_plugin_la_OBJECTS = fts-squat-plugin.lo \
+	fts-backend-squat.lo squat-trie.lo squat-uidlist.lo
+lib21_fts_squat_plugin_la_OBJECTS =  \
+	$(am_lib21_fts_squat_plugin_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+lib21_fts_squat_plugin_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+	$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+	$(AM_CFLAGS) $(CFLAGS) $(lib21_fts_squat_plugin_la_LDFLAGS) \
+	$(LDFLAGS) -o $@
+am_squat_test_OBJECTS = squat-test.$(OBJEXT)
+squat_test_OBJECTS = $(am_squat_test_OBJECTS)
+am__DEPENDENCIES_1 =
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/fts-backend-squat.Plo \
+	./$(DEPDIR)/fts-squat-plugin.Plo ./$(DEPDIR)/squat-test.Po \
+	./$(DEPDIR)/squat-trie.Plo ./$(DEPDIR)/squat-uidlist.Plo
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(lib21_fts_squat_plugin_la_SOURCES) $(squat_test_SOURCES)
+DIST_SOURCES = $(lib21_fts_squat_plugin_la_SOURCES) \
+	$(squat_test_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+HEADERS = $(noinst_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = 
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/plugins/fts
+
+lib21_fts_squat_plugin_la_LDFLAGS = -module -avoid-version
+module_LTLIBRARIES = \
+	lib21_fts_squat_plugin.la
+
+@DOVECOT_PLUGIN_DEPS_TRUE@lib21_fts_squat_plugin_la_LIBADD = \
+@DOVECOT_PLUGIN_DEPS_TRUE@	../fts/lib20_fts_plugin.la
+
+lib21_fts_squat_plugin_la_SOURCES = \
+	fts-squat-plugin.c \
+	fts-backend-squat.c \
+	squat-trie.c \
+	squat-uidlist.c
+
+noinst_HEADERS = \
+	fts-squat-plugin.h \
+	squat-trie.h \
+	squat-trie-private.h \
+	squat-uidlist.h
+
+squat_test_SOURCES = \
+	squat-test.c
+
+common_objects = \
+	squat-trie.lo \
+	squat-uidlist.lo
+
+squat_test_LDADD = \
+	$(common_objects) \
+	$(LIBDOVECOT_STORAGE) \
+	$(LIBDOVECOT)
+
+squat_test_DEPENDENCIES = \
+	$(common_objects) \
+	$(LIBDOVECOT_STORAGE_DEPS) \
+	$(LIBDOVECOT_DEPS)
+
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/plugins/fts-squat/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/plugins/fts-squat/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstPROGRAMS:
+	@list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \
+	echo " rm -f" $$list; \
+	rm -f $$list || exit $$?; \
+	test -n "$(EXEEXT)" || exit 0; \
+	list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+	echo " rm -f" $$list; \
+	rm -f $$list
+
+install-moduleLTLIBRARIES: $(module_LTLIBRARIES)
+	@$(NORMAL_INSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	list2=; for p in $$list; do \
+	  if test -f $$p; then \
+	    list2="$$list2 $$p"; \
+	  else :; fi; \
+	done; \
+	test -z "$$list2" || { \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(moduledir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(moduledir)" || exit 1; \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(moduledir)'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(moduledir)"; \
+	}
+
+uninstall-moduleLTLIBRARIES:
+	@$(NORMAL_UNINSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	for p in $$list; do \
+	  $(am__strip_dir) \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(moduledir)/$$f'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(moduledir)/$$f"; \
+	done
+
+clean-moduleLTLIBRARIES:
+	-test -z "$(module_LTLIBRARIES)" || rm -f $(module_LTLIBRARIES)
+	@list='$(module_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+lib21_fts_squat_plugin.la: $(lib21_fts_squat_plugin_la_OBJECTS) $(lib21_fts_squat_plugin_la_DEPENDENCIES) $(EXTRA_lib21_fts_squat_plugin_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(lib21_fts_squat_plugin_la_LINK) -rpath $(moduledir) $(lib21_fts_squat_plugin_la_OBJECTS) $(lib21_fts_squat_plugin_la_LIBADD) $(LIBS)
+
+squat-test$(EXEEXT): $(squat_test_OBJECTS) $(squat_test_DEPENDENCIES) $(EXTRA_squat_test_DEPENDENCIES) 
+	@rm -f squat-test$(EXEEXT)
+	$(AM_V_CCLD)$(LINK) $(squat_test_OBJECTS) $(squat_test_LDADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-backend-squat.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-squat-plugin.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/squat-test.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/squat-trie.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/squat-uidlist.Plo@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(moduledir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-moduleLTLIBRARIES \
+	clean-noinstPROGRAMS mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/fts-backend-squat.Plo
+	-rm -f ./$(DEPDIR)/fts-squat-plugin.Plo
+	-rm -f ./$(DEPDIR)/squat-test.Po
+	-rm -f ./$(DEPDIR)/squat-trie.Plo
+	-rm -f ./$(DEPDIR)/squat-uidlist.Plo
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-moduleLTLIBRARIES
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/fts-backend-squat.Plo
+	-rm -f ./$(DEPDIR)/fts-squat-plugin.Plo
+	-rm -f ./$(DEPDIR)/squat-test.Po
+	-rm -f ./$(DEPDIR)/squat-trie.Plo
+	-rm -f ./$(DEPDIR)/squat-uidlist.Plo
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-moduleLTLIBRARIES
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-generic clean-libtool clean-moduleLTLIBRARIES \
+	clean-noinstPROGRAMS cscopelist-am ctags ctags-am distclean \
+	distclean-compile distclean-generic distclean-libtool \
+	distclean-tags distdir dvi dvi-am html html-am info info-am \
+	install install-am install-data install-data-am install-dvi \
+	install-dvi-am install-exec install-exec-am install-html \
+	install-html-am install-info install-info-am install-man \
+	install-moduleLTLIBRARIES install-pdf install-pdf-am \
+	install-ps install-ps-am install-strip installcheck \
+	installcheck-am installdirs maintainer-clean \
+	maintainer-clean-generic mostlyclean mostlyclean-compile \
+	mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+	tags tags-am uninstall uninstall-am \
+	uninstall-moduleLTLIBRARIES
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/plugins/fts-squat/fts-backend-squat.c b/src/plugins/fts-squat/fts-backend-squat.c
new file mode 100644
index 0000000..fbd7bbe
--- /dev/null
+++ b/src/plugins/fts-squat/fts-backend-squat.c
@@ -0,0 +1,497 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "unichar.h"
+#include "mail-user.h"
+#include "mail-namespace.h"
+#include "mail-storage-private.h"
+#include "mail-search-build.h"
+#include "squat-trie.h"
+#include "fts-squat-plugin.h"
+
+
+#define SQUAT_FILE_PREFIX "dovecot.index.search"
+
+struct squat_fts_backend {
+	struct fts_backend backend;
+
+	struct mailbox *box;
+	struct squat_trie *trie;
+
+	unsigned int partial_len, full_len;
+	bool refresh;
+};
+
+struct squat_fts_backend_update_context {
+	struct fts_backend_update_context ctx;
+	struct squat_trie_build_context *build_ctx;
+
+	enum squat_index_type squat_type;
+	uint32_t uid;
+	string_t *hdr;
+
+	bool failed;
+};
+
+static struct fts_backend *fts_backend_squat_alloc(void)
+{
+	struct squat_fts_backend *backend;
+
+	backend = i_new(struct squat_fts_backend, 1);
+	backend->backend = fts_backend_squat;
+	return &backend->backend;
+}
+
+static int
+fts_backend_squat_init(struct fts_backend *_backend, const char **error_r)
+{
+	struct squat_fts_backend *backend =
+		(struct squat_fts_backend *)_backend;
+	const char *const *tmp, *env;
+	unsigned int len;
+
+	env = mail_user_plugin_getenv(_backend->ns->user, "fts_squat");
+	if (env == NULL)
+		return 0;
+
+	for (tmp = t_strsplit_spaces(env, " "); *tmp != NULL; tmp++) {
+		if (str_begins(*tmp, "partial=")) {
+			if (str_to_uint(*tmp + 8, &len) < 0 || len == 0) {
+				*error_r = t_strdup_printf(
+					"Invalid partial length: %s", *tmp + 8);
+				return -1;
+			}
+			backend->partial_len = len;
+		} else if (str_begins(*tmp, "full=")) {
+			if (str_to_uint(*tmp + 5, &len) < 0 || len == 0) {
+				*error_r = t_strdup_printf(
+					"Invalid full length: %s", *tmp + 5);
+				return -1;
+			}
+			backend->full_len = len;
+		} else {
+			*error_r = t_strdup_printf("Invalid setting: %s", *tmp);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static void
+fts_backend_squat_unset_box(struct squat_fts_backend *backend)
+{
+	if (backend->trie != NULL)
+		squat_trie_deinit(&backend->trie);
+	backend->box = NULL;
+}
+
+static void fts_backend_squat_deinit(struct fts_backend *_backend)
+{
+	struct squat_fts_backend *backend =
+		(struct squat_fts_backend *)_backend;
+
+	fts_backend_squat_unset_box(backend);
+	i_free(backend);
+}
+
+static int
+fts_backend_squat_set_box(struct squat_fts_backend *backend,
+			  struct mailbox *box)
+{
+	const struct mailbox_permissions *perm;
+	struct mail_storage *storage;
+	struct mailbox_status status;
+	const char *path;
+	enum squat_index_flags flags = 0;
+        int ret;
+
+	if (backend->box == box)
+        {
+		if (backend->refresh) {
+                        ret = squat_trie_refresh(backend->trie);
+                        if (ret < 0)
+				return ret;
+			backend->refresh = FALSE;
+		}
+		return 0;
+	}
+	fts_backend_squat_unset_box(backend);
+	backend->refresh = FALSE;
+	if (box == NULL)
+		return 0;
+
+	perm = mailbox_get_permissions(box);
+	storage = mailbox_get_storage(box);
+	if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0)
+		i_unreached(); /* fts already checked this */
+
+	mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
+	if (storage->set->mmap_disable)
+		flags |= SQUAT_INDEX_FLAG_MMAP_DISABLE;
+	if (storage->set->mail_nfs_index)
+		flags |= SQUAT_INDEX_FLAG_NFS_FLUSH;
+	if (storage->set->dotlock_use_excl)
+		flags |= SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL;
+
+	backend->trie =
+		squat_trie_init(t_strconcat(path, "/"SQUAT_FILE_PREFIX, NULL),
+				status.uidvalidity,
+				storage->set->parsed_lock_method,
+				flags, perm->file_create_mode,
+				perm->file_create_gid);
+
+	if (backend->partial_len != 0)
+		squat_trie_set_partial_len(backend->trie, backend->partial_len);
+	if (backend->full_len != 0)
+		squat_trie_set_full_len(backend->trie, backend->full_len);
+	backend->box = box;
+	return squat_trie_open(backend->trie);
+}
+
+static int
+fts_backend_squat_get_last_uid(struct fts_backend *_backend,
+			       struct mailbox *box, uint32_t *last_uid_r)
+{
+	struct squat_fts_backend *backend =
+		(struct squat_fts_backend *)_backend;
+
+	int ret = fts_backend_squat_set_box(backend, box);
+	if (ret < 0)
+		return -1;
+	return squat_trie_get_last_uid(backend->trie, last_uid_r);
+}
+
+static struct fts_backend_update_context *
+fts_backend_squat_update_init(struct fts_backend *_backend)
+{
+	struct squat_fts_backend_update_context *ctx;
+
+	ctx = i_new(struct squat_fts_backend_update_context, 1);
+	ctx->ctx.backend = _backend;
+	ctx->hdr = str_new(default_pool, 1024*32);
+	return &ctx->ctx;
+}
+
+static int get_all_msg_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids)
+{
+	struct mailbox_transaction_context *t;
+	struct mail_search_context *search_ctx;
+	struct mail_search_args *search_args;
+	struct mail *mail;
+	int ret;
+
+	t = mailbox_transaction_begin(box, 0, __func__);
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_all(search_args);
+	search_ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(search_ctx, &mail)) {
+		/* *2 because even/odd is for body/header */
+		seq_range_array_add_range(uids, mail->uid * 2,
+					  mail->uid * 2 + 1);
+	}
+	ret = mailbox_search_deinit(&search_ctx);
+	(void)mailbox_transaction_commit(&t);
+	return ret;
+}
+
+static int
+fts_backend_squat_update_uid_changed(struct squat_fts_backend_update_context *ctx)
+{
+	int ret = 0;
+
+	if (ctx->uid == 0)
+		return 0;
+
+	if (squat_trie_build_more(ctx->build_ctx, ctx->uid,
+				  SQUAT_INDEX_TYPE_HEADER,
+				  str_data(ctx->hdr), str_len(ctx->hdr)) < 0)
+		ret = -1;
+	str_truncate(ctx->hdr, 0);
+	return ret;
+}
+
+static int
+fts_backend_squat_build_deinit(struct squat_fts_backend_update_context *ctx)
+{
+	struct squat_fts_backend *backend =
+		(struct squat_fts_backend *)ctx->ctx.backend;
+	ARRAY_TYPE(seq_range) uids;
+	int ret = 0;
+
+	if (ctx->build_ctx == NULL)
+		return 0;
+
+	if (fts_backend_squat_update_uid_changed(ctx) < 0)
+		ret = -1;
+
+	i_array_init(&uids, 1024);
+	if (get_all_msg_uids(backend->box, &uids) < 0) {
+		(void)squat_trie_build_deinit(&ctx->build_ctx, NULL);
+		ret = -1;
+	} else {
+		seq_range_array_invert(&uids, 2, (uint32_t)-2);
+		if (squat_trie_build_deinit(&ctx->build_ctx, &uids) < 0)
+			ret = -1;
+	}
+	array_free(&uids);
+	return ret;
+}
+
+static int
+fts_backend_squat_update_deinit(struct fts_backend_update_context *_ctx)
+{
+	struct squat_fts_backend_update_context *ctx =
+		(struct squat_fts_backend_update_context *)_ctx;
+	int ret = ctx->failed ? -1 : 0;
+
+	if (fts_backend_squat_build_deinit(ctx) < 0)
+		ret = -1;
+	str_free(&ctx->hdr);
+	i_free(ctx);
+	return ret;
+}
+
+static void
+fts_backend_squat_update_set_mailbox(struct fts_backend_update_context *_ctx,
+				     struct mailbox *box)
+{
+	struct squat_fts_backend_update_context *ctx =
+		(struct squat_fts_backend_update_context *)_ctx;
+	struct squat_fts_backend *backend =
+		(struct squat_fts_backend *)ctx->ctx.backend;
+
+	if (fts_backend_squat_build_deinit(ctx) < 0)
+		ctx->failed = TRUE;
+	if (fts_backend_squat_set_box(backend, box) < 0)
+		ctx->failed = TRUE;
+	else if (box != NULL) {
+		if (squat_trie_build_init(backend->trie, &ctx->build_ctx) < 0)
+			ctx->failed = TRUE;
+	}
+}
+
+static void
+fts_backend_squat_update_expunge(struct fts_backend_update_context *_ctx ATTR_UNUSED,
+				 uint32_t last_uid ATTR_UNUSED)
+{
+	/* FIXME */
+}
+
+static bool
+fts_backend_squat_update_set_build_key(struct fts_backend_update_context *_ctx,
+				       const struct fts_backend_build_key *key)
+{
+	struct squat_fts_backend_update_context *ctx =
+		(struct squat_fts_backend_update_context *)_ctx;
+
+	if (ctx->failed)
+		return FALSE;
+
+	if (key->uid != ctx->uid) {
+		if (fts_backend_squat_update_uid_changed(ctx) < 0)
+			ctx->failed = TRUE;
+	}
+
+	switch (key->type) {
+	case FTS_BACKEND_BUILD_KEY_HDR:
+	case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+		str_printfa(ctx->hdr, "%s: ", key->hdr_name);
+		ctx->squat_type = SQUAT_INDEX_TYPE_HEADER;
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART:
+		ctx->squat_type = SQUAT_INDEX_TYPE_BODY;
+		break;
+	case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+		i_unreached();
+	}
+	ctx->uid = key->uid;
+	return TRUE;
+}
+
+static void
+fts_backend_squat_update_unset_build_key(struct fts_backend_update_context *_ctx)
+{
+	struct squat_fts_backend_update_context *ctx =
+		(struct squat_fts_backend_update_context *)_ctx;
+
+	if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER)
+		str_append_c(ctx->hdr, '\n');
+}
+
+static int
+fts_backend_squat_update_build_more(struct fts_backend_update_context *_ctx,
+				    const unsigned char *data, size_t size)
+{
+	struct squat_fts_backend_update_context *ctx =
+		(struct squat_fts_backend_update_context *)_ctx;
+
+	if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER) {
+		str_append_data(ctx->hdr, data, size);
+		return 0;
+	}
+	return squat_trie_build_more(ctx->build_ctx, ctx->uid, ctx->squat_type,
+				     data, size);
+}
+
+static int fts_backend_squat_refresh(struct fts_backend *_backend)
+{
+	struct squat_fts_backend *backend =
+		(struct squat_fts_backend *)_backend;
+
+	backend->refresh = TRUE;
+	return 0;
+}
+
+static int fts_backend_squat_optimize(struct fts_backend *_backend ATTR_UNUSED)
+{
+	/* FIXME: drop expunged messages */
+	return 0;
+}
+
+static int squat_lookup_arg(struct squat_fts_backend *backend,
+			    const struct mail_search_arg *arg, bool and_args,
+			    ARRAY_TYPE(seq_range) *definite_uids,
+			    ARRAY_TYPE(seq_range) *maybe_uids)
+{
+	enum squat_index_type squat_type;
+	ARRAY_TYPE(seq_range) tmp_definite_uids, tmp_maybe_uids;
+	string_t *dtc;
+	uint32_t last_uid;
+	int ret;
+
+	switch (arg->type) {
+	case SEARCH_TEXT:
+		squat_type = SQUAT_INDEX_TYPE_HEADER |
+			SQUAT_INDEX_TYPE_BODY;
+		break;
+	case SEARCH_BODY:
+		squat_type = SQUAT_INDEX_TYPE_BODY;
+		break;
+	case SEARCH_HEADER:
+	case SEARCH_HEADER_ADDRESS:
+	case SEARCH_HEADER_COMPRESS_LWSP:
+		squat_type = SQUAT_INDEX_TYPE_HEADER;
+		break;
+	default:
+		return 0;
+	}
+
+	i_array_init(&tmp_definite_uids, 128);
+	i_array_init(&tmp_maybe_uids, 128);
+
+	dtc = t_str_new(128);
+	if (backend->backend.ns->user->
+	    default_normalizer(arg->value.str, strlen(arg->value.str), dtc) < 0)
+		i_panic("squat: search key not utf8");
+
+	ret = squat_trie_lookup(backend->trie, str_c(dtc), squat_type,
+				&tmp_definite_uids, &tmp_maybe_uids);
+	if (arg->match_not) {
+		/* definite -> non-match
+		   maybe -> maybe
+		   non-match -> maybe */
+		array_clear(&tmp_maybe_uids);
+
+		if (squat_trie_get_last_uid(backend->trie, &last_uid) < 0)
+			i_unreached();
+		seq_range_array_add_range(&tmp_maybe_uids, 1, last_uid);
+		seq_range_array_remove_seq_range(&tmp_maybe_uids,
+						 &tmp_definite_uids);
+		array_clear(&tmp_definite_uids);
+	}
+
+	if (and_args) {
+		/* AND:
+		   definite && definite -> definite
+		   definite && maybe -> maybe
+		   maybe && maybe -> maybe */
+
+		/* put definites among maybies, so they can be intersected */
+		seq_range_array_merge(maybe_uids, definite_uids);
+		seq_range_array_merge(&tmp_maybe_uids, &tmp_definite_uids);
+
+		seq_range_array_intersect(maybe_uids, &tmp_maybe_uids);
+		seq_range_array_intersect(definite_uids, &tmp_definite_uids);
+		/* remove duplicate maybies that are also definites */
+		seq_range_array_remove_seq_range(maybe_uids, definite_uids);
+	} else {
+		/* OR:
+		   definite || definite -> definite
+		   definite || maybe -> definite
+		   maybe || maybe -> maybe */
+
+		/* remove maybies that are now definites */
+		seq_range_array_remove_seq_range(&tmp_maybe_uids,
+						 definite_uids);
+		seq_range_array_remove_seq_range(maybe_uids,
+						 &tmp_definite_uids);
+
+		seq_range_array_merge(definite_uids, &tmp_definite_uids);
+		seq_range_array_merge(maybe_uids, &tmp_maybe_uids);
+	}
+
+	array_free(&tmp_definite_uids);
+	array_free(&tmp_maybe_uids);
+	return ret < 0 ? -1 : 1;
+}
+
+static int
+fts_backend_squat_lookup(struct fts_backend *_backend, struct mailbox *box,
+			 struct mail_search_arg *args,
+			 enum fts_lookup_flags flags,
+			 struct fts_result *result)
+{
+	struct squat_fts_backend *backend =
+		(struct squat_fts_backend *)_backend;
+	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
+	bool first = TRUE;
+	int ret;
+
+	ret = fts_backend_squat_set_box(backend, box);
+	if (ret < 0)
+		return -1;
+
+	for (; args != NULL; args = args->next) {
+		ret = squat_lookup_arg(backend, args, first ? FALSE : and_args,
+				       &result->definite_uids,
+				       &result->maybe_uids);
+		if (ret < 0)
+			return -1;
+		if (ret > 0) {
+			args->match_always = TRUE;
+			first = FALSE;
+		}
+	}
+	return 0;
+}
+
+struct fts_backend fts_backend_squat = {
+	.name = "squat",
+	.flags = FTS_BACKEND_FLAG_NORMALIZE_INPUT,
+
+	{
+		fts_backend_squat_alloc,
+		fts_backend_squat_init,
+		fts_backend_squat_deinit,
+		fts_backend_squat_get_last_uid,
+		fts_backend_squat_update_init,
+		fts_backend_squat_update_deinit,
+		fts_backend_squat_update_set_mailbox,
+		fts_backend_squat_update_expunge,
+		fts_backend_squat_update_set_build_key,
+		fts_backend_squat_update_unset_build_key,
+		fts_backend_squat_update_build_more,
+		fts_backend_squat_refresh,
+		NULL,
+		fts_backend_squat_optimize,
+		fts_backend_default_can_lookup,
+		fts_backend_squat_lookup,
+		NULL,
+		NULL
+	}
+};
diff --git a/src/plugins/fts-squat/fts-squat-plugin.c b/src/plugins/fts-squat/fts-squat-plugin.c
new file mode 100644
index 0000000..59d9383
--- /dev/null
+++ b/src/plugins/fts-squat/fts-squat-plugin.c
@@ -0,0 +1,18 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "fts-squat-plugin.h"
+
+const char *fts_squat_plugin_version = DOVECOT_ABI_VERSION;
+
+void fts_squat_plugin_init(struct module *module ATTR_UNUSED)
+{
+	fts_backend_register(&fts_backend_squat);
+}
+
+void fts_squat_plugin_deinit(void)
+{
+	fts_backend_unregister(fts_backend_squat.name);
+}
+
+const char *fts_squat_plugin_dependencies[] = { "fts", NULL };
diff --git a/src/plugins/fts-squat/fts-squat-plugin.h b/src/plugins/fts-squat/fts-squat-plugin.h
new file mode 100644
index 0000000..0d6bfcb
--- /dev/null
+++ b/src/plugins/fts-squat/fts-squat-plugin.h
@@ -0,0 +1,14 @@
+#ifndef FTS_SQUAT_PLUGIN_H
+#define FTS_SQUAT_PLUGIN_H
+
+#include "fts-api-private.h"
+
+struct module;
+
+extern const char *fts_squat_plugin_dependencies[];
+extern struct fts_backend fts_backend_squat;
+
+void fts_squat_plugin_init(struct module *module);
+void fts_squat_plugin_deinit(void);
+
+#endif
diff --git a/src/plugins/fts-squat/squat-test.c b/src/plugins/fts-squat/squat-test.c
new file mode 100644
index 0000000..b55646c
--- /dev/null
+++ b/src/plugins/fts-squat/squat-test.c
@@ -0,0 +1,197 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "file-lock.h"
+#include "istream.h"
+#include "time-util.h"
+#include "unichar.h"
+#include "squat-trie.h"
+#include "squat-uidlist.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <time.h>
+#include <sys/time.h>
+
+static void result_print(ARRAY_TYPE(seq_range) *result)
+{
+	const struct seq_range *range;
+	unsigned int i, count;
+
+	range = array_get(result, &count);
+	for (i = 0; i < count; i++) {
+		if (i != 0)
+			printf(",");
+		printf("%u", range[i].seq1);
+		if (range[i].seq1 != range[i].seq2)
+			printf("-%u", range[i].seq2);
+	}
+	printf("\n");
+}
+
+int main(int argc ATTR_UNUSED, char *argv[])
+{
+	const char *trie_path = "/tmp/squat-test-index.search";
+	const char *uidlist_path = "/tmp/squat-test-index.search.uids";
+	struct squat_trie *trie;
+	struct squat_trie_build_context *build_ctx;
+	struct istream *input;
+	struct stat trie_st, uidlist_st;
+	ARRAY_TYPE(seq_range) definite_uids, maybe_uids;
+	char *line, *str, buf[4096];
+	buffer_t *valid;
+	int ret, fd;
+	unsigned int last = 0, seq = 1, node_count, uidlist_count;
+	size_t len;
+	enum squat_index_type index_type;
+	bool data_header = TRUE, first = TRUE, skip_body = FALSE;
+	bool mime_header = TRUE;
+	size_t trie_mem, uidlist_mem;
+	clock_t clock_start, clock_end;
+	struct timeval tv_start, tv_end;
+	double cputime;
+
+	lib_init();
+	i_unlink_if_exists(trie_path);
+	i_unlink_if_exists(uidlist_path);
+	trie = squat_trie_init(trie_path, time(NULL),
+			       FILE_LOCK_METHOD_FCNTL, 0, 0600, (gid_t)-1);
+
+	clock_start = clock();
+	i_gettimeofday(&tv_start);
+
+	fd = open(argv[1], O_RDONLY);
+	if (fd == -1)
+		return 1;
+
+	if (squat_trie_build_init(trie, &build_ctx) < 0)
+		return 1;
+
+	valid = buffer_create_dynamic(default_pool, 4096);
+	input = i_stream_create_fd(fd, SIZE_MAX);
+	ret = 0;
+	while (ret == 0 && (line = i_stream_read_next_line(input)) != NULL) {
+		if (last != input->v_offset/(1024*100)) {
+			fprintf(stderr, "\r%ukB", (unsigned)(input->v_offset/1024));
+			fflush(stderr);
+			last = input->v_offset/(1024*100);
+		}
+		if (str_begins(line, "From ")) {
+			if (!first)
+				seq++;
+			data_header = TRUE;
+			skip_body = FALSE;
+			mime_header = TRUE;
+			continue;
+		}
+		first = FALSE;
+
+		if (str_begins(line, "--")) {
+			skip_body = FALSE;
+			mime_header = TRUE;
+		}
+
+		if (mime_header) {
+			if (*line == '\0') {
+				data_header = FALSE;
+				mime_header = FALSE;
+				continue;
+			}
+
+			if (strncasecmp(line, "Content-Type:", 13) == 0 &&
+			    strncasecmp(line, "Content-Type: text/", 19) != 0 &&
+			    strncasecmp(line, "Content-Type: message/", 22) != 0)
+				skip_body = TRUE;
+			else if (strncasecmp(line, "Content-Transfer-Encoding: base64", 33) == 0)
+				skip_body = TRUE;
+		} else if (skip_body)
+			continue;
+		if (*line == '\0')
+			continue;
+
+		/* we're actually indexing here headers as bodies and bodies
+		   as headers. it doesn't really matter in this test, and
+		   fixing it would require storing headers temporarily
+		   elsewhere and index them only after the body */
+		index_type = !data_header ? SQUAT_INDEX_TYPE_HEADER :
+			SQUAT_INDEX_TYPE_BODY;
+
+		buffer_set_used_size(valid, 0);
+		len = strlen(line);
+		if (uni_utf8_get_valid_data((const unsigned char *)line,
+					    len, valid)) {
+			ret = squat_trie_build_more(build_ctx, seq, index_type,
+						    (const void *)line, len);
+		} else if (valid->used > 0) {
+			ret = squat_trie_build_more(build_ctx, seq, index_type,
+						    valid->data, valid->used);
+		}
+	}
+	buffer_free(&valid);
+	if (squat_trie_build_deinit(&build_ctx, NULL) < 0)
+		ret = -1;
+	if (ret < 0) {
+		printf("build broken\n");
+		return 1;
+	}
+
+	clock_end = clock();
+	i_gettimeofday(&tv_end);
+
+	cputime = (double)(clock_end - clock_start) / CLOCKS_PER_SEC;
+	fprintf(stderr, "\n - Index time: %.2f CPU seconds, "
+		"%.2f real seconds (%.02fMB/CPUs)\n", cputime,
+		timeval_diff_msecs(&tv_end, &tv_start)/1000.0,
+		input->v_offset / cputime / (1024*1024));
+
+	if (stat(trie_path, &trie_st) < 0)
+		i_error("stat(%s) failed: %m", trie_path);
+	if (stat(uidlist_path, &uidlist_st) < 0)
+		i_error("stat(%s) failed: %m", uidlist_path);
+
+	trie_mem = squat_trie_mem_used(trie, &node_count);
+	uidlist_mem = squat_uidlist_mem_used(squat_trie_get_uidlist(trie),
+					     &uidlist_count);
+	fprintf(stderr, " - memory: %uk for trie, %uk for uidlist\n",
+		(unsigned)(trie_mem/1024), (unsigned)(uidlist_mem/1024));
+	fprintf(stderr, " - %"PRIuUOFF_T" bytes in %u nodes (%.02f%%)\n",
+		trie_st.st_size, node_count,
+		trie_st.st_size / (float)input->v_offset * 100.0);
+	fprintf(stderr, " - %"PRIuUOFF_T" bytes in %u UID lists (%.02f%%)\n",
+		uidlist_st.st_size, uidlist_count,
+		uidlist_st.st_size / (float)input->v_offset * 100.0);
+	fprintf(stderr, " - %"PRIuUOFF_T" bytes total of %"
+		PRIuUOFF_T" (%.02f%%)\n",
+		(trie_st.st_size + uidlist_st.st_size), input->v_offset,
+		(trie_st.st_size + uidlist_st.st_size) /
+		(float)input->v_offset * 100.0);
+
+	i_stream_unref(&input);
+	i_close_fd(&fd);
+
+	i_array_init(&definite_uids, 128);
+	i_array_init(&maybe_uids, 128);
+	while ((str = fgets(buf, sizeof(buf), stdin)) != NULL) {
+		ret = strlen(str)-1;
+		str[ret] = 0;
+
+		i_gettimeofday(&tv_start);
+		ret = squat_trie_lookup(trie, str, SQUAT_INDEX_TYPE_HEADER |
+					SQUAT_INDEX_TYPE_BODY,
+					&definite_uids, &maybe_uids);
+		if (ret < 0)
+			printf("error\n");
+		else {
+			i_gettimeofday(&tv_end);
+			printf(" - Search took %.05f CPU seconds\n",
+			       timeval_diff_usecs(&tv_end, &tv_start)/1000000.0);
+			printf(" - definite uids: ");
+			result_print(&definite_uids);
+			printf(" - maybe uids: ");
+			result_print(&maybe_uids);
+		}
+	}
+	return 0;
+}
diff --git a/src/plugins/fts-squat/squat-trie-private.h b/src/plugins/fts-squat/squat-trie-private.h
new file mode 100644
index 0000000..b079554
--- /dev/null
+++ b/src/plugins/fts-squat/squat-trie-private.h
@@ -0,0 +1,192 @@
+#ifndef SQUAT_TRIE_PRIVATE_H
+#define SQUAT_TRIE_PRIVATE_H
+
+#include "file-dotlock.h"
+#include "squat-trie.h"
+
+#define SQUAT_TRIE_VERSION 2
+#define SQUAT_TRIE_LOCK_TIMEOUT 60
+#define SQUAT_TRIE_DOTLOCK_STALE_TIMEOUT (15*60)
+
+struct squat_file_header {
+	uint8_t version;
+	uint8_t unused[3];
+
+	uint32_t indexid;
+	uint32_t uidvalidity;
+	uint32_t used_file_size;
+	uint32_t deleted_space;
+	uint32_t node_count;
+
+	uint32_t root_offset;
+	uint32_t root_unused_uids;
+	uint32_t root_next_uid;
+	uint32_t root_uidlist_idx;
+
+	uint8_t partial_len;
+	uint8_t full_len;
+	uint8_t normalize_map[256];
+};
+
+/*
+   node file: FIXME: no up-to-date
+
+   struct squat_file_header;
+
+   // children are written before their parents
+   node[] {
+     uint8_t child_count;
+     unsigned char chars[child_count];
+     packed neg_diff_to_first_child_offset; // relative to node
+     packed diff_to_prev_offset[child_count-1];
+     packed[child_count] {
+       // unused_uids_count == uid if have_uid_offset bit is zero
+       (unused_uids_count << 1) | (have_uid_offset);
+       [diff_to_prev_uid_offset;] // first one is relative to zero
+     }
+   }
+*/
+
+struct squat_node {
+	unsigned int child_count:8;
+
+	/* children.leaf_string contains this many bytes */
+	unsigned int leaf_string_length:16;
+
+	/* TRUE = children.data contains our children.
+	   FALSE = children.offset contains offset to our children in the
+	   index file. */
+	bool children_not_mapped:1;
+	/* When allocating our children, use a sequential array. */
+	bool want_sequential:1;
+	/* This node's children are in a sequential array, meaning that the
+	   first SEQUENTIAL_COUNT children have chars[n] = n. */
+	bool have_sequential:1;
+
+	/* Number of UIDs that exists in parent node but not in this one
+	   (i.e. number of UIDs [0..next_uid-1] not in this node's uidlist).
+	   This is mainly used when adding new UIDs to our children to set
+	   the UID to be relative to this node's UID list. */
+	uint32_t unused_uids;
+
+	/* next_uid=0 means there are no UIDs in this node, otherwise
+	   next_uid-1 is the last UID added to this node. */
+	uint32_t next_uid;
+	uint32_t uid_list_idx;
+
+	/*
+	   struct {
+	     unsigned char chars[child_count];
+	     struct squat_node[child_count];
+	   } *children;
+	*/
+	union {
+		/* children_not_mapped determines if data or offset should
+		   be used. */
+		void *data;
+		unsigned char *leaf_string;
+		unsigned char static_leaf_string[sizeof(void *)];
+		uint32_t offset;
+	} children;
+};
+/* Return pointer to node.children.chars[] */
+#define NODE_CHILDREN_CHARS(node) \
+	((unsigned char *)(node)->children.data)
+/* Return pointer to node.children.node[] */
+#define NODE_CHILDREN_NODES(_node) \
+	((struct squat_node *)(NODE_CHILDREN_CHARS(_node) + \
+			       MEM_ALIGN((_node)->child_count)))
+/* Return number of bytes allocated in node.children.data */
+#define NODE_CHILDREN_ALLOC_SIZE(child_count) \
+	(MEM_ALIGN(child_count) + \
+	 ((child_count) / 8 + 1) * 8 * sizeof(struct squat_node))
+/* Return TRUE if children.leaf_string is set. */
+#define NODE_IS_DYNAMIC_LEAF(node) \
+	((node)->leaf_string_length > \
+		sizeof((node)->children.static_leaf_string))
+/* Return node's leaf string. Assumes that it is set. */
+#define NODE_LEAF_STRING(node) \
+	(NODE_IS_DYNAMIC_LEAF(node) ? \
+	 (node)->children.leaf_string : (node)->children.static_leaf_string)
+struct squat_trie {
+	struct squat_node root;
+	struct squat_uidlist *uidlist;
+
+	struct squat_file_header hdr;
+	size_t node_alloc_size;
+	unsigned int unmapped_child_count;
+
+	enum squat_index_flags flags;
+	enum file_lock_method lock_method;
+	mode_t create_mode;
+	gid_t create_gid;
+	uint32_t uidvalidity;
+
+	char *path;
+	int fd;
+	struct file_cache *file_cache;
+	struct dotlock_settings dotlock_set;
+
+	uoff_t locked_file_size;
+	const void *data;
+	size_t data_size;
+
+	void *mmap_base;
+	size_t mmap_size;
+
+	unsigned char default_normalize_map[256];
+	unsigned int default_partial_len;
+	unsigned int default_full_len;
+
+	bool corrupted:1;
+};
+
+#define SQUAT_PACK_MAX_SIZE ((sizeof(uint32_t) * 8 + 7) / 7)
+
+static inline void squat_pack_num(uint8_t **p, uint32_t num)
+{
+	/* number continues as long as the highest bit is set */
+	while (num >= 0x80) {
+		**p = (num & 0x7f) | 0x80;
+		*p += 1;
+		num >>= 7;
+	}
+
+	**p = num;
+	*p += 1;
+}
+
+static inline uint32_t squat_unpack_num(const uint8_t **p, const uint8_t *end)
+{
+	const uint8_t *c = *p;
+	uint32_t value = 0;
+	unsigned int bits = 0;
+
+	for (;;) {
+		if (unlikely(c == end)) {
+			/* we should never see EOF */
+			return 0;
+		}
+
+		value |= (*c & 0x7f) << bits;
+		if (*c < 0x80)
+			break;
+
+		bits += 7;
+		c++;
+	}
+
+	if (unlikely(bits >= 32)) {
+		/* broken input */
+		*p = end;
+		return 0;
+	}
+
+	*p = c + 1;
+	return value;
+}
+
+int squat_trie_create_fd(struct squat_trie *trie, const char *path, int flags);
+void squat_trie_delete(struct squat_trie *trie);
+
+#endif
diff --git a/src/plugins/fts-squat/squat-trie.c b/src/plugins/fts-squat/squat-trie.c
new file mode 100644
index 0000000..d006817
--- /dev/null
+++ b/src/plugins/fts-squat/squat-trie.c
@@ -0,0 +1,2096 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "read-full.h"
+#include "istream.h"
+#include "ostream.h"
+#include "unichar.h"
+#include "nfs-workarounds.h"
+#include "file-cache.h"
+#include "seq-range-array.h"
+#include "squat-uidlist.h"
+#include "squat-trie-private.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+#define DEFAULT_NORMALIZE_MAP_CHARS \
+	"EOTIRSACDNLMVUGPHBFWYXKJQZ0123456789@.-+#$%_&"
+#define DEFAULT_PARTIAL_LEN 4
+#define DEFAULT_FULL_LEN 4
+
+#define MAX_FAST_LEVEL 3
+#define SEQUENTIAL_COUNT 46
+
+#define TRIE_BYTES_LEFT(n) \
+	((n) * SQUAT_PACK_MAX_SIZE)
+#define TRIE_READAHEAD_SIZE \
+	I_MAX(4096, 1 + 256 + TRIE_BYTES_LEFT(256))
+
+struct squat_trie_build_context {
+	struct squat_trie *trie;
+	struct ostream *output;
+	struct squat_uidlist_build_context *uidlist_build_ctx;
+
+	struct file_lock *file_lock;
+	struct dotlock *dotlock;
+
+	uint32_t first_uid;
+	bool compress_nodes:1;
+};
+
+struct squat_trie_iterate_node {
+	struct squat_node *node;
+	ARRAY_TYPE(seq_range) shifts;
+	unsigned int idx;
+};
+
+struct squat_trie_iterate_context {
+	struct squat_trie *trie;
+	struct squat_trie_iterate_node cur;
+	ARRAY(struct squat_trie_iterate_node) parents;
+	bool failed;
+};
+
+static int squat_trie_map(struct squat_trie *trie, bool building);
+
+void squat_trie_delete(struct squat_trie *trie)
+{
+	i_unlink_if_exists(trie->path);
+	squat_uidlist_delete(trie->uidlist);
+}
+
+static void squat_trie_set_corrupted(struct squat_trie *trie)
+{
+	trie->corrupted = TRUE;
+	i_error("Corrupted file %s", trie->path);
+	squat_trie_delete(trie);
+}
+
+static void squat_trie_normalize_map_build(struct squat_trie *trie)
+{
+	static unsigned char valid_chars[] =
+		DEFAULT_NORMALIZE_MAP_CHARS;
+	unsigned int i, j;
+
+	memset(trie->default_normalize_map, 0,
+	       sizeof(trie->default_normalize_map));
+
+#if 1
+	for (i = 0, j = 1; i < sizeof(valid_chars)-1; i++) {
+		unsigned char chr = valid_chars[i];
+
+		if (chr >= 'A' && chr <= 'Z')
+			trie->default_normalize_map[chr-'A'+'a'] = j;
+		trie->default_normalize_map[chr] = j++;
+	}
+	i_assert(j <= SEQUENTIAL_COUNT);
+
+	for (i = 128; i < 256; i++)
+		trie->default_normalize_map[i] = j++;
+#else
+	for (i = 0; i < sizeof(valid_chars)-1; i++) {
+		unsigned char chr = valid_chars[i];
+
+		if (chr >= 'A' && chr <= 'Z')
+			trie->default_normalize_map[chr-'A'+'a'] = chr;
+		trie->default_normalize_map[chr] = chr;
+	}
+	for (i = 128; i < 256; i++)
+		trie->default_normalize_map[i] = i_toupper(i);
+#endif
+}
+
+static void node_free(struct squat_trie *trie, struct squat_node *node)
+{
+	struct squat_node *children;
+	unsigned int i;
+
+	if (node->leaf_string_length > 0) {
+		if (NODE_IS_DYNAMIC_LEAF(node))
+			i_free(node->children.leaf_string);
+	} else if (!node->children_not_mapped) {
+		children = NODE_CHILDREN_NODES(node);
+
+		trie->node_alloc_size -=
+			NODE_CHILDREN_ALLOC_SIZE(node->child_count);
+		for (i = 0; i < node->child_count; i++)
+			node_free(trie, &children[i]);
+
+		i_free(node->children.data);
+	}
+}
+
+struct squat_trie *
+squat_trie_init(const char *path, uint32_t uidvalidity,
+		enum file_lock_method lock_method, enum squat_index_flags flags,
+		mode_t mode, gid_t gid)
+{
+	struct squat_trie *trie;
+
+	trie = i_new(struct squat_trie, 1);
+	trie->path = i_strdup(path);
+	trie->uidlist = squat_uidlist_init(trie);
+	trie->fd = -1;
+	trie->lock_method = lock_method;
+	trie->uidvalidity = uidvalidity;
+	trie->flags = flags;
+	trie->create_mode = mode;
+	trie->create_gid = gid;
+	squat_trie_normalize_map_build(trie);
+
+	trie->dotlock_set.use_excl_lock =
+		(flags & SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL) != 0;
+	trie->dotlock_set.nfs_flush = (flags & SQUAT_INDEX_FLAG_NFS_FLUSH) != 0;
+	trie->dotlock_set.timeout = SQUAT_TRIE_LOCK_TIMEOUT;
+	trie->dotlock_set.stale_timeout = SQUAT_TRIE_DOTLOCK_STALE_TIMEOUT;
+	trie->default_partial_len = DEFAULT_PARTIAL_LEN;
+	trie->default_full_len = DEFAULT_FULL_LEN;
+	return trie;
+}
+
+static void squat_trie_close_fd(struct squat_trie *trie)
+{
+	trie->data = NULL;
+	trie->data_size = 0;
+
+	if (trie->mmap_size != 0) {
+		if (munmap(trie->mmap_base, trie->mmap_size) < 0)
+			i_error("munmap(%s) failed: %m", trie->path);
+		trie->mmap_base = NULL;
+		trie->mmap_size = 0;
+	}
+	i_close_fd_path(&trie->fd, trie->path);
+}
+
+static void squat_trie_close(struct squat_trie *trie)
+{
+	trie->corrupted = FALSE;
+	node_free(trie, &trie->root);
+	i_zero(&trie->root);
+	i_zero(&trie->hdr);
+
+	squat_trie_close_fd(trie);
+	if (trie->file_cache != NULL)
+		file_cache_free(&trie->file_cache);
+	trie->locked_file_size = 0;
+}
+
+void squat_trie_deinit(struct squat_trie **_trie)
+{
+	struct squat_trie *trie = *_trie;
+
+	*_trie = NULL;
+	squat_trie_close(trie);
+	squat_uidlist_deinit(trie->uidlist);
+	i_free(trie->path);
+	i_free(trie);
+}
+
+void squat_trie_set_partial_len(struct squat_trie *trie, unsigned int len)
+{
+	trie->default_partial_len = len;
+}
+
+void squat_trie_set_full_len(struct squat_trie *trie, unsigned int len)
+{
+	trie->default_full_len = len;
+}
+
+static void squat_trie_header_init(struct squat_trie *trie)
+{
+	i_zero(&trie->hdr);
+	trie->hdr.version = SQUAT_TRIE_VERSION;
+	trie->hdr.indexid = time(NULL);
+	trie->hdr.uidvalidity = trie->uidvalidity;
+	trie->hdr.partial_len = trie->default_partial_len;
+	trie->hdr.full_len = trie->default_full_len;
+
+	i_assert(sizeof(trie->hdr.normalize_map) ==
+		 sizeof(trie->default_normalize_map));
+	memcpy(trie->hdr.normalize_map, trie->default_normalize_map,
+	       sizeof(trie->hdr.normalize_map));
+}
+
+static int squat_trie_open_fd(struct squat_trie *trie)
+{
+	trie->fd = open(trie->path, O_RDWR);
+	if (trie->fd == -1) {
+		if (errno == ENOENT) {
+			squat_trie_header_init(trie);
+			return 0;
+		}
+		i_error("open(%s) failed: %m", trie->path);
+		return -1;
+	}
+	if (trie->file_cache != NULL)
+		file_cache_set_fd(trie->file_cache, trie->fd);
+	return 0;
+}
+
+int squat_trie_open(struct squat_trie *trie)
+{
+	squat_trie_close(trie);
+
+	if (squat_trie_open_fd(trie) < 0)
+		return -1;
+	return squat_trie_map(trie, FALSE);
+}
+
+static int squat_trie_is_file_stale(struct squat_trie *trie)
+{
+	struct stat st, st2;
+
+	if ((trie->flags & SQUAT_INDEX_FLAG_NFS_FLUSH) != 0)
+		nfs_flush_file_handle_cache(trie->path);
+	if (nfs_safe_stat(trie->path, &st) < 0) {
+		if (errno == ENOENT)
+			return 1;
+
+		i_error("stat(%s) failed: %m", trie->path);
+		return -1;
+	}
+	if (fstat(trie->fd, &st2) < 0) {
+		if (errno == ESTALE)
+			return 1;
+		i_error("fstat(%s) failed: %m", trie->path);
+		return -1;
+	}
+	trie->locked_file_size = st2.st_size;
+
+	if (st.st_ino == st2.st_ino && CMP_DEV_T(st.st_dev, st2.st_dev)) {
+		i_assert(trie->locked_file_size >= trie->data_size);
+		return 0;
+	}
+	return 1;
+}
+
+int squat_trie_refresh(struct squat_trie *trie)
+{
+	int ret;
+
+	ret = squat_trie_is_file_stale(trie);
+	if (ret > 0)
+		ret = squat_trie_open(trie);
+	return ret;
+}
+
+static int squat_trie_lock(struct squat_trie *trie, int lock_type,
+			   struct file_lock **file_lock_r,
+			   struct dotlock **dotlock_r)
+{
+	const char *error;
+	int ret;
+
+	i_assert(trie->fd != -1);
+
+	*file_lock_r = NULL;
+	*dotlock_r = NULL;
+
+	for (;;) {
+		if (trie->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
+			struct file_lock_settings lock_set = {
+				.lock_method = trie->lock_method,
+			};
+			ret = file_wait_lock(trie->fd, trie->path, lock_type,
+					     &lock_set, SQUAT_TRIE_LOCK_TIMEOUT,
+					     file_lock_r, &error);
+			if (ret < 0) {
+				i_error("squat trie %s: %s",
+					trie->path, error);
+			}
+		} else {
+			ret = file_dotlock_create(&trie->dotlock_set,
+						  trie->path, 0, dotlock_r);
+		}
+		if (ret == 0) {
+			i_error("squat trie %s: Locking timed out", trie->path);
+			return 0;
+		}
+		if (ret < 0)
+			return -1;
+
+		/* if the trie has been compressed, we need to reopen the
+		   file and try to lock again */
+		ret = squat_trie_is_file_stale(trie);
+		if (ret == 0)
+			break;
+
+		if (*file_lock_r != NULL)
+			file_unlock(file_lock_r);
+		else
+			file_dotlock_delete(dotlock_r);
+		if (ret < 0)
+			return -1;
+
+		squat_trie_close(trie);
+		if (squat_trie_open_fd(trie) < 0)
+			return -1;
+		if (trie->fd == -1)
+			return 0;
+	}
+
+	if ((trie->flags & SQUAT_INDEX_FLAG_NFS_FLUSH) != 0)
+		nfs_flush_read_cache_locked(trie->path, trie->fd);
+	return 1;
+}
+
+static void
+node_make_sequential(struct squat_trie *trie, struct squat_node *node, int level)
+{
+	const unsigned int alloc_size =
+		NODE_CHILDREN_ALLOC_SIZE(SEQUENTIAL_COUNT);
+	struct squat_node *children;
+	unsigned char *chars;
+	unsigned int i;
+
+	i_assert(node->child_count == 0);
+
+	trie->node_alloc_size += alloc_size;
+
+	node->want_sequential = FALSE;
+	node->have_sequential = TRUE;
+
+	node->child_count = SEQUENTIAL_COUNT;
+	node->children.data = i_malloc(alloc_size);
+
+	chars = NODE_CHILDREN_CHARS(node);
+	for (i = 0; i < SEQUENTIAL_COUNT; i++)
+		chars[i] = i;
+
+	if (level < MAX_FAST_LEVEL) {
+		children = NODE_CHILDREN_NODES(node);
+		for (i = 0; i < SEQUENTIAL_COUNT; i++)
+			children[i].want_sequential = TRUE;
+	}
+}
+
+static unsigned int
+node_add_child(struct squat_trie *trie, struct squat_node *node,
+	       unsigned char chr, int level)
+{
+	unsigned int old_child_count = node->child_count;
+	struct squat_node *children, *old_children;
+	unsigned char *chars;
+	size_t old_size, new_size;
+
+	i_assert(node->leaf_string_length == 0);
+
+	if (node->want_sequential) {
+		node_make_sequential(trie, node, level);
+
+		if (chr < SEQUENTIAL_COUNT)
+			return chr;
+		old_child_count = SEQUENTIAL_COUNT;
+	}
+
+	node->child_count++;
+	new_size = NODE_CHILDREN_ALLOC_SIZE(node->child_count);
+
+	if (old_child_count == 0) {
+		/* first child */
+		node->children.data = i_malloc(new_size);
+		trie->node_alloc_size += new_size;
+	} else {
+		old_size = NODE_CHILDREN_ALLOC_SIZE(old_child_count);
+		if (old_size != new_size) {
+			trie->node_alloc_size += new_size - old_size;
+			node->children.data = i_realloc(node->children.data,
+							old_size, new_size);
+		}
+
+		children = NODE_CHILDREN_NODES(node);
+		old_children = (void *)(NODE_CHILDREN_CHARS(node) +
+					MEM_ALIGN(old_child_count));
+		if (children != old_children) {
+			memmove(children, old_children,
+				old_child_count * sizeof(struct squat_node));
+		}
+	}
+
+	chars = NODE_CHILDREN_CHARS(node);
+	i_assert(chars != NULL);
+	chars[node->child_count - 1] = chr;
+	return node->child_count - 1;
+}
+
+static int
+trie_file_cache_read(struct squat_trie *trie, size_t offset, size_t size)
+{
+	if (trie->file_cache == NULL)
+		return 0;
+
+	if (file_cache_read(trie->file_cache, offset, size) < 0) {
+		i_error("read(%s) failed: %m", trie->path);
+		return -1;
+	}
+	trie->data = file_cache_get_map(trie->file_cache, &trie->data_size);
+	return 0;
+}
+
+static int
+node_read_children(struct squat_trie *trie, struct squat_node *node, int level)
+{
+	const uint8_t *data, *end;
+	const unsigned char *child_chars;
+	struct squat_node *child, *children = NULL;
+	uoff_t node_offset;
+	unsigned int i, child_idx, child_count;
+	uoff_t base_offset;
+	uint32_t num;
+
+	i_assert(node->children_not_mapped);
+	i_assert(!node->have_sequential);
+	i_assert(trie->unmapped_child_count > 0);
+	i_assert(trie->data_size <= trie->locked_file_size);
+
+	trie->unmapped_child_count--;
+	node_offset = node->children.offset;
+	node->children_not_mapped = FALSE;
+	node->children.data = NULL;
+
+	if (trie_file_cache_read(trie, node_offset, TRIE_READAHEAD_SIZE) < 0)
+		return -1;
+	if (unlikely(node_offset >= trie->data_size)) {
+		squat_trie_set_corrupted(trie);
+		return -1;
+	}
+
+	data = CONST_PTR_OFFSET(trie->data, node_offset);
+	end = CONST_PTR_OFFSET(trie->data, trie->data_size);
+	child_count = *data++;
+	if (unlikely(node_offset + child_count >= trie->data_size)) {
+		squat_trie_set_corrupted(trie);
+		return -1;
+	}
+
+	if (child_count == 0)
+		return 0;
+
+	child_chars = data;
+	data += child_count;
+
+	/* get child offsets */
+	base_offset = node_offset;
+	for (i = 0; i < child_count; i++) {
+		/* we always start with !have_sequential, so at i=0 this
+		   check always goes to add the first child */
+		if (node->have_sequential && child_chars[i] < SEQUENTIAL_COUNT)
+			child_idx = child_chars[i];
+		else {
+			child_idx = node_add_child(trie, node, child_chars[i],
+						   level);
+			children = NODE_CHILDREN_NODES(node);
+		}
+
+		i_assert(children != NULL);
+
+		child = &children[child_idx];
+
+		/* 1) child offset */
+		num = squat_unpack_num(&data, end);
+		if (num == 0) {
+			/* no children */
+		} else {
+			if ((num & 1) != 0) {
+				base_offset += num >> 1;
+			} else {
+				base_offset -= num >> 1;
+			}
+			if (base_offset >= trie->locked_file_size) {
+				squat_trie_set_corrupted(trie);
+				return -1;
+			}
+			trie->unmapped_child_count++;
+			child->children_not_mapped = TRUE;
+			child->children.offset = base_offset;
+		}
+
+		/* 2) uidlist */
+		child->uid_list_idx = squat_unpack_num(&data, end);
+		if (child->uid_list_idx == 0) {
+			/* we don't write nodes with empty uidlists */
+			squat_trie_set_corrupted(trie);
+			return -1;
+		}
+		if (!UIDLIST_IS_SINGLETON(child->uid_list_idx)) {
+			/* 3) next uid */
+			child->next_uid = squat_unpack_num(&data, end) + 1;
+		} else {
+			uint32_t idx = child->uid_list_idx;
+
+			child->next_uid = 1 +
+				squat_uidlist_singleton_last_uid(idx);
+		}
+
+		/* 4) unused uids + leaf string flag */
+		num = squat_unpack_num(&data, end);
+		child->unused_uids = num >> 1;
+		if ((num & 1) != 0) {
+			/* leaf string */
+			unsigned int len;
+			unsigned char *dest;
+
+			/* 5) leaf string length */
+			len = child->leaf_string_length =
+				squat_unpack_num(&data, end) + 1;
+			if (!NODE_IS_DYNAMIC_LEAF(child))
+				dest = child->children.static_leaf_string;
+			else {
+				dest = child->children.leaf_string =
+					i_malloc(len);
+			}
+
+			if (trie->file_cache != NULL) {
+				/* the string may be long -
+				   recalculate the end pos */
+				size_t offset, size;
+
+				offset = (const char *)data -
+					(const char *)trie->data;
+				size = len + TRIE_BYTES_LEFT(child_count - i);
+
+				if (trie_file_cache_read(trie, offset,
+							 size) < 0)
+					return -1;
+				data = CONST_PTR_OFFSET(trie->data, offset);
+				end = CONST_PTR_OFFSET(trie->data,
+						       trie->data_size);
+				child_chars = CONST_PTR_OFFSET(trie->data,
+							       node_offset + 1);
+			}
+
+			if ((size_t)(end - data) < len) {
+				squat_trie_set_corrupted(trie);
+				return -1;
+			}
+			memcpy(dest, data, len);
+			data += len;
+		}
+	}
+	if (unlikely(data == end)) {
+		/* we should never get this far */
+		squat_trie_set_corrupted(trie);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+node_write_children(struct squat_trie_build_context *ctx,
+		    struct squat_node *node, const uoff_t *node_offsets)
+{
+	struct squat_node *children;
+	const unsigned char *chars;
+	uint8_t child_count, buf[SQUAT_PACK_MAX_SIZE * 5], *bufp;
+	uoff_t base_offset;
+	unsigned int i;
+
+	chars = NODE_CHILDREN_CHARS(node);
+	children = NODE_CHILDREN_NODES(node);
+
+	base_offset = ctx->output->offset;
+	child_count = node->child_count;
+	o_stream_nsend(ctx->output, &child_count, 1);
+	o_stream_nsend(ctx->output, chars, child_count);
+
+	for (i = 0; i < child_count; i++) {
+		bufp = buf;
+		/* 1) child offset */
+		if (node_offsets[i] == 0)
+			*bufp++ = 0;
+		else if (node_offsets[i] >= base_offset) {
+			squat_pack_num(&bufp,
+				((node_offsets[i] - base_offset) << 1) | 1);
+			base_offset = node_offsets[i];
+		} else {
+			squat_pack_num(&bufp,
+				       (base_offset - node_offsets[i]) << 1);
+			base_offset = node_offsets[i];
+		}
+
+		/* 2) uidlist */
+		squat_pack_num(&bufp, children[i].uid_list_idx);
+		if (!UIDLIST_IS_SINGLETON(children[i].uid_list_idx)) {
+			/* 3) next uid */
+			squat_pack_num(&bufp, children[i].next_uid - 1);
+		}
+
+		if (children[i].leaf_string_length == 0) {
+			/* 4a) unused uids */
+			squat_pack_num(&bufp, children[i].unused_uids << 1);
+			o_stream_nsend(ctx->output, buf, bufp - buf);
+		} else {
+			i_assert(node_offsets[i] == 0);
+			/* 4b) unused uids + flag */
+			squat_pack_num(&bufp, (children[i].unused_uids << 1) | 1);
+			/* 5) leaf string length */
+			squat_pack_num(&bufp, children[i].leaf_string_length - 1);
+			o_stream_nsend(ctx->output, buf, bufp - buf);
+			o_stream_nsend(ctx->output,
+				       NODE_LEAF_STRING(&children[i]),
+				       children[i].leaf_string_length);
+		}
+	}
+}
+
+static inline void
+node_add_uid(struct squat_trie_build_context *ctx, uint32_t uid,
+	     struct squat_node *node)
+{
+	if (uid < node->next_uid) {
+		/* duplicate */
+		return;
+	}
+	node->unused_uids += uid - node->next_uid;
+	node->next_uid = uid + 1;
+
+	node->uid_list_idx =
+		squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
+					    node->uid_list_idx, uid);
+}
+
+static void
+node_split_string(struct squat_trie_build_context *ctx, struct squat_node *node)
+{
+	struct squat_node *child;
+	unsigned char *str;
+	unsigned int uid, idx, leafstr_len = node->leaf_string_length;
+
+	i_assert(leafstr_len > 0);
+
+	/* make a copy of the leaf string and convert to normal node by
+	   removing it. */
+	str = t_malloc_no0(leafstr_len);
+	if (!NODE_IS_DYNAMIC_LEAF(node))
+		memcpy(str, node->children.static_leaf_string, leafstr_len);
+	else {
+		memcpy(str, node->children.leaf_string, leafstr_len);
+		i_free(node->children.leaf_string);
+	}
+	node->leaf_string_length = 0;
+
+	/* create a new child node for the rest of the string */
+	idx = node_add_child(ctx->trie, node, str[0], MAX_FAST_LEVEL);
+	child = NODE_CHILDREN_NODES(node) + idx;
+
+	/* update uidlist to contain all of parent's UIDs */
+	child->next_uid =  node->next_uid - node->unused_uids;
+	for (uid = 0; uid < child->next_uid; uid++) {
+		child->uid_list_idx =
+			squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
+						    child->uid_list_idx, uid);
+	}
+
+	i_assert(!child->have_sequential && child->children.data == NULL);
+	if (leafstr_len > 1) {
+		/* make the child a leaf string */
+		leafstr_len--;
+		child->leaf_string_length = leafstr_len;
+		if (!NODE_IS_DYNAMIC_LEAF(child)) {
+			memcpy(child->children.static_leaf_string,
+			       str + 1, leafstr_len);
+		} else {
+			child->children.leaf_string = i_malloc(leafstr_len);
+			memcpy(child->children.leaf_string,
+			       str + 1, leafstr_len);
+		}
+	}
+}
+
+static bool
+node_leaf_string_add_or_split(struct squat_trie_build_context *ctx,
+			      struct squat_node *node,
+			      const unsigned char *data, unsigned int data_len)
+{
+	const unsigned char *str = NODE_LEAF_STRING(node);
+	const unsigned int leafstr_len = node->leaf_string_length;
+	unsigned int i;
+
+	if (data_len != leafstr_len) {
+		/* different lengths, can't match */
+		T_BEGIN {
+			node_split_string(ctx, node);
+		} T_END;
+		return FALSE;
+	}
+
+	for (i = 0; i < data_len; i++) {
+		if (data[i] != str[i]) {
+			/* non-match */
+			T_BEGIN {
+				node_split_string(ctx, node);
+			} T_END;
+			return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+static int squat_build_add(struct squat_trie_build_context *ctx, uint32_t uid,
+			   const unsigned char *data, unsigned int size)
+{
+	struct squat_trie *trie = ctx->trie;
+	struct squat_node *node = &trie->root;
+	const unsigned char *end = data + size;
+	unsigned char *chars;
+	unsigned int idx;
+	int level = 0;
+
+	for (;;) {
+		if (node->children_not_mapped) {
+			if (unlikely(node_read_children(trie, node, level) < 0))
+				return -1;
+		}
+
+		if (node->leaf_string_length != 0) {
+			/* the whole string must match or we need to split
+			   the node */
+			if (node_leaf_string_add_or_split(ctx, node, data,
+							  end - data)) {
+				node_add_uid(ctx, uid, node);
+				return 0;
+			}
+		}
+
+		node_add_uid(ctx, uid, node);
+
+		if (unlikely(uid < node->unused_uids)) {
+			squat_trie_set_corrupted(trie);
+			return -1;
+		}
+		/* child node's UIDs are relative to ours. so for example if
+		   we're adding UID 4 and this node now has [2,4] UIDs,
+		   unused_uids=3 and so the child node will be adding
+		   UID 4-3 = 1. */
+		uid -= node->unused_uids;
+
+		if (data == end)
+			return 0;
+		level++;
+
+		if (node->have_sequential) {
+			i_assert(node->child_count >= SEQUENTIAL_COUNT);
+			if (*data < SEQUENTIAL_COUNT) {
+				idx = *data;
+				goto found;
+			}
+			idx = SEQUENTIAL_COUNT;
+		} else {
+			idx = 0;
+		}
+		chars = NODE_CHILDREN_CHARS(node);
+		for (; idx < node->child_count; idx++) {
+			if (chars[idx] == *data)
+				goto found;
+		}
+		break;
+	found:
+		data++;
+		node = NODE_CHILDREN_NODES(node) + idx;
+	}
+
+	/* create new children */
+	i_assert(node->leaf_string_length == 0);
+
+	for (;;) {
+		idx = node_add_child(trie, node, *data,
+				     size - (end - data) + 1);
+		node = NODE_CHILDREN_NODES(node) + idx;
+
+		node_add_uid(ctx, uid, node);
+		uid = 0;
+
+		if (++data == end)
+			break;
+
+		if (!node->have_sequential) {
+			/* convert the node into a leaf string */
+			unsigned int len = end - data;
+
+			i_assert(node->children.data == NULL);
+			node->leaf_string_length = len;
+			if (!NODE_IS_DYNAMIC_LEAF(node)) {
+				memcpy(node->children.static_leaf_string,
+				       data, len);
+			} else {
+				node->children.leaf_string = i_malloc(len);
+				memcpy(node->children.leaf_string, data, len);
+			}
+			break;
+		}
+	}
+	return 0;
+}
+
+static int
+squat_build_word_bytes(struct squat_trie_build_context *ctx, uint32_t uid,
+		       const unsigned char *data, unsigned int size)
+{
+	struct squat_trie *trie = ctx->trie;
+	unsigned int i;
+
+	if (trie->hdr.full_len <= trie->hdr.partial_len)
+		i = 0;
+	else {
+		/* the first word is longer than others */
+		if (squat_build_add(ctx, uid, data,
+				    I_MIN(size, trie->hdr.full_len)) < 0)
+			return -1;
+		i = 1;
+	}
+
+	for (; i < size; i++) {
+		if (squat_build_add(ctx, uid, data + i,
+				    I_MIN(trie->hdr.partial_len, size-i)) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int
+squat_build_word(struct squat_trie_build_context *ctx, uint32_t uid,
+		 const unsigned char *data, const uint8_t *char_lengths,
+		 unsigned int size)
+{
+	struct squat_trie *trie = ctx->trie;
+	unsigned int i, j, bytelen;
+
+	if (char_lengths == NULL) {
+		/* optimization path: all characters are bytes */
+		return squat_build_word_bytes(ctx, uid, data, size);
+	}
+
+	if (trie->hdr.full_len <= trie->hdr.partial_len)
+		i = 0;
+	else {
+		/* the first word is longer than others */
+		bytelen = 0;
+		for (j = 0; j < trie->hdr.full_len && bytelen < size; j++)
+			bytelen += char_lengths[bytelen];
+		i_assert(bytelen <= size);
+
+		if (squat_build_add(ctx, uid, data, bytelen) < 0)
+			return -1;
+		i = char_lengths[0];
+	}
+
+	for (; i < size; i += char_lengths[i]) {
+		bytelen = 0;
+		for (j = 0; j < trie->hdr.partial_len && i+bytelen < size; j++)
+			bytelen += char_lengths[i + bytelen];
+		i_assert(i + bytelen <= size);
+
+		if (squat_build_add(ctx, uid, data + i, bytelen) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static unsigned char *
+squat_data_normalize(struct squat_trie *trie, const unsigned char *data,
+		     unsigned int size)
+{
+	static const unsigned char replacement_utf8[] = { 0xef, 0xbf, 0xbd };
+	unsigned char *dest;
+	unsigned int i;
+
+	dest = t_malloc_no0(size);
+	for (i = 0; i < size; i++) {
+		if (data[i] == replacement_utf8[0] && i + 2 < size &&
+		    data[i+1] == replacement_utf8[1] &&
+		    data[i+2] == replacement_utf8[2]) {
+			/* Don't index replacement character */
+			dest[i++] = 0;
+			dest[i++] = 0;
+			dest[i] = 0;
+		} else {
+			dest[i] = trie->hdr.normalize_map[data[i]];
+		}
+	}
+	return dest;
+}
+
+static int
+squat_trie_build_more_real(struct squat_trie_build_context *ctx,
+			   uint32_t uid, enum squat_index_type type,
+			   const unsigned char *input, unsigned int size)
+{
+	struct squat_trie *trie = ctx->trie;
+	const unsigned char *data;
+	uint8_t *char_lengths;
+	unsigned int i, start = 0;
+	bool multibyte_chars = FALSE;
+	int ret = 0;
+
+	uid = uid * 2 + (type == SQUAT_INDEX_TYPE_HEADER ? 1 : 0);
+
+	char_lengths = t_malloc_no0(size);
+	data = squat_data_normalize(trie, input, size);
+	for (i = 0; i < size; i++) {
+		char_lengths[i] = uni_utf8_char_bytes(input[i]);
+		if (char_lengths[i] != 1)
+			multibyte_chars = TRUE;
+		if (data[i] != '\0')
+			continue;
+
+		while (start < i && data[start] == '\0')
+			start++;
+		if (i != start) {
+			if (squat_build_word(ctx, uid, data + start,
+					     !multibyte_chars ? NULL :
+					     char_lengths + start,
+					     i - start) < 0) {
+				ret = -1;
+				start = i;
+				break;
+			}
+		}
+		start = i + 1;
+	}
+	while (start < i && data[start] == '\0')
+		start++;
+	if (i != start) {
+		if (squat_build_word(ctx, uid, data + start,
+				     !multibyte_chars ? NULL :
+				     char_lengths + start, i - start) < 0)
+			ret = -1;
+	}
+	return ret;
+}
+
+int squat_trie_build_more(struct squat_trie_build_context *ctx,
+			  uint32_t uid, enum squat_index_type type,
+			  const unsigned char *input, unsigned int size)
+{
+	int ret = 0;
+
+	if (size != 0) T_BEGIN {
+		ret = squat_trie_build_more_real(ctx, uid, type, input, size);
+	} T_END;
+	return ret;
+}
+
+static void
+node_drop_unused_children(struct squat_trie *trie, struct squat_node *node)
+{
+	unsigned char *chars;
+	struct squat_node *children_src, *children_dest;
+	unsigned int i, j, orig_child_count = node->child_count;
+
+	chars = NODE_CHILDREN_CHARS(node);
+	children_src = NODE_CHILDREN_NODES(node);
+
+	/* move chars */
+	for (i = j = 0; i < orig_child_count; i++) {
+		if (children_src[i].next_uid != 0)
+			chars[j++] = chars[i];
+	}
+	node->child_count = j;
+
+	/* move children. note that children_dest may point to different
+	   location than children_src, although they both point to the
+	   same node. */
+	children_dest = NODE_CHILDREN_NODES(node);
+	for (i = j = 0; i < orig_child_count; i++) {
+		if (children_src[i].next_uid != 0)
+			children_dest[j++] = children_src[i];
+		else
+			node_free(trie, &children_src[i]);
+	}
+}
+
+static int
+squat_write_node(struct squat_trie_build_context *ctx, struct squat_node *node,
+		 uoff_t *node_offset_r, int level)
+{
+	struct squat_trie *trie = ctx->trie;
+	struct squat_node *children;
+	unsigned int i;
+	uoff_t *node_offsets;
+	uint8_t child_count;
+	int ret;
+
+	i_assert(node->next_uid != 0);
+
+	if (node->children_not_mapped && ctx->compress_nodes) {
+		if (node_read_children(trie, node, MAX_FAST_LEVEL) < 0)
+			return -1;
+	}
+
+	node->have_sequential = FALSE;
+	node_drop_unused_children(trie, node);
+
+	child_count = node->child_count;
+	if (child_count == 0) {
+		i_assert(!node->children_not_mapped ||
+			 node->leaf_string_length == 0);
+		*node_offset_r = !node->children_not_mapped ? 0 :
+			node->children.offset;
+		return 0;
+	}
+	i_assert(!node->children_not_mapped);
+
+	trie->hdr.node_count++;
+
+	children = NODE_CHILDREN_NODES(node);
+	node_offsets = t_new(uoff_t, child_count);
+	for (i = 0; i < child_count; i++) {
+		T_BEGIN {
+			ret = squat_write_node(ctx, &children[i],
+					       &node_offsets[i], level + 1);
+		} T_END;
+		if (ret < 0)
+			return -1;
+	}
+
+	*node_offset_r = ctx->output->offset;
+	node_write_children(ctx, node, node_offsets);
+	return 0;
+}
+
+static int squat_write_nodes(struct squat_trie_build_context *ctx)
+{
+	struct squat_trie *trie = ctx->trie;
+	uoff_t node_offset;
+	int ret;
+
+	if (ctx->trie->root.next_uid == 0)
+		return 0;
+
+	T_BEGIN {
+		ret = squat_write_node(ctx, &ctx->trie->root, &node_offset, 0);
+	} T_END;
+	if (ret < 0)
+		return -1;
+
+	trie->hdr.root_offset = node_offset;
+	trie->hdr.root_unused_uids = trie->root.unused_uids;
+	trie->hdr.root_next_uid = trie->root.next_uid;
+	trie->hdr.root_uidlist_idx = trie->root.uid_list_idx;
+	return 0;
+}
+
+static struct squat_trie_iterate_context *
+squat_trie_iterate_init(struct squat_trie *trie)
+{
+	struct squat_trie_iterate_context *ctx;
+
+	ctx = i_new(struct squat_trie_iterate_context, 1);
+	ctx->trie = trie;
+	ctx->cur.node = &trie->root;
+	i_array_init(&ctx->parents, trie->hdr.partial_len*2);
+	return ctx;
+}
+
+static int
+squat_trie_iterate_deinit(struct squat_trie_iterate_context *ctx)
+{
+	struct squat_trie_iterate_node *node;
+	int ret = ctx->failed ? -1 : 0;
+
+	if (array_is_created(&ctx->cur.shifts)) {
+		array_foreach_modifiable(&ctx->parents, node)
+			array_free(&node->shifts);
+		array_free(&ctx->cur.shifts);
+	}
+	array_free(&ctx->parents);
+	i_free(ctx);
+	return ret;
+}
+
+static struct squat_node *
+squat_trie_iterate_first(struct squat_trie_iterate_context *ctx)
+{
+	if (ctx->cur.node->children_not_mapped) {
+		if (node_read_children(ctx->trie, ctx->cur.node, 1) < 0) {
+			ctx->failed = TRUE;
+			return NULL;
+		}
+	}
+	return ctx->cur.node;
+}
+
+static struct squat_node *
+squat_trie_iterate_next(struct squat_trie_iterate_context *ctx,
+			ARRAY_TYPE(seq_range) *shifts_r)
+{
+	struct squat_trie_iterate_node *iter_nodes;
+	struct squat_node *children;
+	unsigned int count, shift_count = 0;
+
+	while (ctx->cur.idx == ctx->cur.node->child_count ||
+	       ctx->cur.node->uid_list_idx == 0)
+	{
+		iter_nodes = array_get_modifiable(&ctx->parents, &count);
+		if (count == 0)
+			return NULL;
+
+		if (array_is_created(&ctx->cur.shifts))
+			array_free(&ctx->cur.shifts);
+		ctx->cur = iter_nodes[count-1];
+		array_delete(&ctx->parents, count-1, 1);
+	}
+
+	*shifts_r = ctx->cur.shifts;
+	if (array_is_created(&ctx->cur.shifts))
+		shift_count = array_count(&ctx->cur.shifts);
+
+	children = NODE_CHILDREN_NODES(ctx->cur.node);
+	while (children[ctx->cur.idx++].uid_list_idx == 0) {
+		if (ctx->cur.idx == ctx->cur.node->child_count) {
+			/* no more non-empty children in this node */
+			return squat_trie_iterate_next(ctx, shifts_r);
+		}
+	}
+	array_push_back(&ctx->parents, &ctx->cur);
+	ctx->cur.node = &children[ctx->cur.idx-1];
+	ctx->cur.idx = 0;
+	if (shift_count != 0)
+		i_array_init(&ctx->cur.shifts, shift_count);
+	else
+		i_zero(&ctx->cur.shifts);
+	return squat_trie_iterate_first(ctx);
+}
+
+static void
+squat_uidlist_update_expunged_uids(const ARRAY_TYPE(seq_range) *shifts_arr,
+				   ARRAY_TYPE(seq_range) *child_shifts,
+				   ARRAY_TYPE(seq_range) *uids_arr,
+				   struct squat_trie *trie,
+				   struct squat_node *node, bool do_shifts)
+{
+	const struct seq_range *shifts;
+	struct seq_range *uids, shift;
+	unsigned int i, uid_idx, uid_count, shift_count;
+	uint32_t child_shift_seq1, child_shift_count, seq_high;
+	unsigned int shift_sum = 0, child_sum = 0;
+
+	if (!array_is_created(shifts_arr)) {
+		i_assert(node->uid_list_idx != 0 || node->child_count == 0);
+		return;
+	}
+
+	/* we'll recalculate this */
+	node->unused_uids = 0;
+
+	uids = array_get_modifiable(uids_arr, &uid_count);
+	shifts = array_get(shifts_arr, &shift_count);
+	for (i = 0, uid_idx = 0, seq_high = 0;; ) {
+		/* skip UID ranges until we skip/overlap shifts */
+		while (uid_idx < uid_count &&
+		       (i == shift_count ||
+			I_MAX(shifts[i].seq1, seq_high) > uids[uid_idx].seq2))
+		{
+			i_assert(uids[uid_idx].seq1 >= shift_sum);
+			uids[uid_idx].seq1 -= shift_sum;
+			uids[uid_idx].seq2 -= shift_sum;
+			child_sum += uids[uid_idx].seq2 -
+				uids[uid_idx].seq1 + 1;
+
+			if (uid_idx > 0 &&
+			    uids[uid_idx-1].seq2 >= uids[uid_idx].seq1 - 1) {
+				/* we can merge this and the previous range */
+				i_assert(uids[uid_idx-1].seq2 ==
+					 uids[uid_idx].seq1 - 1);
+				uids[uid_idx-1].seq2 = uids[uid_idx].seq2;
+				array_delete(uids_arr, uid_idx, 1);
+				uids = array_get_modifiable(uids_arr,
+							    &uid_count);
+			} else {
+				if (uid_idx == 0)
+					node->unused_uids += uids[0].seq1;
+				else {
+					node->unused_uids +=
+						uids[uid_idx].seq1 -
+						uids[uid_idx-1].seq2 - 1;
+				}
+				uid_idx++;
+			}
+		}
+		if (uid_idx == uid_count)
+			break;
+
+		shift.seq1 = I_MAX(shifts[i].seq1, seq_high);
+		shift.seq2 = shifts[i].seq2;
+		if (shift.seq2 < uids[uid_idx].seq1) {
+			/* shift is entirely before UID range */
+			shift_sum += shift.seq2 - shift.seq1 + 1;
+			i++;
+		} else {
+			/* handle shifts before UID range */
+			if (shift.seq1 < uids[uid_idx].seq1) {
+				shift_sum += uids[uid_idx].seq1 - shift.seq1;
+				shift.seq1 = uids[uid_idx].seq1;
+			}
+			/* update child shifts */
+			child_shift_seq1 = child_sum +
+				shift.seq1 - uids[uid_idx].seq1;
+			child_shift_count =
+				I_MIN(shift.seq2, uids[uid_idx].seq2) -
+				shift.seq1 + 1;
+			seq_range_array_add_range(child_shifts,
+						  child_shift_seq1,
+						  child_shift_seq1 +
+						  child_shift_count - 1);
+			child_sum += child_shift_count;
+
+			/* if the shifts continue after the UID range,
+			   treat it in the next loop iteration */
+			if (shift.seq2 <= uids[uid_idx].seq2)
+				i++;
+			else
+				seq_high = uids[uid_idx].seq2 + 1;
+
+			/* update UIDs - no matter where within the UID range
+			   the shifts happened, the result is the same:
+			   shift number of UIDs are removed, and the rest
+			   are decreased by shift_sum.
+
+			      123     uids child_shifts
+			   a) s    -> 12   1
+			   b)  s   -> 12   2
+			   c)   s  -> 12   3
+			*/
+			if (uids[uid_idx].seq1 +
+			    child_shift_count > uids[uid_idx].seq2) {
+				/* removed completely */
+				array_delete(uids_arr, uid_idx, 1);
+				uids = array_get_modifiable(uids_arr,
+							    &uid_count);
+			} else if (do_shifts) {
+				/* the next loop iteration fixes the UIDs */
+				uids[uid_idx].seq1 += child_shift_count;
+			} else {
+				seq_range_array_remove_range(uids_arr,
+					shift.seq1,
+					I_MIN(shift.seq2, uids[uid_idx].seq2));
+				uids = array_get_modifiable(uids_arr,
+							    &uid_count);
+			}
+			shift_sum += child_shift_count;
+		}
+		if (!do_shifts) {
+			/* root node - UIDs are only removed, not shifted */
+			shift_sum = 0;
+		}
+	}
+
+	if (uid_count == 0) {
+		/* no UIDs left, delete the node's children and mark it
+		   unused */
+		if (!NODE_IS_DYNAMIC_LEAF(node))
+			node_free(trie, node);
+
+		node->child_count = 0;
+		node->have_sequential = FALSE;
+		node->next_uid = 0;
+	} else {
+		if (do_shifts)
+			node->next_uid = uids[uid_count-1].seq2 + 1;
+		else {
+			node->unused_uids += (node->next_uid - 1) -
+				uids[uid_count-1].seq2;
+		}
+	}
+}
+
+static int
+squat_trie_expunge_uidlists(struct squat_trie_build_context *ctx,
+			    struct squat_uidlist_rebuild_context *rebuild_ctx,
+			    struct squat_trie_iterate_context *iter,
+			    const ARRAY_TYPE(seq_range) *expunged_uids)
+{
+	struct squat_node *node;
+	ARRAY_TYPE(seq_range) uid_range, root_shifts, shifts;
+	bool shift = FALSE;
+	int ret = 0;
+
+	node = squat_trie_iterate_first(iter);
+	if (node->uid_list_idx == 0)
+		return 0;
+
+	i_array_init(&uid_range, 1024);
+	i_array_init(&root_shifts, array_count(expunged_uids));
+	array_append_array(&root_shifts, expunged_uids);
+
+	if (array_count(expunged_uids) > 0)
+		i_array_init(&iter->cur.shifts, array_count(expunged_uids));
+
+	shifts = root_shifts;
+	do {
+		i_assert(node->uid_list_idx != 0);
+		array_clear(&uid_range);
+		if (squat_uidlist_get_seqrange(ctx->trie->uidlist,
+					       node->uid_list_idx,
+					       &uid_range) < 0) {
+			ret = -1;
+			break;
+		}
+		squat_uidlist_update_expunged_uids(&shifts, &iter->cur.shifts,
+						   &uid_range, ctx->trie, node,
+						   shift);
+		node->uid_list_idx =
+			squat_uidlist_rebuild_nextu(rebuild_ctx, &uid_range);
+		i_assert(node->uid_list_idx != 0 || node->next_uid == 0);
+
+		node = squat_trie_iterate_next(iter, &shifts);
+		shift = TRUE;
+	} while (node != NULL);
+	array_free(&uid_range);
+	array_free(&root_shifts);
+	return ret;
+}
+
+static int
+squat_trie_renumber_uidlists2(struct squat_trie_build_context *ctx,
+			      struct squat_uidlist_rebuild_context *rebuild_ctx,
+			      struct squat_trie_iterate_context *iter)
+{
+	struct squat_node *node;
+	ARRAY_TYPE(seq_range) shifts;
+	ARRAY_TYPE(uint32_t) uids;
+	int ret = 0;
+
+	node = squat_trie_iterate_first(iter);
+	if (node->uid_list_idx == 0)
+		return 0;
+
+	i_array_init(&uids, 1024);
+	while (node != NULL) {
+		i_assert(node->uid_list_idx != 0);
+		if (!UIDLIST_IS_SINGLETON(node->uid_list_idx)) {
+			/* rebuild the uidlist */
+			array_clear(&uids);
+			if (squat_uidlist_get(ctx->trie->uidlist,
+					      node->uid_list_idx, &uids) < 0) {
+				ret = -1;
+				break;
+			}
+			node->uid_list_idx =
+				squat_uidlist_rebuild_next(rebuild_ctx, &uids);
+		}
+		node = squat_trie_iterate_next(iter, &shifts);
+	}
+	array_free(&uids);
+	return ret;
+}
+
+static int
+squat_trie_renumber_uidlists(struct squat_trie_build_context *ctx,
+			     const ARRAY_TYPE(seq_range) *expunged_uids,
+			     bool compress)
+{
+	struct squat_trie_iterate_context *iter;
+	struct squat_uidlist_rebuild_context *rebuild_ctx;
+	time_t now;
+	int ret = 0;
+
+	if ((ret = squat_uidlist_rebuild_init(ctx->uidlist_build_ctx,
+					      compress, &rebuild_ctx)) <= 0)
+		return ret;
+
+	now = time(NULL);
+	ctx->trie->hdr.indexid =
+		I_MAX((unsigned int)now, ctx->trie->hdr.indexid + 1);
+
+	iter = squat_trie_iterate_init(ctx->trie);
+	if (expunged_uids != NULL) {
+		ret = squat_trie_expunge_uidlists(ctx, rebuild_ctx, iter,
+						  expunged_uids);
+	} else {
+		ret = squat_trie_renumber_uidlists2(ctx, rebuild_ctx, iter);
+	}
+	if (squat_trie_iterate_deinit(iter) < 0)
+		ret = -1;
+
+	/* lock the trie before we rename uidlist */
+	i_assert(ctx->file_lock == NULL && ctx->dotlock == NULL);
+	if (squat_trie_lock(ctx->trie, F_WRLCK,
+			    &ctx->file_lock, &ctx->dotlock) <= 0)
+		ret = -1;
+	return squat_uidlist_rebuild_finish(rebuild_ctx, ret < 0);
+}
+
+static bool squat_trie_check_header(struct squat_trie *trie)
+{
+	if (trie->hdr.version != SQUAT_TRIE_VERSION ||
+	    trie->hdr.uidvalidity != trie->uidvalidity)
+		return FALSE;
+
+	if (trie->hdr.partial_len > trie->hdr.full_len) {
+		i_error("Corrupted %s: partial len > full len", trie->path);
+		return FALSE;
+	}
+	if (trie->hdr.full_len == 0) {
+		i_error("Corrupted %s: full len=0", trie->path);
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static int squat_trie_map_header(struct squat_trie *trie)
+{
+	int ret;
+
+	if (trie->locked_file_size == 0) {
+		/* newly created file */
+		squat_trie_header_init(trie);
+		return 1;
+	}
+	i_assert(trie->fd != -1);
+
+	if ((trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) != 0) {
+		ret = pread_full(trie->fd, &trie->hdr, sizeof(trie->hdr), 0);
+		if (ret <= 0) {
+			if (ret < 0) {
+				i_error("pread(%s) failed: %m", trie->path);
+				return -1;
+			}
+			i_error("Corrupted %s: File too small", trie->path);
+			return 0;
+		}
+		trie->data = NULL;
+		trie->data_size = 0;
+	} else {
+		if (trie->locked_file_size < sizeof(trie->hdr)) {
+			i_error("Corrupted %s: File too small", trie->path);
+			return 0;
+		}
+		if (trie->mmap_size != 0) {
+			if (munmap(trie->mmap_base, trie->mmap_size) < 0)
+				i_error("munmap(%s) failed: %m", trie->path);
+		}
+
+		trie->mmap_size = trie->locked_file_size;
+		trie->mmap_base = mmap(NULL, trie->mmap_size,
+				       PROT_READ | PROT_WRITE,
+				       MAP_SHARED, trie->fd, 0);
+		if (trie->mmap_base == MAP_FAILED) {
+			trie->data = trie->mmap_base = NULL;
+			trie->data_size = trie->mmap_size = 0;
+			i_error("mmap(%s) failed: %m", trie->path);
+			return -1;
+		}
+		memcpy(&trie->hdr, trie->mmap_base, sizeof(trie->hdr));
+		trie->data = trie->mmap_base;
+		trie->data_size = trie->mmap_size;
+	}
+
+	return squat_trie_check_header(trie) ? 1 : 0;
+}
+
+static int squat_trie_map(struct squat_trie *trie, bool building)
+{
+	struct file_lock *file_lock = NULL;
+	struct dotlock *dotlock = NULL;
+	bool changed;
+	int ret;
+
+	if (trie->fd != -1) {
+		if (squat_trie_lock(trie, F_RDLCK, &file_lock, &dotlock) <= 0)
+			return -1;
+		if ((trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) != 0 &&
+		    trie->file_cache == NULL)
+			trie->file_cache = file_cache_new_path(trie->fd, trie->path);
+	}
+
+	ret = squat_trie_map_header(trie);
+	if (ret == 0) {
+		if (file_lock != NULL)
+			file_unlock(&file_lock);
+		else
+			file_dotlock_delete(&dotlock);
+		squat_trie_delete(trie);
+		squat_trie_close(trie);
+		squat_trie_header_init(trie);
+	}
+	changed = trie->root.children.offset != trie->hdr.root_offset;
+
+	if (changed || trie->hdr.root_offset == 0) {
+		node_free(trie, &trie->root);
+		i_zero(&trie->root);
+		trie->root.want_sequential = TRUE;
+		trie->root.unused_uids = trie->hdr.root_unused_uids;
+		trie->root.next_uid = trie->hdr.root_next_uid;
+		trie->root.uid_list_idx = trie->hdr.root_uidlist_idx;
+		trie->root.children.offset = trie->hdr.root_offset;
+
+		if (trie->hdr.root_offset == 0) {
+			trie->unmapped_child_count = 0;
+			trie->root.children_not_mapped = FALSE;
+		} else {
+			trie->unmapped_child_count = 1;
+			trie->root.children_not_mapped = TRUE;
+		}
+	}
+
+	if (ret >= 0 && !building) {
+		/* do this while we're still locked */
+		ret = squat_uidlist_refresh(trie->uidlist);
+	}
+
+	if (file_lock != NULL)
+		file_unlock(&file_lock);
+	if (dotlock != NULL)
+		file_dotlock_delete(&dotlock);
+	if (ret < 0)
+		return -1;
+
+	return trie->hdr.root_offset == 0 || !changed ? 0 :
+		node_read_children(trie, &trie->root, 1);
+}
+
+int squat_trie_create_fd(struct squat_trie *trie, const char *path, int flags)
+{
+	mode_t old_mask;
+	int fd;
+
+	old_mask = umask(0);
+	fd = open(path, O_RDWR | O_CREAT | flags, trie->create_mode);
+	umask(old_mask);
+	if (fd == -1) {
+		i_error("creat(%s) failed: %m", path);
+		return -1;
+	}
+	if (trie->create_gid != (gid_t)-1) {
+		if (fchown(fd, (uid_t)-1, trie->create_gid) < 0) {
+			i_error("fchown(%s, -1, %ld) failed: %m",
+				path, (long)trie->create_gid);
+			i_close_fd(&fd);
+			return -1;
+		}
+	}
+	return fd;
+}
+
+int squat_trie_build_init(struct squat_trie *trie,
+			  struct squat_trie_build_context **ctx_r)
+{
+	struct squat_trie_build_context *ctx;
+	struct squat_uidlist_build_context *uidlist_build_ctx;
+
+	if (trie->fd == -1) {
+		trie->fd = squat_trie_create_fd(trie, trie->path, 0);
+		if (trie->fd == -1)
+			return -1;
+
+		if (trie->file_cache != NULL)
+			file_cache_set_fd(trie->file_cache, trie->fd);
+		i_assert(trie->locked_file_size == 0);
+	}
+
+	/* uidlist locks building */
+	if (squat_uidlist_build_init(trie->uidlist, &uidlist_build_ctx) < 0)
+		return -1;
+
+	if (squat_trie_map(trie, TRUE) < 0) {
+		squat_uidlist_build_deinit(&uidlist_build_ctx);
+		return -1;
+	}
+
+	ctx = i_new(struct squat_trie_build_context, 1);
+	ctx->trie = trie;
+	ctx->uidlist_build_ctx = uidlist_build_ctx;
+	ctx->first_uid = trie->root.next_uid;
+
+	*ctx_r = ctx;
+	return 0;
+}
+
+static int squat_trie_write_lock(struct squat_trie_build_context *ctx)
+{
+	if (ctx->file_lock != NULL || ctx->dotlock != NULL)
+		return 0;
+
+	if (squat_trie_lock(ctx->trie, F_WRLCK,
+			    &ctx->file_lock, &ctx->dotlock) <= 0)
+		return -1;
+	return 0;
+}
+
+static int squat_trie_write(struct squat_trie_build_context *ctx)
+{
+	struct squat_trie *trie = ctx->trie;
+	struct file_lock *file_lock = NULL;
+	struct ostream *output;
+	const char *path, *error;
+	int fd = -1, ret = 0;
+
+	if ((trie->hdr.used_file_size > sizeof(trie->hdr) &&
+	    trie->unmapped_child_count < trie->hdr.node_count/4) || 1) {
+		/* we might as well recreate the file */
+		ctx->compress_nodes = TRUE;
+
+		path = t_strconcat(trie->path, ".tmp", NULL);
+		fd = squat_trie_create_fd(trie, path, O_TRUNC);
+		if (fd == -1)
+			return -1;
+
+		if (trie->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
+			struct file_lock_settings lock_set = {
+				.lock_method = trie->lock_method,
+			};
+			ret = file_wait_lock(fd, path, F_WRLCK, &lock_set,
+					     SQUAT_TRIE_LOCK_TIMEOUT,
+					     &file_lock, &error);
+			if (ret <= 0) {
+				i_error("file_wait_lock(%s) failed: %s",
+					path, error);
+				i_close_fd(&fd);
+				return -1;
+			}
+		}
+
+		output = o_stream_create_fd(fd, 0);
+		o_stream_cork(output);
+		o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr));
+	} else {
+		/* we need to lock only while header is being written */
+		path = trie->path;
+		ctx->compress_nodes =
+			trie->hdr.used_file_size == sizeof(trie->hdr);
+
+		if (trie->hdr.used_file_size == 0) {
+			/* lock before opening the file, in case we reopen it */
+			if (squat_trie_write_lock(ctx) < 0)
+				return -1;
+		}
+		output = o_stream_create_fd(trie->fd, 0);
+		o_stream_cork(output);
+
+		if (trie->hdr.used_file_size != 0)
+			(void)o_stream_seek(output, trie->hdr.used_file_size);
+		else
+			o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr));
+	}
+
+	ctx->output = output;
+	ret = squat_write_nodes(ctx);
+	ctx->output = NULL;
+
+	/* write 1 byte guard at the end of file, so that we can verify broken
+	   squat_unpack_num() input by checking if data==end */
+	o_stream_nsend(output, "", 1);
+
+	if (trie->corrupted)
+		ret = -1;
+	if (ret == 0)
+		ret = squat_trie_write_lock(ctx);
+	if (ret == 0) {
+		trie->hdr.used_file_size = output->offset;
+		(void)o_stream_seek(output, 0);
+		o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr));
+	}
+	if (o_stream_finish(output) < 0) {
+		i_error("write(%s) failed: %s", path,
+			o_stream_get_error(output));
+		ret = -1;
+	}
+	o_stream_destroy(&output);
+
+	if (fd == -1) {
+		/* appended to the existing file */
+		i_assert(file_lock == NULL);
+		return ret;
+	}
+
+	/* recreating the trie file */
+	if (ret < 0) {
+		if (close(fd) < 0)
+			i_error("close(%s) failed: %m", path);
+		fd = -1;
+	} else if (rename(path, trie->path) < 0) {
+		i_error("rename(%s, %s) failed: %m", path, trie->path);
+		ret = -1;
+	}
+
+	if (ret < 0) {
+		i_unlink_if_exists(path);
+		file_lock_free(&file_lock);
+	} else {
+		squat_trie_close_fd(trie);
+		trie->fd = fd;
+		trie->locked_file_size = trie->hdr.used_file_size;
+		if (trie->file_cache != NULL)
+			file_cache_set_fd(trie->file_cache, trie->fd);
+
+		file_lock_free(&ctx->file_lock);
+		ctx->file_lock = file_lock;
+	}
+	return ret;
+}
+
+int squat_trie_build_deinit(struct squat_trie_build_context **_ctx,
+			    const ARRAY_TYPE(seq_range) *expunged_uids)
+{
+	struct squat_trie_build_context *ctx = *_ctx;
+	bool compress, unlock = TRUE;
+	int ret;
+
+	*_ctx = NULL;
+
+	compress = (ctx->trie->root.next_uid - ctx->first_uid) > 10;
+
+	/* keep trie locked while header is being written and when files are
+	   being renamed, so that while trie is read locked, uidlist can't
+	   change under. */
+	squat_uidlist_build_flush(ctx->uidlist_build_ctx);
+	ret = squat_trie_renumber_uidlists(ctx, expunged_uids, compress);
+	if (ret == 0) {
+		ret = squat_trie_write(ctx);
+		if (ret < 0)
+			unlock = FALSE;
+	}
+
+	if (ret == 0)
+		ret = squat_uidlist_build_finish(ctx->uidlist_build_ctx);
+	if (ctx->file_lock != NULL) {
+		if (unlock)
+			file_unlock(&ctx->file_lock);
+		else
+			file_lock_free(&ctx->file_lock);
+	}
+	if (ctx->dotlock != NULL)
+		file_dotlock_delete(&ctx->dotlock);
+	squat_uidlist_build_deinit(&ctx->uidlist_build_ctx);
+
+	i_free(ctx);
+	return ret;
+}
+
+int squat_trie_get_last_uid(struct squat_trie *trie, uint32_t *last_uid_r)
+{
+	if (trie->fd == -1) {
+		if (squat_trie_open(trie) < 0)
+			return -1;
+	}
+
+	*last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
+	return 0;
+}
+
+static int
+squat_trie_lookup_data(struct squat_trie *trie, const unsigned char *data,
+		       unsigned int size, ARRAY_TYPE(seq_range) *uids)
+{
+	struct squat_node *node = &trie->root;
+	unsigned char *chars;
+	unsigned int idx;
+	int level = 0;
+
+	array_clear(uids);
+
+	for (;;) {
+		if (node->children_not_mapped) {
+			if (node_read_children(trie, node, level) < 0)
+				return -1;
+		}
+		if (node->leaf_string_length != 0) {
+			unsigned int len = node->leaf_string_length;
+			const unsigned char *str;
+
+			if (len > sizeof(node->children.static_leaf_string))
+				str = node->children.leaf_string;
+			else
+				str = node->children.static_leaf_string;
+
+			if (size > len || memcmp(data, str, size) != 0)
+				return 0;
+
+			/* match */
+			break;
+		}
+
+		if (size == 0)
+			break;
+		level++;
+
+		if (node->have_sequential) {
+			if (*data < SEQUENTIAL_COUNT) {
+				idx = *data;
+				goto found;
+			}
+			idx = SEQUENTIAL_COUNT;
+		} else {
+			idx = 0;
+		}
+		chars = NODE_CHILDREN_CHARS(node);
+		for (; idx < node->child_count; idx++) {
+			if (chars[idx] == *data)
+				goto found;
+		}
+		return 0;
+	found:
+		/* follow to children */
+		if (level == 1) {
+			/* root level, add all UIDs */
+			if (squat_uidlist_get_seqrange(trie->uidlist,
+						       node->uid_list_idx,
+						       uids) < 0)
+				return -1;
+		} else {
+			if (squat_uidlist_filter(trie->uidlist,
+						 node->uid_list_idx, uids) < 0)
+				return -1;
+		}
+		data++;
+		size--;
+		node = NODE_CHILDREN_NODES(node) + idx;
+	}
+
+	if (squat_uidlist_filter(trie->uidlist, node->uid_list_idx, uids) < 0)
+		return -1;
+	return 1;
+}
+
+static void
+squat_trie_filter_type(enum squat_index_type type,
+		       const ARRAY_TYPE(seq_range) *src,
+		       ARRAY_TYPE(seq_range) *dest)
+{
+	const struct seq_range *src_range;
+	struct seq_range new_range;
+	unsigned int i, count, mask;
+	uint32_t next_seq, uid;
+
+	array_clear(dest);
+	src_range = array_get(src, &count);
+	if (count == 0)
+		return;
+
+	if ((type & SQUAT_INDEX_TYPE_HEADER) != 0 &&
+	    (type & SQUAT_INDEX_TYPE_BODY) != 0) {
+		/* everything is fine, just fix the UIDs */
+		new_range.seq1 = src_range[0].seq1 / 2;
+		new_range.seq2 = src_range[0].seq2 / 2;
+		for (i = 1; i < count; i++) {
+			next_seq = src_range[i].seq1 / 2;
+			if (next_seq == new_range.seq2 + 1) {
+				/* we can continue the previous range */
+			} else {
+				array_push_back(dest, &new_range);
+				new_range.seq1 = src_range[i].seq1 / 2;
+			}
+			new_range.seq2 = src_range[i].seq2 / 2;
+		}
+		array_push_back(dest, &new_range);
+		return;
+	}
+
+	/* we'll have to drop either header or body UIDs */
+	mask = (type & SQUAT_INDEX_TYPE_HEADER) != 0 ? 1 : 0;
+	for (i = 0; i < count; i++) {
+		for (uid = src_range[i].seq1; uid <= src_range[i].seq2; uid++) {
+			if ((uid & 1) == mask)
+				seq_range_array_add(dest, uid/2);
+		}
+	}
+}
+
+struct squat_trie_lookup_context {
+	struct squat_trie *trie;
+	enum squat_index_type type;
+
+	ARRAY_TYPE(seq_range) *definite_uids, *maybe_uids;
+	ARRAY_TYPE(seq_range) tmp_uids, tmp_uids2;
+	bool first;
+};
+
+static int
+squat_trie_lookup_partial(struct squat_trie_lookup_context *ctx,
+			  const unsigned char *data, uint8_t *char_lengths,
+			  unsigned int size)
+{
+	const unsigned int partial_len = ctx->trie->hdr.partial_len;
+	unsigned int char_idx, max_chars, i, j, bytelen;
+	int ret;
+
+	for (i = 0, max_chars = 0; i < size; max_chars++)
+		i += char_lengths[i];
+	i_assert(max_chars > 0);
+
+	i = 0; char_idx = 0;
+	do {
+		bytelen = 0;
+		for (j = 0; j < partial_len && i+bytelen < size; j++)
+			bytelen += char_lengths[i + bytelen];
+
+		ret = squat_trie_lookup_data(ctx->trie, data + i, bytelen,
+					     &ctx->tmp_uids);
+		if (ret <= 0) {
+			array_clear(ctx->maybe_uids);
+			return ret;
+		}
+
+		if (ctx->first) {
+			squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
+					       ctx->maybe_uids);
+			ctx->first = FALSE;
+		} else {
+			squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
+					       &ctx->tmp_uids2);
+			seq_range_array_intersect(ctx->maybe_uids,
+						  &ctx->tmp_uids2);
+		}
+		i += char_lengths[i];
+		char_idx++;
+	} while (max_chars - char_idx >= partial_len);
+	return 1;
+}
+
+static void squat_trie_add_unknown(struct squat_trie *trie,
+				   ARRAY_TYPE(seq_range) *maybe_uids)
+{
+	struct seq_range *range, new_range;
+	unsigned int count;
+	uint32_t last_uid;
+
+	last_uid = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
+
+	range = array_get_modifiable(maybe_uids, &count);
+	if (count > 0 && range[count-1].seq2 == last_uid) {
+		/* increase the range */
+		range[count-1].seq2 = (uint32_t)-1;
+	} else {
+		new_range.seq1 = last_uid + 1;
+		new_range.seq2 = (uint32_t)-1;
+		array_push_back(maybe_uids, &new_range);
+	}
+}
+
+static int
+squat_trie_lookup_real(struct squat_trie *trie, const char *str,
+		       enum squat_index_type type,
+		       ARRAY_TYPE(seq_range) *definite_uids,
+		       ARRAY_TYPE(seq_range) *maybe_uids)
+{
+	struct squat_trie_lookup_context ctx;
+	unsigned char *data;
+	uint8_t *char_lengths;
+	unsigned int i, start, bytes, str_bytelen, str_charlen;
+	bool searched = FALSE;
+	int ret = 0;
+
+	array_clear(definite_uids);
+	array_clear(maybe_uids);
+
+	i_zero(&ctx);
+	ctx.trie = trie;
+	ctx.type = type;
+	ctx.definite_uids = definite_uids;
+	ctx.maybe_uids = maybe_uids;
+	i_array_init(&ctx.tmp_uids, 128);
+	i_array_init(&ctx.tmp_uids2, 128);
+	ctx.first = TRUE;
+
+	str_bytelen = strlen(str);
+	char_lengths = str_bytelen == 0 ? NULL : t_malloc0(str_bytelen);
+	for (i = 0, str_charlen = 0; i < str_bytelen; str_charlen++) {
+		bytes = uni_utf8_char_bytes(str[i]);
+		char_lengths[i] = bytes;
+		i += bytes;
+	}
+	data = squat_data_normalize(trie, (const unsigned char *)str,
+				    str_bytelen);
+
+	for (i = start = 0; i < str_bytelen && ret >= 0; i += char_lengths[i]) {
+		if (data[i] != '\0')
+			continue;
+
+		/* string has nonindexed characters.
+		   search it in parts. */
+		if (i != start) {
+			ret = squat_trie_lookup_partial(&ctx, data + start,
+							char_lengths + start,
+							i - start);
+			searched = TRUE;
+		}
+		start = i + char_lengths[i];
+	}
+
+	if (start == 0) {
+		if (str_charlen <= trie->hdr.partial_len ||
+		    trie->hdr.full_len > trie->hdr.partial_len) {
+			ret = squat_trie_lookup_data(trie, data, str_bytelen,
+						     &ctx.tmp_uids);
+			if (ret > 0) {
+				squat_trie_filter_type(type, &ctx.tmp_uids,
+						       definite_uids);
+			}
+		} else {
+			array_clear(definite_uids);
+		}
+
+		if (str_charlen <= trie->hdr.partial_len ||
+		    trie->hdr.partial_len == 0) {
+			/* we have the result */
+			array_clear(maybe_uids);
+		} else {
+			ret = squat_trie_lookup_partial(&ctx, data + start,
+							char_lengths + start,
+							i - start);
+		}
+	} else if (str_bytelen > 0) {
+		/* string has nonindexed characters. finish the search. */
+		array_clear(definite_uids);
+		if (i != start && ret >= 0) {
+			ret = squat_trie_lookup_partial(&ctx, data + start,
+							char_lengths + start,
+							i - start);
+		} else if (!searched) {
+			/* string has only nonindexed chars,
+			   list all root UIDs as maybes */
+			ret = squat_uidlist_get_seqrange(trie->uidlist,
+							trie->root.uid_list_idx,
+							&ctx.tmp_uids);
+			squat_trie_filter_type(type, &ctx.tmp_uids,
+					       maybe_uids);
+		}
+	} else {
+		/* zero string length - list all root UIDs as definite
+		   answers */
+#if 0 /* FIXME: this code is never actually reached now. */
+		ret = squat_uidlist_get_seqrange(trie->uidlist,
+						 trie->root.uid_list_idx,
+						 &ctx.tmp_uids);
+		squat_trie_filter_type(type, &ctx.tmp_uids,
+				       definite_uids);
+#else
+		i_unreached();
+#endif
+	}
+	seq_range_array_remove_seq_range(maybe_uids, definite_uids);
+	squat_trie_add_unknown(trie, maybe_uids);
+	array_free(&ctx.tmp_uids);
+	array_free(&ctx.tmp_uids2);
+	return ret < 0 ? -1 : 0;
+}
+
+int squat_trie_lookup(struct squat_trie *trie, const char *str,
+		      enum squat_index_type type,
+		      ARRAY_TYPE(seq_range) *definite_uids,
+		      ARRAY_TYPE(seq_range) *maybe_uids)
+{
+	int ret;
+
+	T_BEGIN {
+		ret = squat_trie_lookup_real(trie, str, type,
+					     definite_uids, maybe_uids);
+	} T_END;
+	return ret;
+}
+
+struct squat_uidlist *squat_trie_get_uidlist(struct squat_trie *trie)
+{
+	return trie->uidlist;
+}
+
+size_t squat_trie_mem_used(struct squat_trie *trie, unsigned int *count_r)
+{
+	*count_r = trie->hdr.node_count;
+	return trie->node_alloc_size;
+}
diff --git a/src/plugins/fts-squat/squat-trie.h b/src/plugins/fts-squat/squat-trie.h
new file mode 100644
index 0000000..91530b8
--- /dev/null
+++ b/src/plugins/fts-squat/squat-trie.h
@@ -0,0 +1,54 @@
+#ifndef SQUAT_TRIE_H
+#define SQUAT_TRIE_H
+
+#include "file-lock.h"
+#include "seq-range-array.h"
+
+enum squat_index_flags {
+	SQUAT_INDEX_FLAG_MMAP_DISABLE		= 0x01,
+	SQUAT_INDEX_FLAG_NFS_FLUSH		= 0x02,
+	SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL	= 0x04
+};
+
+enum squat_index_type {
+	SQUAT_INDEX_TYPE_HEADER	= 0x01,
+	SQUAT_INDEX_TYPE_BODY	= 0x02
+};
+
+struct squat_trie_build_context;
+
+struct squat_trie *
+squat_trie_init(const char *path, uint32_t uidvalidity,
+		enum file_lock_method lock_method,
+		enum squat_index_flags flags, mode_t mode, gid_t gid);
+void squat_trie_deinit(struct squat_trie **trie);
+
+void squat_trie_set_partial_len(struct squat_trie *trie, unsigned int len);
+void squat_trie_set_full_len(struct squat_trie *trie, unsigned int len);
+
+int squat_trie_open(struct squat_trie *trie);
+int squat_trie_refresh(struct squat_trie *trie);
+
+int squat_trie_build_init(struct squat_trie *trie,
+			  struct squat_trie_build_context **ctx_r);
+/* bodies must be added before headers */
+int squat_trie_build_more(struct squat_trie_build_context *ctx,
+			  uint32_t uid, enum squat_index_type type,
+			  const unsigned char *data, unsigned int size);
+/* if expunged_uids is non-NULL, they may be removed from the index if they
+   still exist. */
+int squat_trie_build_deinit(struct squat_trie_build_context **ctx,
+			    const ARRAY_TYPE(seq_range) *expunged_uids)
+	ATTR_NULL(2);
+
+int squat_trie_get_last_uid(struct squat_trie *trie, uint32_t *last_uid_r);
+/* type specifies if we're looking at header, body or both */
+int squat_trie_lookup(struct squat_trie *trie, const char *str,
+		      enum squat_index_type type,
+		      ARRAY_TYPE(seq_range) *definite_uids,
+		      ARRAY_TYPE(seq_range) *maybe_uids);
+
+struct squat_uidlist *squat_trie_get_uidlist(struct squat_trie *trie);
+size_t squat_trie_mem_used(struct squat_trie *trie, unsigned int *count_r);
+
+#endif
diff --git a/src/plugins/fts-squat/squat-uidlist.c b/src/plugins/fts-squat/squat-uidlist.c
new file mode 100644
index 0000000..facb8d0
--- /dev/null
+++ b/src/plugins/fts-squat/squat-uidlist.c
@@ -0,0 +1,1624 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "sort.h"
+#include "bsearch-insert-pos.h"
+#include "file-cache.h"
+#include "file-lock.h"
+#include "read-full.h"
+#include "write-full.h"
+#include "ostream.h"
+#include "mmap-util.h"
+#include "squat-trie-private.h"
+#include "squat-uidlist.h"
+
+#include <stdio.h>
+#include <sys/stat.h>
+
+#define UIDLIST_LIST_SIZE 31
+#define UIDLIST_BLOCK_LIST_COUNT 100
+#define UID_LIST_MASK_RANGE 0x80000000U
+
+/* set = points to uidlist index number, unset = points to uidlist offset */
+#define UID_LIST_POINTER_MASK_LIST_IDX 0x80000000U
+
+#define UIDLIST_PACKED_FLAG_BITMASK 1
+#define UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER 2
+
+struct uidlist_list {
+	unsigned int uid_count:31;
+	bool uid_begins_with_pointer:1;
+	uint32_t uid_list[UIDLIST_LIST_SIZE];
+};
+
+struct squat_uidlist {
+	struct squat_trie *trie;
+
+	char *path;
+	int fd;
+	struct file_cache *file_cache;
+
+	struct file_lock *file_lock;
+	struct dotlock *dotlock;
+	uoff_t locked_file_size;
+
+	void *mmap_base;
+	size_t mmap_size;
+	struct squat_uidlist_file_header hdr;
+
+	const void *data;
+	size_t data_size;
+
+	unsigned int cur_block_count;
+	const uint32_t *cur_block_offsets;
+	const uint32_t *cur_block_end_indexes;
+
+	size_t max_size;
+	bool corrupted:1;
+	bool building:1;
+};
+
+struct squat_uidlist_build_context {
+	struct squat_uidlist *uidlist;
+	struct ostream *output;
+
+	ARRAY_TYPE(uint32_t) block_offsets;
+	ARRAY_TYPE(uint32_t) block_end_indexes;
+
+	ARRAY(struct uidlist_list) lists;
+	uint32_t list_start_idx;
+
+	struct squat_uidlist_file_header build_hdr;
+	bool need_reopen:1;
+};
+
+struct squat_uidlist_rebuild_context {
+	struct squat_uidlist *uidlist;
+	struct squat_uidlist_build_context *build_ctx;
+
+	int fd;
+	struct ostream *output;
+
+	ARRAY_TYPE(uint32_t) new_block_offsets, new_block_end_indexes;
+	uoff_t cur_block_start_offset;
+
+	uint32_t list_sizes[UIDLIST_BLOCK_LIST_COUNT];
+	uint32_t next_uid_list_idx;
+	unsigned int list_idx;
+	unsigned int new_count;
+};
+
+static void squat_uidlist_close(struct squat_uidlist *uidlist);
+
+void squat_uidlist_delete(struct squat_uidlist *uidlist)
+{
+	i_unlink_if_exists(uidlist->path);
+}
+
+static void squat_uidlist_set_corrupted(struct squat_uidlist *uidlist,
+					const char *reason)
+{
+	if (uidlist->corrupted)
+		return;
+	uidlist->corrupted = TRUE;
+
+	i_error("Corrupted squat uidlist file %s: %s", uidlist->path, reason);
+	squat_trie_delete(uidlist->trie);
+}
+
+static int
+uidlist_write_array(struct ostream *output, const uint32_t *uid_list,
+		    unsigned int uid_count, uint32_t packed_flags,
+		    uint32_t offset, bool write_size, uint32_t *size_r)
+{
+	uint8_t *uidbuf, *bufp, sizebuf[SQUAT_PACK_MAX_SIZE], *sizebufp;
+	uint8_t listbuf[SQUAT_PACK_MAX_SIZE], *listbufp = listbuf;
+	uint32_t uid, uid2, prev, base_uid, size_value;
+	unsigned int i, bitmask_len, uid_list_len;
+	unsigned int idx, max_idx, mask;
+	bool datastack;
+	int num;
+
+	if ((packed_flags & UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER) != 0)
+		squat_pack_num(&listbufp, offset);
+
+	/* @UNSAFE */
+	base_uid = uid_list[0] & ~UID_LIST_MASK_RANGE;
+	datastack = uid_count < 1024*8/SQUAT_PACK_MAX_SIZE;
+	if (datastack)
+		uidbuf = t_malloc_no0(SQUAT_PACK_MAX_SIZE * uid_count);
+	else
+		uidbuf = i_malloc(SQUAT_PACK_MAX_SIZE * uid_count);
+	bufp = uidbuf;
+	squat_pack_num(&bufp, base_uid);
+
+	bitmask_len = (uid_list[uid_count-1] - base_uid + 7) / 8 +
+		(bufp - uidbuf);
+	if (bitmask_len < uid_count) {
+	bitmask_build:
+		i_assert(bitmask_len < SQUAT_PACK_MAX_SIZE*uid_count);
+
+		memset(bufp, 0, bitmask_len - (bufp - uidbuf));
+		if ((uid_list[0] & UID_LIST_MASK_RANGE) == 0) {
+			i = 1;
+			uid = i == uid_count ? 0 : uid_list[i];
+		} else {
+			i = 0;
+			uid = uid_list[0] + 1;
+		}
+		base_uid++;
+
+		for (; i < uid_count; i++) {
+			i_assert((uid & ~UID_LIST_MASK_RANGE) >= base_uid);
+			if ((uid & UID_LIST_MASK_RANGE) == 0) {
+				uid -= base_uid;
+				uid2 = uid;
+			} else {
+				uid &= ~UID_LIST_MASK_RANGE;
+				uid -= base_uid;
+				uid2 = uid_list[i+1] - base_uid;
+				i++;
+			}
+
+			if (uid2 - uid < 3*8) {
+				for (; uid <= uid2; uid++)
+					bufp[uid / 8] |= 1 << (uid % 8);
+			} else {
+				/* first byte */
+				idx = uid / 8;
+				num = uid % 8;
+				if (num != 0) {
+					uid += 8 - num;
+					for (mask = 0; num < 8; num++)
+						mask |= 1 << num;
+					bufp[idx++] |= mask;
+				}
+
+				/* middle bytes */
+				num = uid2 % 8;
+				max_idx = idx + (uid2 - num - uid)/8;
+				for (; idx < max_idx; idx++, uid += 8)
+					bufp[idx] = 0xff;
+
+				/* last byte */
+				for (mask = 0; num >= 0; num--)
+					mask |= 1 << num;
+				bufp[idx] |= mask;
+			}
+			uid = i+1 == uid_count ? 0 : uid_list[i+1];
+		}
+		uid_list_len = bitmask_len;
+		packed_flags |= UIDLIST_PACKED_FLAG_BITMASK;
+	} else {
+		bufp = uidbuf;
+		prev = 0;
+		for (i = 0; i < uid_count; i++) {
+			uid = uid_list[i];
+			if (unlikely((uid & ~UID_LIST_MASK_RANGE) < prev)) {
+				if (!datastack)
+					i_free(uidbuf);
+				return -1;
+			}
+			if ((uid & UID_LIST_MASK_RANGE) == 0) {
+				squat_pack_num(&bufp, (uid - prev) << 1);
+				prev = uid + 1;
+			} else {
+				uid &= ~UID_LIST_MASK_RANGE;
+				squat_pack_num(&bufp, 1 | (uid - prev) << 1);
+				squat_pack_num(&bufp, uid_list[i+1] - uid - 1);
+				prev = uid_list[i+1] + 1;
+				i++;
+			}
+		}
+		uid_list_len = bufp - uidbuf;
+		if (uid_list_len > bitmask_len) {
+			bufp = uidbuf;
+			squat_pack_num(&bufp, base_uid);
+			goto bitmask_build;
+		}
+	}
+
+	size_value = ((uid_list_len +
+		       (listbufp - listbuf)) << 2) | packed_flags;
+	if (write_size) {
+		sizebufp = sizebuf;
+		squat_pack_num(&sizebufp, size_value);
+		o_stream_nsend(output, sizebuf, sizebufp - sizebuf);
+	}
+	o_stream_nsend(output, listbuf, listbufp - listbuf);
+	o_stream_nsend(output, uidbuf, uid_list_len);
+	if (!datastack)
+		i_free(uidbuf);
+
+	*size_r = size_value;
+	return 0;
+}
+
+static int
+uidlist_write(struct ostream *output, const struct uidlist_list *list,
+	      bool write_size, uint32_t *size_r)
+{
+	const uint32_t *uid_list = list->uid_list;
+	uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp;
+	uint32_t uid_count = list->uid_count;
+	uint32_t packed_flags = 0;
+	uint32_t offset = 0;
+	int ret;
+
+	if (list->uid_begins_with_pointer) {
+		/* continued UID list */
+		packed_flags |= UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER;
+		if ((uid_list[0] & UID_LIST_POINTER_MASK_LIST_IDX) != 0) {
+			offset = ((uid_list[0] & ~UID_LIST_POINTER_MASK_LIST_IDX) << 1) | 1;
+			if (list->uid_count == 1) {
+				bufp = buf;
+				squat_pack_num(&bufp, offset);
+				o_stream_nsend(output, buf, bufp - buf);
+				*size_r = (bufp - buf) << 2 | packed_flags;
+				return 0;
+			}
+		} else if (unlikely(output->offset <= uid_list[0])) {
+			i_assert(output->closed);
+			return -1;
+		} else {
+			i_assert(list->uid_count > 1);
+			offset = (output->offset - uid_list[0]) << 1;
+		}
+		uid_list++;
+		uid_count--;
+	}
+
+	T_BEGIN {
+		ret = uidlist_write_array(output, uid_list, uid_count,
+					  packed_flags, offset,
+					  write_size, size_r);
+	} T_END;
+	return ret;
+}
+
+static void squat_uidlist_map_blocks_set_pointers(struct squat_uidlist *uidlist)
+{
+	const void *base;
+	size_t end_index_size, end_size;
+
+	base = CONST_PTR_OFFSET(uidlist->data, uidlist->hdr.block_list_offset +
+				sizeof(uint32_t));
+
+	end_index_size = uidlist->cur_block_count * sizeof(uint32_t);
+	end_size = end_index_size + uidlist->cur_block_count * sizeof(uint32_t);
+	if (end_size <= uidlist->data_size) {
+		uidlist->cur_block_end_indexes = base;
+		uidlist->cur_block_offsets =
+			CONST_PTR_OFFSET(base, end_index_size);
+	} else {
+		uidlist->cur_block_end_indexes = NULL;
+		uidlist->cur_block_offsets = NULL;
+	}
+}
+
+static int uidlist_file_cache_read(struct squat_uidlist *uidlist,
+				   size_t offset, size_t size)
+{
+	if (uidlist->file_cache == NULL)
+		return 0;
+
+	if (file_cache_read(uidlist->file_cache, offset, size) < 0) {
+		i_error("read(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	uidlist->data = file_cache_get_map(uidlist->file_cache,
+					   &uidlist->data_size);
+	squat_uidlist_map_blocks_set_pointers(uidlist);
+	return 0;
+}
+
+static int squat_uidlist_map_blocks(struct squat_uidlist *uidlist)
+{
+	const struct squat_uidlist_file_header *hdr = &uidlist->hdr;
+	const void *base;
+	uint32_t block_count, blocks_offset, blocks_size, i, verify_count;
+
+	if (hdr->block_list_offset == 0) {
+		/* empty file */
+		uidlist->cur_block_count = 0;
+		return 1;
+	}
+
+	/* get number of blocks */
+	if (uidlist_file_cache_read(uidlist, hdr->block_list_offset,
+				    sizeof(block_count)) < 0)
+		return -1;
+	blocks_offset = hdr->block_list_offset + sizeof(block_count);
+	if (blocks_offset > uidlist->data_size) {
+		squat_uidlist_set_corrupted(uidlist, "block list outside file");
+		return 0;
+	}
+
+	i_assert(uidlist->data != NULL);
+	base = CONST_PTR_OFFSET(uidlist->data, hdr->block_list_offset);
+	memcpy(&block_count, base, sizeof(block_count));
+
+	/* map the blocks */
+	blocks_size = block_count * sizeof(uint32_t)*2;
+	if (uidlist_file_cache_read(uidlist, blocks_offset, blocks_size) < 0)
+		return -1;
+	if (blocks_offset + blocks_size > uidlist->data_size) {
+		squat_uidlist_set_corrupted(uidlist, "block list outside file");
+		return 0;
+	}
+
+	uidlist->cur_block_count = block_count;
+	squat_uidlist_map_blocks_set_pointers(uidlist);
+
+	i_assert(uidlist->cur_block_end_indexes != NULL);
+
+	/* verify just a couple of the end indexes to make sure they
+	   look correct */
+	verify_count = I_MIN(block_count, 8);
+	for (i = 1; i < verify_count; i++) {
+		if (unlikely(uidlist->cur_block_end_indexes[i-1] >=
+			     uidlist->cur_block_end_indexes[i])) {
+			squat_uidlist_set_corrupted(uidlist,
+				"block list corrupted");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+static int squat_uidlist_map_header(struct squat_uidlist *uidlist)
+{
+	if (uidlist->hdr.indexid == 0) {
+		/* still being built */
+		return 1;
+	}
+	if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) {
+		/* see if trie was recreated */
+		(void)squat_trie_open(uidlist->trie);
+	}
+	if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) {
+		squat_uidlist_set_corrupted(uidlist, "wrong indexid");
+		return 0;
+	}
+	if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr) ||
+	    (uidlist->hdr.used_file_size > uidlist->mmap_size &&
+	     uidlist->mmap_base != NULL)) {
+		squat_uidlist_set_corrupted(uidlist, "broken used_file_size");
+		return 0;
+	}
+	return squat_uidlist_map_blocks(uidlist);
+}
+
+static void squat_uidlist_unmap(struct squat_uidlist *uidlist)
+{
+	if (uidlist->mmap_size != 0) {
+		if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
+			i_error("munmap(%s) failed: %m", uidlist->path);
+		uidlist->mmap_base = NULL;
+		uidlist->mmap_size = 0;
+	}
+	uidlist->cur_block_count = 0;
+	uidlist->cur_block_end_indexes = NULL;
+	uidlist->cur_block_offsets = NULL;
+}
+
+static int squat_uidlist_mmap(struct squat_uidlist *uidlist)
+{
+	struct stat st;
+
+	if (fstat(uidlist->fd, &st) < 0) {
+		i_error("fstat(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	if (st.st_size < (off_t)sizeof(uidlist->hdr)) {
+		squat_uidlist_set_corrupted(uidlist, "File too small");
+		return -1;
+	}
+
+	squat_uidlist_unmap(uidlist);
+	uidlist->mmap_size = st.st_size;
+	uidlist->mmap_base = mmap(NULL, uidlist->mmap_size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_SHARED, uidlist->fd, 0);
+	if (uidlist->mmap_base == MAP_FAILED) {
+		uidlist->data = uidlist->mmap_base = NULL;
+		uidlist->data_size = uidlist->mmap_size = 0;
+		i_error("mmap(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	uidlist->data = uidlist->mmap_base;
+	uidlist->data_size = uidlist->mmap_size;
+	return 0;
+}
+
+static int squat_uidlist_map(struct squat_uidlist *uidlist)
+{
+	const struct squat_uidlist_file_header *mmap_hdr = uidlist->mmap_base;
+	int ret;
+
+	if (mmap_hdr != NULL && !uidlist->building &&
+	    uidlist->hdr.block_list_offset == mmap_hdr->block_list_offset) {
+		/* file hasn't changed */
+		return 1;
+	}
+
+	if ((uidlist->trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) == 0) {
+		if (mmap_hdr == NULL || uidlist->building ||
+		    uidlist->mmap_size < mmap_hdr->used_file_size) {
+			if (squat_uidlist_mmap(uidlist) < 0)
+				return -1;
+		}
+
+		if (!uidlist->building) {
+			memcpy(&uidlist->hdr, uidlist->mmap_base,
+			       sizeof(uidlist->hdr));
+		}
+	} else if (uidlist->building) {
+		/* we want to update blocks mapping, but using the header
+		   in memory */
+	} else {
+		ret = pread_full(uidlist->fd, &uidlist->hdr,
+				 sizeof(uidlist->hdr), 0);
+		if (ret <= 0) {
+			if (ret < 0) {
+				i_error("pread(%s) failed: %m", uidlist->path);
+				return -1;
+			}
+			i_error("Corrupted %s: File too small", uidlist->path);
+			return 0;
+		}
+		uidlist->data = NULL;
+		uidlist->data_size = 0;
+	}
+	if (uidlist->file_cache == NULL &&
+	    (uidlist->trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) != 0)
+		uidlist->file_cache = file_cache_new_path(uidlist->fd, uidlist->path);
+	return squat_uidlist_map_header(uidlist);
+}
+
+static int squat_uidlist_read_to_memory(struct squat_uidlist *uidlist)
+{
+	size_t i, page_size = mmap_get_page_size();
+
+	if (uidlist->file_cache != NULL) {
+		return uidlist_file_cache_read(uidlist, 0,
+					       uidlist->hdr.used_file_size);
+	}
+	/* Tell the kernel we're going to use the uidlist data, so it loads
+	   it into memory and keeps it there. */
+	(void)madvise(uidlist->mmap_base, uidlist->mmap_size, MADV_WILLNEED);
+	/* It also speeds up a bit for us to sequentially load everything
+	   into memory, although at least Linux catches up quite fast even
+	   without this code. Compiler can quite easily optimize away this
+	   entire for loop, but volatile seems to help with gcc 4.2. */
+	for (i = 0; i < uidlist->mmap_size; i += page_size)
+		((const volatile char *)uidlist->data)[i];
+	return 0;
+}
+
+static void squat_uidlist_free_from_memory(struct squat_uidlist *uidlist)
+{
+	size_t page_size = mmap_get_page_size();
+
+	if (uidlist->file_cache != NULL) {
+		file_cache_invalidate(uidlist->file_cache,
+				      page_size, UOFF_T_MAX);
+	} else {
+		(void)madvise(uidlist->mmap_base, uidlist->mmap_size,
+			      MADV_DONTNEED);
+	}
+}
+
+struct squat_uidlist *squat_uidlist_init(struct squat_trie *trie)
+{
+	struct squat_uidlist *uidlist;
+
+	uidlist = i_new(struct squat_uidlist, 1);
+	uidlist->trie = trie;
+	uidlist->path = i_strconcat(trie->path, ".uids", NULL);
+	uidlist->fd = -1;
+
+	return uidlist;
+}
+
+void squat_uidlist_deinit(struct squat_uidlist *uidlist)
+{
+	squat_uidlist_close(uidlist);
+
+	i_free(uidlist->path);
+	i_free(uidlist);
+}
+
+static int squat_uidlist_open(struct squat_uidlist *uidlist)
+{
+	squat_uidlist_close(uidlist);
+
+	uidlist->fd = open(uidlist->path, O_RDWR);
+	if (uidlist->fd == -1) {
+		if (errno == ENOENT) {
+			i_zero(&uidlist->hdr);
+			return 0;
+		}
+		i_error("open(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	return squat_uidlist_map(uidlist) <= 0 ? -1 : 0;
+}
+
+static void squat_uidlist_close(struct squat_uidlist *uidlist)
+{
+	i_assert(!uidlist->building);
+
+	squat_uidlist_unmap(uidlist);
+	if (uidlist->file_cache != NULL)
+		file_cache_free(&uidlist->file_cache);
+	file_lock_free(&uidlist->file_lock);
+	if (uidlist->dotlock != NULL)
+		file_dotlock_delete(&uidlist->dotlock);
+	i_close_fd_path(&uidlist->fd, uidlist->path);
+	uidlist->corrupted = FALSE;
+}
+
+int squat_uidlist_refresh(struct squat_uidlist *uidlist)
+{
+	/* we assume here that trie is locked, so that we don't need to worry
+	   about it when reading the header */
+	if (uidlist->fd == -1 ||
+	    uidlist->hdr.indexid != uidlist->trie->hdr.indexid) {
+		if (squat_uidlist_open(uidlist) < 0)
+			return -1;
+	} else {
+		if (squat_uidlist_map(uidlist) <= 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int squat_uidlist_is_file_stale(struct squat_uidlist *uidlist)
+{
+	struct stat st, st2;
+
+	i_assert(uidlist->fd != -1);
+
+	if (stat(uidlist->path, &st) < 0) {
+		if (errno == ENOENT)
+			return 1;
+
+		i_error("stat(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	if (fstat(uidlist->fd, &st2) < 0) {
+		i_error("fstat(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	uidlist->locked_file_size = st2.st_size;
+
+	return st.st_ino == st2.st_ino &&
+		CMP_DEV_T(st.st_dev, st2.st_dev) ? 0 : 1;
+}
+
+static int squat_uidlist_lock(struct squat_uidlist *uidlist)
+{
+	const char *error;
+	int ret;
+
+	for (;;) {
+		i_assert(uidlist->fd != -1);
+		i_assert(uidlist->file_lock == NULL);
+		i_assert(uidlist->dotlock == NULL);
+
+		if (uidlist->trie->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
+			struct file_lock_settings lock_set = {
+				.lock_method = uidlist->trie->lock_method,
+			};
+			ret = file_wait_lock(uidlist->fd, uidlist->path,
+					     F_WRLCK, &lock_set,
+					     SQUAT_TRIE_LOCK_TIMEOUT,
+					     &uidlist->file_lock, &error);
+			if (ret < 0) {
+				i_error("squat uidlist %s: %s",
+					uidlist->path, error);
+			}
+		} else {
+			ret = file_dotlock_create(&uidlist->trie->dotlock_set,
+						  uidlist->path, 0,
+						  &uidlist->dotlock);
+		}
+		if (ret == 0) {
+			i_error("squat uidlist %s: Locking timed out",
+				uidlist->path);
+			return 0;
+		}
+		if (ret < 0)
+			return -1;
+
+		ret = squat_uidlist_is_file_stale(uidlist);
+		if (ret == 0)
+			break;
+
+		if (uidlist->file_lock != NULL)
+			file_unlock(&uidlist->file_lock);
+		else
+			file_dotlock_delete(&uidlist->dotlock);
+		if (ret < 0)
+			return -1;
+
+		squat_uidlist_close(uidlist);
+		uidlist->fd = squat_trie_create_fd(uidlist->trie,
+						   uidlist->path, 0);
+		if (uidlist->fd == -1)
+			return -1;
+	}
+	return 1;
+}
+
+static int squat_uidlist_open_or_create(struct squat_uidlist *uidlist)
+{
+	int ret;
+
+	if (uidlist->fd == -1) {
+		uidlist->fd = squat_trie_create_fd(uidlist->trie,
+						   uidlist->path, 0);
+		if (uidlist->fd == -1)
+			return -1;
+	}
+	if (squat_uidlist_lock(uidlist) <= 0)
+		return -1;
+
+	if (uidlist->locked_file_size != 0) {
+		if ((ret = squat_uidlist_map(uidlist)) < 0)
+			return -1;
+		if (ret == 0) {
+			/* broken file, truncate */
+			if (ftruncate(uidlist->fd, 0) < 0) {
+				i_error("ftruncate(%s) failed: %m",
+					uidlist->path);
+				return -1;
+			}
+			uidlist->locked_file_size = 0;
+		}
+	}
+	if (uidlist->locked_file_size == 0) {
+		/* write using 0 until we're finished */
+		i_zero(&uidlist->hdr);
+		if (write_full(uidlist->fd, &uidlist->hdr,
+			       sizeof(uidlist->hdr)) < 0) {
+			i_error("write(%s) failed: %m", uidlist->path);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+int squat_uidlist_build_init(struct squat_uidlist *uidlist,
+			     struct squat_uidlist_build_context **ctx_r)
+{
+	struct squat_uidlist_build_context *ctx;
+	int ret;
+
+	i_assert(!uidlist->building);
+
+	ret = squat_uidlist_open_or_create(uidlist);
+	if (ret == 0 &&
+	    lseek(uidlist->fd, uidlist->hdr.used_file_size, SEEK_SET) < 0) {
+		i_error("lseek(%s) failed: %m", uidlist->path);
+		ret = -1;
+	}
+
+	if (ret < 0) {
+		if (uidlist->file_lock != NULL)
+			file_unlock(&uidlist->file_lock);
+		if (uidlist->dotlock != NULL)
+			file_dotlock_delete(&uidlist->dotlock);
+		return -1;
+	}
+
+	ctx = i_new(struct squat_uidlist_build_context, 1);
+	ctx->uidlist = uidlist;
+	ctx->output = o_stream_create_fd(uidlist->fd, 0);
+	if (ctx->output->offset == 0) {
+		struct squat_uidlist_file_header hdr;
+
+		i_zero(&hdr);
+		o_stream_nsend(ctx->output, &hdr, sizeof(hdr));
+	}
+	o_stream_cork(ctx->output);
+	i_array_init(&ctx->lists, 10240);
+	i_array_init(&ctx->block_offsets, 128);
+	i_array_init(&ctx->block_end_indexes, 128);
+	ctx->list_start_idx = uidlist->hdr.count;
+	ctx->build_hdr = uidlist->hdr;
+
+	uidlist->building = TRUE;
+	*ctx_r = ctx;
+	return 0;
+}
+
+static void
+uidlist_write_block_list_and_header(struct squat_uidlist_build_context *ctx,
+				    struct ostream *output,
+				    ARRAY_TYPE(uint32_t) *block_offsets,
+				    ARRAY_TYPE(uint32_t) *block_end_indexes,
+				    bool write_old_blocks)
+{
+	struct squat_uidlist *uidlist = ctx->uidlist;
+	unsigned int align, old_block_count, new_block_count;
+	uint32_t block_offset_count;
+	uoff_t block_list_offset;
+
+	i_assert(uidlist->trie->hdr.indexid != 0);
+	ctx->build_hdr.indexid = uidlist->trie->hdr.indexid;
+
+	if (array_count(block_end_indexes) == 0) {
+		ctx->build_hdr.used_file_size = output->offset;
+		ctx->build_hdr.block_list_offset = 0;
+		uidlist->hdr = ctx->build_hdr;
+		return;
+	}
+
+	align = output->offset % sizeof(uint32_t);
+	if (align != 0) {
+		static char null[sizeof(uint32_t)-1] = { 0, };
+
+		o_stream_nsend(output, null, sizeof(uint32_t) - align);
+	}
+	block_list_offset = output->offset;
+
+	new_block_count = array_count(block_offsets);
+	old_block_count = write_old_blocks ? uidlist->cur_block_count : 0;
+
+	block_offset_count = new_block_count + old_block_count;
+	o_stream_nsend(output, &block_offset_count, sizeof(block_offset_count));
+	/* write end indexes */
+	o_stream_nsend(output, uidlist->cur_block_end_indexes,
+		       old_block_count * sizeof(uint32_t));
+	o_stream_nsend(output, array_front(block_end_indexes),
+		       new_block_count * sizeof(uint32_t));
+	/* write offsets */
+	o_stream_nsend(output, uidlist->cur_block_offsets,
+		       old_block_count * sizeof(uint32_t));
+	o_stream_nsend(output, array_front(block_offsets),
+		       new_block_count * sizeof(uint32_t));
+	(void)o_stream_flush(output);
+
+	/* update header - it's written later when trie is locked */
+	ctx->build_hdr.block_list_offset = block_list_offset;
+	ctx->build_hdr.used_file_size = output->offset;
+	uidlist->hdr = ctx->build_hdr;
+}
+
+void squat_uidlist_build_flush(struct squat_uidlist_build_context *ctx)
+{
+	struct uidlist_list *lists;
+	uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp;
+	unsigned int i, j, count, max;
+	uint32_t block_offset, block_end_idx, start_offset;
+	uint32_t list_sizes[UIDLIST_BLOCK_LIST_COUNT];
+	size_t mem_size;
+
+	if (ctx->uidlist->corrupted)
+		return;
+
+	lists = array_get_modifiable(&ctx->lists, &count);
+	if (count == 0)
+		return;
+
+	/* write the lists and save the written sizes to uid_list[0] */
+	for (i = 0; i < count; i += UIDLIST_BLOCK_LIST_COUNT) {
+		start_offset = ctx->output->offset;
+		max = I_MIN(count - i, UIDLIST_BLOCK_LIST_COUNT);
+		for (j = 0; j < max; j++) {
+			if (uidlist_write(ctx->output, &lists[i+j],
+					  FALSE, &list_sizes[j]) < 0) {
+				squat_uidlist_set_corrupted(ctx->uidlist,
+							    "Broken uidlists");
+				return;
+			}
+		}
+
+		block_offset = ctx->output->offset;
+		block_end_idx = ctx->list_start_idx + i + max;
+		array_push_back(&ctx->block_offsets, &block_offset);
+		array_push_back(&ctx->block_end_indexes, &block_end_idx);
+
+		/* write the full size of the uidlists */
+		bufp = buf;
+		squat_pack_num(&bufp, block_offset - start_offset);
+		o_stream_nsend(ctx->output, buf, bufp - buf);
+
+		/* write the sizes/flags */
+		for (j = 0; j < max; j++) {
+			bufp = buf;
+			squat_pack_num(&bufp, list_sizes[j]);
+			o_stream_nsend(ctx->output, buf, bufp - buf);
+		}
+	}
+
+	mem_size = ctx->lists.arr.buffer->used +
+		ctx->block_offsets.arr.buffer->used +
+		ctx->block_end_indexes.arr.buffer->used;
+	if (ctx->uidlist->max_size < mem_size)
+		ctx->uidlist->max_size = mem_size;
+
+	ctx->list_start_idx += count;
+	array_clear(&ctx->lists);
+
+	uidlist_write_block_list_and_header(ctx, ctx->output,
+					    &ctx->block_offsets,
+					    &ctx->block_end_indexes, TRUE);
+
+	(void)squat_uidlist_map(ctx->uidlist);
+
+	array_clear(&ctx->block_offsets);
+	array_clear(&ctx->block_end_indexes);
+}
+
+int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx)
+{
+	if (ctx->uidlist->corrupted)
+		return -1;
+
+	if (!ctx->output->closed) {
+		(void)o_stream_seek(ctx->output, 0);
+		o_stream_nsend(ctx->output,
+			       &ctx->build_hdr, sizeof(ctx->build_hdr));
+		(void)o_stream_seek(ctx->output, ctx->build_hdr.used_file_size);
+	}
+
+	if (o_stream_finish(ctx->output) < 0) {
+		i_error("write() to %s failed: %s", ctx->uidlist->path,
+			o_stream_get_error(ctx->output));
+		return -1;
+	}
+	return 0;
+}
+
+void squat_uidlist_build_deinit(struct squat_uidlist_build_context **_ctx)
+{
+	struct squat_uidlist_build_context *ctx = *_ctx;
+
+	*_ctx = NULL;
+
+	i_assert(array_count(&ctx->lists) == 0 || ctx->uidlist->corrupted);
+	i_assert(ctx->uidlist->building);
+	ctx->uidlist->building = FALSE;
+
+	if (ctx->uidlist->file_lock != NULL)
+		file_unlock(&ctx->uidlist->file_lock);
+	else
+		file_dotlock_delete(&ctx->uidlist->dotlock);
+
+	if (ctx->need_reopen)
+		(void)squat_uidlist_open(ctx->uidlist);
+
+	array_free(&ctx->block_offsets);
+	array_free(&ctx->block_end_indexes);
+	array_free(&ctx->lists);
+	o_stream_ignore_last_errors(ctx->output);
+	o_stream_unref(&ctx->output);
+	i_free(ctx);
+}
+
+int squat_uidlist_rebuild_init(struct squat_uidlist_build_context *build_ctx,
+			       bool compress,
+			       struct squat_uidlist_rebuild_context **ctx_r)
+{
+	struct squat_uidlist_rebuild_context *ctx;
+	struct squat_uidlist_file_header hdr;
+	const char *temp_path;
+	int fd;
+
+	if (build_ctx->build_hdr.link_count == 0)
+		return 0;
+
+	if (!compress) {
+		if (build_ctx->build_hdr.link_count <
+		    build_ctx->build_hdr.count*2/3)
+			return 0;
+	}
+
+	/* make sure the entire uidlist is in memory before beginning,
+	   otherwise the pages are faulted to memory in random order which
+	   takes forever. */
+	if (squat_uidlist_read_to_memory(build_ctx->uidlist) < 0)
+		return -1;
+
+	temp_path = t_strconcat(build_ctx->uidlist->path, ".tmp", NULL);
+	fd = squat_trie_create_fd(build_ctx->uidlist->trie, temp_path, O_TRUNC);
+	if (fd == -1)
+		return -1;
+
+	ctx = i_new(struct squat_uidlist_rebuild_context, 1);
+	ctx->uidlist = build_ctx->uidlist;
+	ctx->build_ctx = build_ctx;
+	ctx->fd = fd;
+	ctx->output = o_stream_create_fd(ctx->fd, 0);
+	ctx->next_uid_list_idx = 0x100;
+	o_stream_cork(ctx->output);
+
+	i_zero(&hdr);
+	o_stream_nsend(ctx->output, &hdr, sizeof(hdr));
+
+	ctx->cur_block_start_offset = ctx->output->offset;
+	i_array_init(&ctx->new_block_offsets,
+		     build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT);
+	i_array_init(&ctx->new_block_end_indexes,
+		     build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT);
+	*ctx_r = ctx;
+	return 1;
+}
+
+static void
+uidlist_rebuild_flush_block(struct squat_uidlist_rebuild_context *ctx)
+{
+	uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp;
+	uint32_t block_offset, block_end_idx;
+	unsigned int i;
+
+	ctx->new_count += ctx->list_idx;
+
+	block_offset = ctx->output->offset;
+	block_end_idx = ctx->new_count;
+	array_push_back(&ctx->new_block_offsets, &block_offset);
+	array_push_back(&ctx->new_block_end_indexes, &block_end_idx);
+
+	/* this block's contents started from cur_block_start_offset and
+	   ended to current offset. write the size of this area. */
+	bufp = buf;
+	squat_pack_num(&bufp, block_offset - ctx->cur_block_start_offset);
+	o_stream_nsend(ctx->output, buf, bufp - buf);
+
+	/* write the sizes/flags */
+	for (i = 0; i < ctx->list_idx; i++) {
+		bufp = buf;
+		squat_pack_num(&bufp, ctx->list_sizes[i]);
+		o_stream_nsend(ctx->output, buf, bufp - buf);
+	}
+	ctx->cur_block_start_offset = ctx->output->offset;
+}
+
+uint32_t squat_uidlist_rebuild_next(struct squat_uidlist_rebuild_context *ctx,
+				    const ARRAY_TYPE(uint32_t) *uids)
+{
+	int ret;
+
+	T_BEGIN {
+		ret = uidlist_write_array(ctx->output, array_front(uids),
+					  array_count(uids), 0, 0, FALSE,
+					  &ctx->list_sizes[ctx->list_idx]);
+	} T_END;
+	if (ret < 0)
+		squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists");
+
+	if (++ctx->list_idx == UIDLIST_BLOCK_LIST_COUNT) {
+		uidlist_rebuild_flush_block(ctx);
+		ctx->list_idx = 0;
+	}
+	return ctx->next_uid_list_idx++ << 1;
+}
+
+uint32_t squat_uidlist_rebuild_nextu(struct squat_uidlist_rebuild_context *ctx,
+				     const ARRAY_TYPE(seq_range) *uids)
+{
+	const struct seq_range *range;
+	ARRAY_TYPE(uint32_t) tmp_uids;
+	uint32_t seq, uid1, ret;
+	unsigned int i, count;
+
+	range = array_get(uids, &count);
+	if (count == 0)
+		return 0;
+
+	if (range[count-1].seq2 < 8) {
+		/* we can use a singleton bitmask */
+		ret = 0;
+		for (i = 0; i < count; i++) {
+			for (seq = range[i].seq1; seq <= range[i].seq2; seq++)
+				ret |= 1 << (seq+1);
+		}
+		return ret;
+	}
+	if (count == 1 && range[0].seq1 == range[0].seq2) {
+		/* single UID */
+		return (range[0].seq1 << 1) | 1;
+	}
+
+	/* convert seq range to our internal representation and use the
+	   normal _rebuild_next() to write it */
+	i_array_init(&tmp_uids, 128);
+	for (i = 0; i < count; i++) {
+		if (range[i].seq1 == range[i].seq2)
+			array_push_back(&tmp_uids, &range[i].seq1);
+		else {
+			uid1 = range[i].seq1 | UID_LIST_MASK_RANGE;
+			array_push_back(&tmp_uids, &uid1);
+			array_push_back(&tmp_uids, &range[i].seq2);
+		}
+	}
+	ret = squat_uidlist_rebuild_next(ctx, &tmp_uids);
+	array_free(&tmp_uids);
+	return ret;
+}
+
+int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx,
+				 bool cancel)
+{
+	const char *temp_path;
+	int ret = 1;
+
+	if (ctx->list_idx != 0)
+		uidlist_rebuild_flush_block(ctx);
+	if (cancel || ctx->uidlist->corrupted)
+		ret = 0;
+
+	temp_path = t_strconcat(ctx->uidlist->path, ".tmp", NULL);
+	if (ret > 0) {
+		ctx->build_ctx->build_hdr.indexid =
+			ctx->uidlist->trie->hdr.indexid;
+		ctx->build_ctx->build_hdr.count = ctx->new_count;
+		ctx->build_ctx->build_hdr.link_count = 0;
+		uidlist_write_block_list_and_header(ctx->build_ctx, ctx->output,
+						    &ctx->new_block_offsets,
+						    &ctx->new_block_end_indexes,
+						    FALSE);
+		(void)o_stream_seek(ctx->output, 0);
+		o_stream_nsend(ctx->output, &ctx->build_ctx->build_hdr,
+			       sizeof(ctx->build_ctx->build_hdr));
+		(void)o_stream_seek(ctx->output,
+				    ctx->build_ctx->build_hdr.used_file_size);
+
+		if (ctx->uidlist->corrupted)
+			ret = -1;
+		else if (o_stream_finish(ctx->output) < 0) {
+			i_error("write(%s) failed: %s", temp_path,
+				o_stream_get_error(ctx->output));
+			ret = -1;
+		} else if (rename(temp_path, ctx->uidlist->path) < 0) {
+			i_error("rename(%s, %s) failed: %m",
+				temp_path, ctx->uidlist->path);
+			ret = -1;
+		}
+		ctx->build_ctx->need_reopen = TRUE;
+	} else {
+		o_stream_abort(ctx->output);
+	}
+
+	/* we no longer require the entire uidlist to be in memory,
+	   let it be used for something more useful. */
+	squat_uidlist_free_from_memory(ctx->uidlist);
+
+	o_stream_unref(&ctx->output);
+	if (close(ctx->fd) < 0)
+		i_error("close(%s) failed: %m", temp_path);
+
+	if (ret <= 0)
+		i_unlink(temp_path);
+	array_free(&ctx->new_block_offsets);
+	array_free(&ctx->new_block_end_indexes);
+	i_free(ctx);
+	return ret < 0 ? -1 : 0;
+}
+
+static void
+uidlist_flush(struct squat_uidlist_build_context *ctx,
+	      struct uidlist_list *list, uint32_t uid)
+{
+	uint32_t size, offset = ctx->output->offset;
+
+	ctx->build_hdr.link_count++;
+	if (uidlist_write(ctx->output, list, TRUE, &size) < 0)
+		squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists");
+
+	list->uid_count = 2;
+	list->uid_begins_with_pointer = TRUE;
+
+	list->uid_list[0] = offset;
+	list->uid_list[1] = uid;
+}
+
+static struct uidlist_list *
+uidlist_add_new(struct squat_uidlist_build_context *ctx, unsigned int count,
+		uint32_t *uid_list_idx_r)
+{
+	struct uidlist_list *list;
+
+	i_assert(array_count(&ctx->lists) +
+		 ctx->list_start_idx == ctx->build_hdr.count);
+	*uid_list_idx_r = (ctx->build_hdr.count + 0x100) << 1;
+	list = array_append_space(&ctx->lists);
+	ctx->build_hdr.count++;
+
+	list->uid_count = count;
+	return list;
+}
+
+uint32_t squat_uidlist_build_add_uid(struct squat_uidlist_build_context *ctx,
+				     uint32_t uid_list_idx, uint32_t uid)
+{
+	struct uidlist_list *list;
+	unsigned int idx, mask;
+	uint32_t *p;
+
+	if ((uid_list_idx & 1) != 0) {
+		/* adding second UID */
+		uint32_t prev_uid = uid_list_idx >> 1;
+
+		i_assert(prev_uid != uid);
+		list = uidlist_add_new(ctx, 2, &uid_list_idx);
+		list->uid_list[0] = prev_uid;
+		if (prev_uid + 1 == uid)
+			list->uid_list[0] |= UID_LIST_MASK_RANGE;
+		list->uid_list[1] = uid;
+		return uid_list_idx;
+	} else if (uid_list_idx < (0x100 << 1)) {
+		uint32_t old_list_idx;
+
+		if (uid < 8) {
+			/* UID lists containing only UIDs 0-7 are saved as
+			   uidlist values 2..511. think of it as a bitmask. */
+			uid_list_idx |= 1 << (uid + 1);
+			i_assert((uid_list_idx & 1) == 0);
+			return uid_list_idx;
+		}
+
+		if (uid_list_idx == 0) {
+			/* first UID */
+			return (uid << 1) | 1;
+		}
+
+		/* create a new list */
+		old_list_idx = uid_list_idx >> 1;
+		list = uidlist_add_new(ctx, 1, &uid_list_idx);
+		/* add the first UID ourself */
+		idx = 0;
+		i_assert((old_list_idx & 0xff) != 0);
+		for (mask = 1; mask <= 128; mask <<= 1, idx++) {
+			if ((old_list_idx & mask) != 0) {
+				list->uid_list[0] = idx;
+				idx++; mask <<= 1;
+				break;
+			}
+		}
+		for (; mask <= 128; mask <<= 1, idx++) {
+			if ((old_list_idx & mask) != 0) {
+				(void)squat_uidlist_build_add_uid(ctx,
+							uid_list_idx, idx);
+			}
+		}
+	}
+
+	/* add to existing list */
+	idx = (uid_list_idx >> 1) - 0x100;
+	if (idx < ctx->list_start_idx) {
+		list = uidlist_add_new(ctx, 2, &uid_list_idx);
+		list->uid_list[0] = UID_LIST_POINTER_MASK_LIST_IDX | idx;
+		list->uid_list[1] = uid;
+		list->uid_begins_with_pointer = TRUE;
+		ctx->build_hdr.link_count++;
+		return uid_list_idx;
+	}
+
+	idx -= ctx->list_start_idx;
+	if (idx >= array_count(&ctx->lists)) {
+		squat_uidlist_set_corrupted(ctx->uidlist,
+					    "missing/broken uidlist");
+		return 0;
+	}
+	list = array_idx_modifiable(&ctx->lists, idx);
+	i_assert(list->uid_count > 0);
+
+	p = &list->uid_list[list->uid_count-1];
+	i_assert(uid != *p || ctx->uidlist->corrupted ||
+		 (list->uid_count == 1 && list->uid_begins_with_pointer));
+	if (uid == *p + 1 &&
+	    (list->uid_count > 1 || !list->uid_begins_with_pointer)) {
+		/* use a range */
+		if (list->uid_count > 1 && (p[-1] & UID_LIST_MASK_RANGE) != 0 &&
+		   (list->uid_count > 2 || !list->uid_begins_with_pointer)) {
+			/* increase the existing range */
+			*p += 1;
+			return uid_list_idx;
+		}
+
+		if (list->uid_count == UIDLIST_LIST_SIZE) {
+			uidlist_flush(ctx, list, uid);
+			return uid_list_idx;
+		}
+		/* create a new range */
+		*p |= UID_LIST_MASK_RANGE;
+	} else {
+		if (list->uid_count == UIDLIST_LIST_SIZE) {
+			uidlist_flush(ctx, list, uid);
+			return uid_list_idx;
+		}
+	}
+
+	p++;
+	list->uid_count++;
+
+	*p = uid;
+	return uid_list_idx;
+}
+
+static void uidlist_array_append(ARRAY_TYPE(uint32_t) *uids, uint32_t uid)
+{
+	uint32_t *uidlist;
+	unsigned int count;
+
+	uidlist = array_get_modifiable(uids, &count);
+	if (count == 0) {
+		array_push_back(uids, &uid);
+		return;
+	}
+	if (uidlist[count-1] + 1 == uid) {
+		if (count > 1 && (uidlist[count-2] &
+				  UID_LIST_MASK_RANGE) != 0) {
+			uidlist[count-1]++;
+			return;
+		}
+		uidlist[count-1] |= UID_LIST_MASK_RANGE;
+	}
+	array_push_back(uids, &uid);
+}
+
+static void uidlist_array_append_range(ARRAY_TYPE(uint32_t) *uids,
+				       uint32_t uid1, uint32_t uid2)
+{
+	uint32_t *uidlist;
+	unsigned int count;
+
+	i_assert(uid1 < uid2);
+
+	uidlist = array_get_modifiable(uids, &count);
+	if (count == 0) {
+		uid1 |= UID_LIST_MASK_RANGE;
+		array_push_back(uids, &uid1);
+		array_push_back(uids, &uid2);
+		return;
+	}
+	if (uidlist[count-1] + 1 == uid1) {
+		if (count > 1 && (uidlist[count-2] &
+				  UID_LIST_MASK_RANGE) != 0) {
+			uidlist[count-1] = uid2;
+			return;
+		}
+		uidlist[count-1] |= UID_LIST_MASK_RANGE;
+	} else {
+		uid1 |= UID_LIST_MASK_RANGE;
+		array_push_back(uids, &uid1);
+	}
+	array_push_back(uids, &uid2);
+}
+
+static int
+squat_uidlist_get_at_offset(struct squat_uidlist *uidlist, uoff_t offset,
+			    uint32_t num, ARRAY_TYPE(uint32_t) *uids)
+{
+	const uint32_t *uid_list;
+	const uint8_t *p, *end;
+	uint32_t size, base_uid, next_uid, flags, prev;
+	uoff_t uidlist_data_offset;
+	unsigned int i, j, count;
+
+	if (num != 0)
+		uidlist_data_offset = offset;
+	else {
+		/* not given, read it */
+		if (uidlist_file_cache_read(uidlist, offset,
+					    SQUAT_PACK_MAX_SIZE) < 0)
+			return -1;
+
+		p = CONST_PTR_OFFSET(uidlist->data, offset);
+		end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size);
+		num = squat_unpack_num(&p, end);
+		uidlist_data_offset = p - (const uint8_t *)uidlist->data;
+	}
+	size = num >> 2;
+
+	if (uidlist_file_cache_read(uidlist, uidlist_data_offset, size) < 0)
+		return -1;
+	if (uidlist_data_offset + size > uidlist->data_size) {
+		squat_uidlist_set_corrupted(uidlist,
+					    "size points outside file");
+		return -1;
+	}
+
+	p = CONST_PTR_OFFSET(uidlist->data, uidlist_data_offset);
+	end = p + size;
+
+	flags = num;
+	if ((flags & UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER) != 0) {
+		/* link to the file */
+		prev = squat_unpack_num(&p, end);
+
+		if ((prev & 1) != 0) {
+			/* pointer to uidlist */
+			prev = ((prev >> 1) + 0x100) << 1;
+			if (squat_uidlist_get(uidlist, prev, uids) < 0)
+				return -1;
+		} else {
+			prev = offset - (prev >> 1);
+			if (squat_uidlist_get_at_offset(uidlist, prev,
+							0, uids) < 0)
+				return -1;
+		}
+		uid_list = array_get(uids, &count);
+		next_uid = count == 0 ? 0 : uid_list[count-1] + 1;
+	} else {
+		next_uid = 0;
+	}
+
+	num = base_uid = squat_unpack_num(&p, end);
+	if ((flags & UIDLIST_PACKED_FLAG_BITMASK) == 0)
+		base_uid >>= 1;
+	if (base_uid < next_uid) {
+		squat_uidlist_set_corrupted(uidlist,
+					    "broken continued uidlist");
+		return -1;
+	}
+
+	if ((flags & UIDLIST_PACKED_FLAG_BITMASK) != 0) {
+		/* bitmask */
+		size = end - p;
+
+		uidlist_array_append(uids, base_uid++);
+		for (i = 0; i < size; i++) {
+			for (j = 0; j < 8; j++, base_uid++) {
+				if ((p[i] & (1 << j)) != 0)
+					uidlist_array_append(uids, base_uid);
+			}
+		}
+	} else {
+		/* range */
+		for (;;) {
+			if ((num & 1) == 0) {
+				uidlist_array_append(uids, base_uid);
+			} else {
+				/* range */
+				uint32_t seq1 = base_uid;
+				base_uid += squat_unpack_num(&p, end) + 1;
+				uidlist_array_append_range(uids, seq1,
+							   base_uid);
+			}
+			if (p == end)
+				break;
+
+			num = squat_unpack_num(&p, end);
+			base_uid += (num >> 1) + 1;
+		}
+	}
+	return 0;
+}
+
+static int
+squat_uidlist_get_offset(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
+			 uint32_t *offset_r, uint32_t *num_r)
+{
+	const uint8_t *p, *end;
+	unsigned int idx;
+	uint32_t num, skip_bytes, uidlists_offset;
+	size_t max_map_size;
+
+	if (uidlist->fd == -1) {
+		squat_uidlist_set_corrupted(uidlist, "no uidlists");
+		return -1;
+	}
+
+	if (bsearch_insert_pos(&uid_list_idx, uidlist->cur_block_end_indexes,
+			       uidlist->cur_block_count,
+			       sizeof(uint32_t), uint32_cmp, &idx))
+		idx++;
+	if (unlikely(idx == uidlist->cur_block_count)) {
+		squat_uidlist_set_corrupted(uidlist, "uidlist not found");
+		return -1;
+	}
+	i_assert(uidlist->cur_block_end_indexes != NULL);
+	if (unlikely(idx > 0 &&
+		     uidlist->cur_block_end_indexes[idx-1] > uid_list_idx)) {
+		squat_uidlist_set_corrupted(uidlist, "broken block list");
+		return -1;
+	}
+
+	/* make sure everything is mapped */
+	uid_list_idx -= idx == 0 ? 0 : uidlist->cur_block_end_indexes[idx-1];
+	max_map_size = SQUAT_PACK_MAX_SIZE * (1+uid_list_idx);
+	if (uidlist_file_cache_read(uidlist, uidlist->cur_block_offsets[idx],
+				    max_map_size) < 0)
+		return -1;
+
+	/* find the uidlist inside the block */
+	i_assert(uidlist->cur_block_offsets != NULL);
+	p = CONST_PTR_OFFSET(uidlist->data, uidlist->cur_block_offsets[idx]);
+	end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size);
+
+	uidlists_offset = uidlist->cur_block_offsets[idx] -
+		squat_unpack_num(&p, end);
+	for (skip_bytes = 0; uid_list_idx > 0; uid_list_idx--) {
+		num = squat_unpack_num(&p, end);
+		skip_bytes += num >> 2;
+	}
+	*offset_r = uidlists_offset + skip_bytes;
+	*num_r = squat_unpack_num(&p, end);
+
+	if (unlikely(p == end)) {
+		squat_uidlist_set_corrupted(uidlist, "broken file");
+		return -1;
+	}
+	if (unlikely(*offset_r > uidlist->mmap_size &&
+		     uidlist->mmap_base != NULL)) {
+		squat_uidlist_set_corrupted(uidlist, "broken offset");
+		return -1;
+	}
+	return 0;
+}
+
+int squat_uidlist_get(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
+		      ARRAY_TYPE(uint32_t) *uids)
+{
+	unsigned int mask;
+	uint32_t uid, offset, num;
+
+	if ((uid_list_idx & 1) != 0) {
+		/* single UID */
+		uid = uid_list_idx >> 1;
+		uidlist_array_append(uids, uid);
+		return 0;
+	} else if (uid_list_idx < (0x100 << 1)) {
+		/* bitmask */
+		for (uid = 0, mask = 2; mask <= 256; mask <<= 1, uid++) {
+			if ((uid_list_idx & mask) != 0)
+				uidlist_array_append(uids, uid);
+		}
+		return 0;
+	}
+
+	uid_list_idx = (uid_list_idx >> 1) - 0x100;
+	if (squat_uidlist_get_offset(uidlist, uid_list_idx, &offset, &num) < 0)
+		return -1;
+	return squat_uidlist_get_at_offset(uidlist, offset, num, uids);
+}
+
+uint32_t squat_uidlist_singleton_last_uid(uint32_t uid_list_idx)
+{
+	unsigned int idx, mask;
+
+	if ((uid_list_idx & 1) != 0) {
+		/* single UID */
+		return uid_list_idx >> 1;
+	} else if (uid_list_idx < (0x100 << 1)) {
+		/* bitmask */
+		if (uid_list_idx == 2) {
+			/* just a quick optimization */
+			return 0;
+		}
+		for (idx = 7, mask = 256; mask > 2; mask >>= 1, idx--) {
+			if ((uid_list_idx & mask) != 0)
+				return idx;
+		}
+	}
+
+	i_unreached();
+	return 0;
+}
+
+int squat_uidlist_get_seqrange(struct squat_uidlist *uidlist,
+			       uint32_t uid_list_idx,
+			       ARRAY_TYPE(seq_range) *seq_range_arr)
+{
+	ARRAY_TYPE(uint32_t) tmp_uid_arr;
+	struct seq_range range;
+	const uint32_t *tmp_uids;
+	unsigned int i, count;
+	int ret;
+
+	i_array_init(&tmp_uid_arr, 128);
+	ret = squat_uidlist_get(uidlist, uid_list_idx, &tmp_uid_arr);
+	if (ret == 0) {
+		tmp_uids = array_get(&tmp_uid_arr, &count);
+		for (i = 0; i < count; i++) {
+			if ((tmp_uids[i] & UID_LIST_MASK_RANGE) == 0)
+				range.seq1 = range.seq2 = tmp_uids[i];
+			else {
+				range.seq1 = tmp_uids[i] & ~UID_LIST_MASK_RANGE;
+				range.seq2 = tmp_uids[++i];
+			}
+			array_push_back(seq_range_arr, &range);
+		}
+	}
+	array_free(&tmp_uid_arr);
+	return ret;
+}
+
+int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
+			 ARRAY_TYPE(seq_range) *uids)
+{
+	const struct seq_range *parent_range;
+	ARRAY_TYPE(seq_range) dest_uids;
+	ARRAY_TYPE(uint32_t) relative_uids;
+	const uint32_t *rel_range;
+	unsigned int i, rel_count, parent_idx, parent_count, diff, parent_uid;
+	uint32_t prev_seq, seq1, seq2;
+	int ret = 0;
+
+	parent_range = array_get(uids, &parent_count);
+	if (parent_count == 0)
+		return 0;
+
+	i_array_init(&relative_uids, 128);
+	i_array_init(&dest_uids, 128);
+	if (squat_uidlist_get(uidlist, uid_list_idx, &relative_uids) < 0)
+		ret = -1;
+
+	parent_idx = 0;
+	rel_range = array_get(&relative_uids, &rel_count);
+	prev_seq = 0; parent_uid = parent_range[0].seq1;
+	for (i = 0; i < rel_count; i++) {
+		if (unlikely(parent_uid == (uint32_t)-1)) {
+			i_error("broken UID ranges");
+			ret = -1;
+			break;
+		}
+		if ((rel_range[i] & UID_LIST_MASK_RANGE) == 0)
+			seq1 = seq2 = rel_range[i];
+		else {
+			seq1 = (rel_range[i] & ~UID_LIST_MASK_RANGE);
+			seq2 = rel_range[++i];
+		}
+		i_assert(seq1 >= prev_seq);
+		diff = seq1 - prev_seq;
+		while (diff > 0) {
+			if (unlikely(parent_uid == (uint32_t)-1)) {
+				i_error("broken UID ranges");
+				ret = -1;
+				break;
+			}
+
+			for (; parent_idx < parent_count; parent_idx++) {
+				if (parent_range[parent_idx].seq2 <= parent_uid)
+					continue;
+				if (parent_uid < parent_range[parent_idx].seq1)
+					parent_uid = parent_range[parent_idx].seq1;
+				else
+					parent_uid++;
+				break;
+			}
+			diff--;
+		}
+		diff = seq2 - seq1 + 1;
+		while (diff > 0) {
+			if (unlikely(parent_uid == (uint32_t)-1)) {
+				i_error("broken UID ranges");
+				ret = -1;
+				break;
+			}
+			seq_range_array_add(&dest_uids, parent_uid);
+			for (; parent_idx < parent_count; parent_idx++) {
+				if (parent_range[parent_idx].seq2 <= parent_uid)
+					continue;
+				if (parent_uid < parent_range[parent_idx].seq1)
+					parent_uid = parent_range[parent_idx].seq1;
+				else
+					parent_uid++;
+				break;
+			}
+			diff--;
+		}
+
+		prev_seq = seq2 + 1;
+	}
+
+	buffer_set_used_size(uids->arr.buffer, 0);
+	array_append_array(uids, &dest_uids);
+
+	array_free(&relative_uids);
+	array_free(&dest_uids);
+	return ret;
+}
+
+size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist,
+			      unsigned int *count_r)
+{
+	*count_r = uidlist->hdr.count;
+	return uidlist->max_size;
+}
diff --git a/src/plugins/fts-squat/squat-uidlist.h b/src/plugins/fts-squat/squat-uidlist.h
new file mode 100644
index 0000000..79ed791
--- /dev/null
+++ b/src/plugins/fts-squat/squat-uidlist.h
@@ -0,0 +1,71 @@
+#ifndef SQUAT_UIDLIST_H
+#define SQUAT_UIDLIST_H
+
+struct squat_trie;
+struct squat_uidlist_build_context;
+struct squat_uidlist_rebuild_context;
+
+struct squat_uidlist_file_header {
+	uint32_t indexid;
+	uint32_t used_file_size;
+	uint32_t block_list_offset;
+	uint32_t count, link_count;
+};
+
+/*
+   uidlist file:
+
+   struct uidlist_header;
+
+   // size includes both prev_offset and uidlist
+   packed (size << 2) | packed_flags; // UIDLIST_PACKED_FLAG_*
+   [packed prev_offset;] // If UIDLIST_PACKED_FLAG_BEGINS_WITH_OFFSET is set
+   if (UIDLIST_PACKED_FLAG_BITMASK) {
+     packed base_uid; // first UID in uidlist
+     uint8_t bitmask[]; // first bit is base_uid+1
+   } else {
+     // FIXME: packed range
+   }
+*/
+
+#define UIDLIST_IS_SINGLETON(idx) \
+	(((idx) & 1) != 0 || (idx) < (0x100 << 1))
+
+struct squat_uidlist *squat_uidlist_init(struct squat_trie *trie);
+void squat_uidlist_deinit(struct squat_uidlist *uidlist);
+
+int squat_uidlist_refresh(struct squat_uidlist *uidlist);
+
+int squat_uidlist_build_init(struct squat_uidlist *uidlist,
+			     struct squat_uidlist_build_context **ctx_r);
+uint32_t squat_uidlist_build_add_uid(struct squat_uidlist_build_context *ctx,
+				     uint32_t uid_list_idx, uint32_t uid);
+void squat_uidlist_build_flush(struct squat_uidlist_build_context *ctx);
+int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx);
+void squat_uidlist_build_deinit(struct squat_uidlist_build_context **ctx);
+
+int squat_uidlist_rebuild_init(struct squat_uidlist_build_context *build_ctx,
+			       bool compress,
+			       struct squat_uidlist_rebuild_context **ctx_r);
+uint32_t squat_uidlist_rebuild_next(struct squat_uidlist_rebuild_context *ctx,
+				    const ARRAY_TYPE(uint32_t) *uids);
+uint32_t squat_uidlist_rebuild_nextu(struct squat_uidlist_rebuild_context *ctx,
+				     const ARRAY_TYPE(seq_range) *uids);
+int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx,
+				 bool cancel);
+
+int squat_uidlist_get(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
+		      ARRAY_TYPE(uint32_t) *uids);
+uint32_t squat_uidlist_singleton_last_uid(uint32_t uid_list_idx);
+
+int squat_uidlist_get_seqrange(struct squat_uidlist *uidlist,
+			       uint32_t uid_list_idx,
+			       ARRAY_TYPE(seq_range) *seq_range_arr);
+int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
+			 ARRAY_TYPE(seq_range) *uids);
+
+void squat_uidlist_delete(struct squat_uidlist *uidlist);
+size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist,
+			      unsigned int *count_r);
+
+#endif
diff --git a/src/plugins/fts/Makefile.am b/src/plugins/fts/Makefile.am
new file mode 100644
index 0000000..2e7753c
--- /dev/null
+++ b/src/plugins/fts/Makefile.am
@@ -0,0 +1,74 @@
+pkglibexecdir = $(libexecdir)/dovecot
+doveadm_moduledir = $(moduledir)/doveadm
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-fts \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-http \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/doveadm
+
+NOPLUGIN_LDFLAGS =
+lib20_doveadm_fts_plugin_la_LDFLAGS = -module -avoid-version
+lib20_fts_plugin_la_LDFLAGS = -module -avoid-version
+
+module_LTLIBRARIES = \
+	lib20_fts_plugin.la
+
+lib20_fts_plugin_la_LIBADD = ../../lib-fts/libfts.la
+
+lib20_fts_plugin_la_SOURCES = \
+	fts-api.c \
+	fts-build-mail.c \
+	fts-expunge-log.c \
+	fts-indexer.c \
+	fts-parser.c \
+	fts-parser-html.c \
+	fts-parser-script.c \
+	fts-parser-tika.c \
+	fts-plugin.c \
+	fts-search.c \
+	fts-search-args.c \
+	fts-search-serialize.c \
+	fts-storage.c \
+	fts-user.c
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = \
+	fts-api.h \
+	fts-api-private.h \
+	fts-expunge-log.h \
+	fts-indexer.h \
+	fts-parser.h \
+	fts-storage.h \
+	fts-user.h
+
+noinst_HEADERS = \
+	doveadm-fts.h \
+	fts-build-mail.h \
+	fts-plugin.h \
+	fts-search-args.h \
+	fts-search-serialize.h
+
+pkglibexec_PROGRAMS = xml2text
+
+xml2text_SOURCES = xml2text.c fts-parser-html.c
+xml2text_CPPFLAGS = $(AM_CPPFLAGS) $(BINARY_CFLAGS)
+xml2text_LDADD = $(LIBDOVECOT) $(BINARY_LDFLAGS)
+xml2text_DEPENDENCIES = $(module_LTLIBRARIES) $(LIBDOVECOT_DEPS)
+
+pkglibexec_SCRIPTS = decode2text.sh
+EXTRA_DIST = $(pkglibexec_SCRIPTS)
+
+doveadm_module_LTLIBRARIES = \
+	lib20_doveadm_fts_plugin.la
+
+lib20_doveadm_fts_plugin_la_SOURCES = \
+	doveadm-fts.c \
+	doveadm-dump-fts-expunge-log.c
diff --git a/src/plugins/fts/Makefile.in b/src/plugins/fts/Makefile.in
new file mode 100644
index 0000000..624f69f
--- /dev/null
+++ b/src/plugins/fts/Makefile.in
@@ -0,0 +1,1140 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+pkglibexec_PROGRAMS = xml2text$(EXEEXT)
+subdir = src/plugins/fts
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+	$(pkginc_lib_HEADERS) $(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__installdirs = "$(DESTDIR)$(pkglibexecdir)" \
+	"$(DESTDIR)$(doveadm_moduledir)" "$(DESTDIR)$(moduledir)" \
+	"$(DESTDIR)$(pkglibexecdir)" "$(DESTDIR)$(pkginc_libdir)"
+PROGRAMS = $(pkglibexec_PROGRAMS)
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+LTLIBRARIES = $(doveadm_module_LTLIBRARIES) $(module_LTLIBRARIES)
+lib20_doveadm_fts_plugin_la_LIBADD =
+am_lib20_doveadm_fts_plugin_la_OBJECTS = doveadm-fts.lo \
+	doveadm-dump-fts-expunge-log.lo
+lib20_doveadm_fts_plugin_la_OBJECTS =  \
+	$(am_lib20_doveadm_fts_plugin_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+lib20_doveadm_fts_plugin_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+	$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+	$(AM_CFLAGS) $(CFLAGS) $(lib20_doveadm_fts_plugin_la_LDFLAGS) \
+	$(LDFLAGS) -o $@
+lib20_fts_plugin_la_DEPENDENCIES = ../../lib-fts/libfts.la
+am_lib20_fts_plugin_la_OBJECTS = fts-api.lo fts-build-mail.lo \
+	fts-expunge-log.lo fts-indexer.lo fts-parser.lo \
+	fts-parser-html.lo fts-parser-script.lo fts-parser-tika.lo \
+	fts-plugin.lo fts-search.lo fts-search-args.lo \
+	fts-search-serialize.lo fts-storage.lo fts-user.lo
+lib20_fts_plugin_la_OBJECTS = $(am_lib20_fts_plugin_la_OBJECTS)
+lib20_fts_plugin_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+	$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+	$(AM_CFLAGS) $(CFLAGS) $(lib20_fts_plugin_la_LDFLAGS) \
+	$(LDFLAGS) -o $@
+am_xml2text_OBJECTS = xml2text-xml2text.$(OBJEXT) \
+	xml2text-fts-parser-html.$(OBJEXT)
+xml2text_OBJECTS = $(am_xml2text_OBJECTS)
+am__DEPENDENCIES_1 =
+SCRIPTS = $(pkglibexec_SCRIPTS)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/doveadm-dump-fts-expunge-log.Plo \
+	./$(DEPDIR)/doveadm-fts.Plo ./$(DEPDIR)/fts-api.Plo \
+	./$(DEPDIR)/fts-build-mail.Plo ./$(DEPDIR)/fts-expunge-log.Plo \
+	./$(DEPDIR)/fts-indexer.Plo ./$(DEPDIR)/fts-parser-html.Plo \
+	./$(DEPDIR)/fts-parser-script.Plo \
+	./$(DEPDIR)/fts-parser-tika.Plo ./$(DEPDIR)/fts-parser.Plo \
+	./$(DEPDIR)/fts-plugin.Plo ./$(DEPDIR)/fts-search-args.Plo \
+	./$(DEPDIR)/fts-search-serialize.Plo \
+	./$(DEPDIR)/fts-search.Plo ./$(DEPDIR)/fts-storage.Plo \
+	./$(DEPDIR)/fts-user.Plo \
+	./$(DEPDIR)/xml2text-fts-parser-html.Po \
+	./$(DEPDIR)/xml2text-xml2text.Po
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(lib20_doveadm_fts_plugin_la_SOURCES) \
+	$(lib20_fts_plugin_la_SOURCES) $(xml2text_SOURCES)
+DIST_SOURCES = $(lib20_doveadm_fts_plugin_la_SOURCES) \
+	$(lib20_fts_plugin_la_SOURCES) $(xml2text_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+pkglibexecdir = $(libexecdir)/dovecot
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = 
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+doveadm_moduledir = $(moduledir)/doveadm
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-fts \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-http \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index \
+	-I$(top_srcdir)/src/doveadm
+
+lib20_doveadm_fts_plugin_la_LDFLAGS = -module -avoid-version
+lib20_fts_plugin_la_LDFLAGS = -module -avoid-version
+module_LTLIBRARIES = \
+	lib20_fts_plugin.la
+
+lib20_fts_plugin_la_LIBADD = ../../lib-fts/libfts.la
+lib20_fts_plugin_la_SOURCES = \
+	fts-api.c \
+	fts-build-mail.c \
+	fts-expunge-log.c \
+	fts-indexer.c \
+	fts-parser.c \
+	fts-parser-html.c \
+	fts-parser-script.c \
+	fts-parser-tika.c \
+	fts-plugin.c \
+	fts-search.c \
+	fts-search-args.c \
+	fts-search-serialize.c \
+	fts-storage.c \
+	fts-user.c
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = \
+	fts-api.h \
+	fts-api-private.h \
+	fts-expunge-log.h \
+	fts-indexer.h \
+	fts-parser.h \
+	fts-storage.h \
+	fts-user.h
+
+noinst_HEADERS = \
+	doveadm-fts.h \
+	fts-build-mail.h \
+	fts-plugin.h \
+	fts-search-args.h \
+	fts-search-serialize.h
+
+xml2text_SOURCES = xml2text.c fts-parser-html.c
+xml2text_CPPFLAGS = $(AM_CPPFLAGS) $(BINARY_CFLAGS)
+xml2text_LDADD = $(LIBDOVECOT) $(BINARY_LDFLAGS)
+xml2text_DEPENDENCIES = $(module_LTLIBRARIES) $(LIBDOVECOT_DEPS)
+pkglibexec_SCRIPTS = decode2text.sh
+EXTRA_DIST = $(pkglibexec_SCRIPTS)
+doveadm_module_LTLIBRARIES = \
+	lib20_doveadm_fts_plugin.la
+
+lib20_doveadm_fts_plugin_la_SOURCES = \
+	doveadm-fts.c \
+	doveadm-dump-fts-expunge-log.c
+
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/plugins/fts/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/plugins/fts/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+install-pkglibexecPROGRAMS: $(pkglibexec_PROGRAMS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkglibexecdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkglibexecdir)" || exit 1; \
+	fi; \
+	for p in $$list; do echo "$$p $$p"; done | \
+	sed 's/$(EXEEXT)$$//' | \
+	while read p p1; do if test -f $$p \
+	 || test -f $$p1 \
+	  ; then echo "$$p"; echo "$$p"; else :; fi; \
+	done | \
+	sed -e 'p;s,.*/,,;n;h' \
+	    -e 's|.*|.|' \
+	    -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \
+	sed 'N;N;N;s,\n, ,g' | \
+	$(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \
+	  { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+	    if ($$2 == $$4) files[d] = files[d] " " $$1; \
+	    else { print "f", $$3 "/" $$4, $$1; } } \
+	  END { for (d in files) print "f", d, files[d] }' | \
+	while read type dir files; do \
+	    if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+	    test -z "$$files" || { \
+	    echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(pkglibexecdir)$$dir'"; \
+	    $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(pkglibexecdir)$$dir" || exit $$?; \
+	    } \
+	; done
+
+uninstall-pkglibexecPROGRAMS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \
+	files=`for p in $$list; do echo "$$p"; done | \
+	  sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \
+	      -e 's/$$/$(EXEEXT)/' \
+	`; \
+	test -n "$$list" || exit 0; \
+	echo " ( cd '$(DESTDIR)$(pkglibexecdir)' && rm -f" $$files ")"; \
+	cd "$(DESTDIR)$(pkglibexecdir)" && rm -f $$files
+
+clean-pkglibexecPROGRAMS:
+	@list='$(pkglibexec_PROGRAMS)'; test -n "$$list" || exit 0; \
+	echo " rm -f" $$list; \
+	rm -f $$list || exit $$?; \
+	test -n "$(EXEEXT)" || exit 0; \
+	list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+	echo " rm -f" $$list; \
+	rm -f $$list
+
+install-doveadm_moduleLTLIBRARIES: $(doveadm_module_LTLIBRARIES)
+	@$(NORMAL_INSTALL)
+	@list='$(doveadm_module_LTLIBRARIES)'; test -n "$(doveadm_moduledir)" || list=; \
+	list2=; for p in $$list; do \
+	  if test -f $$p; then \
+	    list2="$$list2 $$p"; \
+	  else :; fi; \
+	done; \
+	test -z "$$list2" || { \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(doveadm_moduledir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(doveadm_moduledir)" || exit 1; \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(doveadm_moduledir)'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(doveadm_moduledir)"; \
+	}
+
+uninstall-doveadm_moduleLTLIBRARIES:
+	@$(NORMAL_UNINSTALL)
+	@list='$(doveadm_module_LTLIBRARIES)'; test -n "$(doveadm_moduledir)" || list=; \
+	for p in $$list; do \
+	  $(am__strip_dir) \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(doveadm_moduledir)/$$f'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(doveadm_moduledir)/$$f"; \
+	done
+
+clean-doveadm_moduleLTLIBRARIES:
+	-test -z "$(doveadm_module_LTLIBRARIES)" || rm -f $(doveadm_module_LTLIBRARIES)
+	@list='$(doveadm_module_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+install-moduleLTLIBRARIES: $(module_LTLIBRARIES)
+	@$(NORMAL_INSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	list2=; for p in $$list; do \
+	  if test -f $$p; then \
+	    list2="$$list2 $$p"; \
+	  else :; fi; \
+	done; \
+	test -z "$$list2" || { \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(moduledir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(moduledir)" || exit 1; \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(moduledir)'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(moduledir)"; \
+	}
+
+uninstall-moduleLTLIBRARIES:
+	@$(NORMAL_UNINSTALL)
+	@list='$(module_LTLIBRARIES)'; test -n "$(moduledir)" || list=; \
+	for p in $$list; do \
+	  $(am__strip_dir) \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(moduledir)/$$f'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(moduledir)/$$f"; \
+	done
+
+clean-moduleLTLIBRARIES:
+	-test -z "$(module_LTLIBRARIES)" || rm -f $(module_LTLIBRARIES)
+	@list='$(module_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+lib20_doveadm_fts_plugin.la: $(lib20_doveadm_fts_plugin_la_OBJECTS) $(lib20_doveadm_fts_plugin_la_DEPENDENCIES) $(EXTRA_lib20_doveadm_fts_plugin_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(lib20_doveadm_fts_plugin_la_LINK) -rpath $(doveadm_moduledir) $(lib20_doveadm_fts_plugin_la_OBJECTS) $(lib20_doveadm_fts_plugin_la_LIBADD) $(LIBS)
+
+lib20_fts_plugin.la: $(lib20_fts_plugin_la_OBJECTS) $(lib20_fts_plugin_la_DEPENDENCIES) $(EXTRA_lib20_fts_plugin_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(lib20_fts_plugin_la_LINK) -rpath $(moduledir) $(lib20_fts_plugin_la_OBJECTS) $(lib20_fts_plugin_la_LIBADD) $(LIBS)
+
+xml2text$(EXEEXT): $(xml2text_OBJECTS) $(xml2text_DEPENDENCIES) $(EXTRA_xml2text_DEPENDENCIES) 
+	@rm -f xml2text$(EXEEXT)
+	$(AM_V_CCLD)$(LINK) $(xml2text_OBJECTS) $(xml2text_LDADD) $(LIBS)
+install-pkglibexecSCRIPTS: $(pkglibexec_SCRIPTS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkglibexec_SCRIPTS)'; test -n "$(pkglibexecdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkglibexecdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkglibexecdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \
+	done | \
+	sed -e 'p;s,.*/,,;n' \
+	    -e 'h;s|.*|.|' \
+	    -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \
+	$(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \
+	  { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+	    if ($$2 == $$4) { files[d] = files[d] " " $$1; \
+	      if (++n[d] == $(am__install_max)) { \
+		print "f", d, files[d]; n[d] = 0; files[d] = "" } } \
+	    else { print "f", d "/" $$4, $$1 } } \
+	  END { for (d in files) print "f", d, files[d] }' | \
+	while read type dir files; do \
+	     if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+	     test -z "$$files" || { \
+	       echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(pkglibexecdir)$$dir'"; \
+	       $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(pkglibexecdir)$$dir" || exit $$?; \
+	     } \
+	; done
+
+uninstall-pkglibexecSCRIPTS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkglibexec_SCRIPTS)'; test -n "$(pkglibexecdir)" || exit 0; \
+	files=`for p in $$list; do echo "$$p"; done | \
+	       sed -e 's,.*/,,;$(transform)'`; \
+	dir='$(DESTDIR)$(pkglibexecdir)'; $(am__uninstall_files_from_dir)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-fts-expunge-log.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-fts.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-api.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-build-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-expunge-log.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-indexer.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-parser-html.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-parser-script.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-parser-tika.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-parser.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-plugin.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-search-args.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-search-serialize.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-search.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-storage.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fts-user.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xml2text-fts-parser-html.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xml2text-xml2text.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+xml2text-xml2text.o: xml2text.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xml2text-xml2text.o -MD -MP -MF $(DEPDIR)/xml2text-xml2text.Tpo -c -o xml2text-xml2text.o `test -f 'xml2text.c' || echo '$(srcdir)/'`xml2text.c
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/xml2text-xml2text.Tpo $(DEPDIR)/xml2text-xml2text.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='xml2text.c' object='xml2text-xml2text.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xml2text-xml2text.o `test -f 'xml2text.c' || echo '$(srcdir)/'`xml2text.c
+
+xml2text-xml2text.obj: xml2text.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xml2text-xml2text.obj -MD -MP -MF $(DEPDIR)/xml2text-xml2text.Tpo -c -o xml2text-xml2text.obj `if test -f 'xml2text.c'; then $(CYGPATH_W) 'xml2text.c'; else $(CYGPATH_W) '$(srcdir)/xml2text.c'; fi`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/xml2text-xml2text.Tpo $(DEPDIR)/xml2text-xml2text.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='xml2text.c' object='xml2text-xml2text.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xml2text-xml2text.obj `if test -f 'xml2text.c'; then $(CYGPATH_W) 'xml2text.c'; else $(CYGPATH_W) '$(srcdir)/xml2text.c'; fi`
+
+xml2text-fts-parser-html.o: fts-parser-html.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xml2text-fts-parser-html.o -MD -MP -MF $(DEPDIR)/xml2text-fts-parser-html.Tpo -c -o xml2text-fts-parser-html.o `test -f 'fts-parser-html.c' || echo '$(srcdir)/'`fts-parser-html.c
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/xml2text-fts-parser-html.Tpo $(DEPDIR)/xml2text-fts-parser-html.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='fts-parser-html.c' object='xml2text-fts-parser-html.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xml2text-fts-parser-html.o `test -f 'fts-parser-html.c' || echo '$(srcdir)/'`fts-parser-html.c
+
+xml2text-fts-parser-html.obj: fts-parser-html.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xml2text-fts-parser-html.obj -MD -MP -MF $(DEPDIR)/xml2text-fts-parser-html.Tpo -c -o xml2text-fts-parser-html.obj `if test -f 'fts-parser-html.c'; then $(CYGPATH_W) 'fts-parser-html.c'; else $(CYGPATH_W) '$(srcdir)/fts-parser-html.c'; fi`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/xml2text-fts-parser-html.Tpo $(DEPDIR)/xml2text-fts-parser-html.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='fts-parser-html.c' object='xml2text-fts-parser-html.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xml2text_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xml2text-fts-parser-html.obj `if test -f 'fts-parser-html.c'; then $(CYGPATH_W) 'fts-parser-html.c'; else $(CYGPATH_W) '$(srcdir)/fts-parser-html.c'; fi`
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+check: check-am
+all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(SCRIPTS) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkglibexecdir)" "$(DESTDIR)$(doveadm_moduledir)" "$(DESTDIR)$(moduledir)" "$(DESTDIR)$(pkglibexecdir)" "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-doveadm_moduleLTLIBRARIES clean-generic clean-libtool \
+	clean-moduleLTLIBRARIES clean-pkglibexecPROGRAMS \
+	mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/doveadm-dump-fts-expunge-log.Plo
+	-rm -f ./$(DEPDIR)/doveadm-fts.Plo
+	-rm -f ./$(DEPDIR)/fts-api.Plo
+	-rm -f ./$(DEPDIR)/fts-build-mail.Plo
+	-rm -f ./$(DEPDIR)/fts-expunge-log.Plo
+	-rm -f ./$(DEPDIR)/fts-indexer.Plo
+	-rm -f ./$(DEPDIR)/fts-parser-html.Plo
+	-rm -f ./$(DEPDIR)/fts-parser-script.Plo
+	-rm -f ./$(DEPDIR)/fts-parser-tika.Plo
+	-rm -f ./$(DEPDIR)/fts-parser.Plo
+	-rm -f ./$(DEPDIR)/fts-plugin.Plo
+	-rm -f ./$(DEPDIR)/fts-search-args.Plo
+	-rm -f ./$(DEPDIR)/fts-search-serialize.Plo
+	-rm -f ./$(DEPDIR)/fts-search.Plo
+	-rm -f ./$(DEPDIR)/fts-storage.Plo
+	-rm -f ./$(DEPDIR)/fts-user.Plo
+	-rm -f ./$(DEPDIR)/xml2text-fts-parser-html.Po
+	-rm -f ./$(DEPDIR)/xml2text-xml2text.Po
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-doveadm_moduleLTLIBRARIES \
+	install-moduleLTLIBRARIES install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am: install-pkglibexecPROGRAMS install-pkglibexecSCRIPTS
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/doveadm-dump-fts-expunge-log.Plo
+	-rm -f ./$(DEPDIR)/doveadm-fts.Plo
+	-rm -f ./$(DEPDIR)/fts-api.Plo
+	-rm -f ./$(DEPDIR)/fts-build-mail.Plo
+	-rm -f ./$(DEPDIR)/fts-expunge-log.Plo
+	-rm -f ./$(DEPDIR)/fts-indexer.Plo
+	-rm -f ./$(DEPDIR)/fts-parser-html.Plo
+	-rm -f ./$(DEPDIR)/fts-parser-script.Plo
+	-rm -f ./$(DEPDIR)/fts-parser-tika.Plo
+	-rm -f ./$(DEPDIR)/fts-parser.Plo
+	-rm -f ./$(DEPDIR)/fts-plugin.Plo
+	-rm -f ./$(DEPDIR)/fts-search-args.Plo
+	-rm -f ./$(DEPDIR)/fts-search-serialize.Plo
+	-rm -f ./$(DEPDIR)/fts-search.Plo
+	-rm -f ./$(DEPDIR)/fts-storage.Plo
+	-rm -f ./$(DEPDIR)/fts-user.Plo
+	-rm -f ./$(DEPDIR)/xml2text-fts-parser-html.Po
+	-rm -f ./$(DEPDIR)/xml2text-xml2text.Po
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-doveadm_moduleLTLIBRARIES \
+	uninstall-moduleLTLIBRARIES uninstall-pkginc_libHEADERS \
+	uninstall-pkglibexecPROGRAMS uninstall-pkglibexecSCRIPTS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \
+	clean-doveadm_moduleLTLIBRARIES clean-generic clean-libtool \
+	clean-moduleLTLIBRARIES clean-pkglibexecPROGRAMS cscopelist-am \
+	ctags ctags-am distclean distclean-compile distclean-generic \
+	distclean-libtool distclean-tags distdir dvi dvi-am html \
+	html-am info info-am install install-am install-data \
+	install-data-am install-doveadm_moduleLTLIBRARIES install-dvi \
+	install-dvi-am install-exec install-exec-am install-html \
+	install-html-am install-info install-info-am install-man \
+	install-moduleLTLIBRARIES install-pdf install-pdf-am \
+	install-pkginc_libHEADERS install-pkglibexecPROGRAMS \
+	install-pkglibexecSCRIPTS install-ps install-ps-am \
+	install-strip installcheck installcheck-am installdirs \
+	maintainer-clean maintainer-clean-generic mostlyclean \
+	mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
+	pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \
+	uninstall-doveadm_moduleLTLIBRARIES \
+	uninstall-moduleLTLIBRARIES uninstall-pkginc_libHEADERS \
+	uninstall-pkglibexecPROGRAMS uninstall-pkglibexecSCRIPTS
+
+.PRECIOUS: Makefile
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/plugins/fts/decode2text.sh b/src/plugins/fts/decode2text.sh
new file mode 100755
index 0000000..1c881ff
--- /dev/null
+++ b/src/plugins/fts/decode2text.sh
@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# Example attachment decoder script. The attachment comes from stdin, and
+# the script is expected to output UTF-8 data to stdout. (If the output isn't
+# UTF-8, everything except valid UTF-8 sequences are dropped from it.)
+
+# The attachment decoding is enabled by setting:
+#
+# plugin {
+#   fts_decoder = decode2text
+# }
+# service decode2text {
+#   executable = script /usr/local/libexec/dovecot/decode2text.sh
+#   user = dovecot
+#   unix_listener decode2text {
+#     mode = 0666
+#   }
+# }
+
+libexec_dir=`dirname $0`
+content_type=$1
+
+# The second parameter is the format's filename extension, which is used when
+# found from a filename of application/octet-stream. You can also add more
+# extensions by giving more parameters.
+formats='application/pdf pdf
+application/x-pdf pdf
+application/msword doc
+application/mspowerpoint ppt
+application/vnd.ms-powerpoint ppt
+application/ms-excel xls
+application/x-msexcel xls
+application/vnd.ms-excel xls
+application/vnd.openxmlformats-officedocument.wordprocessingml.document docx
+application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx
+application/vnd.openxmlformats-officedocument.presentationml.presentation pptx
+application/vnd.oasis.opendocument.text odt
+application/vnd.oasis.opendocument.spreadsheet ods
+application/vnd.oasis.opendocument.presentation odp
+'
+
+if [ "$content_type" = "" ]; then
+  echo "$formats"
+  exit 0
+fi
+
+fmt=`echo "$formats" | grep -w "^$content_type" | cut -d ' ' -f 2`
+if [ "$fmt" = "" ]; then
+  echo "Content-Type: $content_type not supported" >&2
+  exit 1
+fi
+
+# most decoders can't handle stdin directly, so write the attachment
+# to a temp file
+path=`mktemp`
+trap "rm -f $path" 0 1 2 3 14 15
+cat > $path
+
+xmlunzip() {
+  name=$1
+
+  tempdir=`mktemp -d`
+  if [ "$tempdir" = "" ]; then 
+    exit 1
+  fi
+  trap "rm -rf $path $tempdir" 0 1 2 3 14 15
+  cd $tempdir || exit 1
+  unzip -q "$path" 2>/dev/null || exit 0
+  find . -name "$name" -print0 | xargs -0 cat |
+    $libexec_dir/xml2text
+}
+
+wait_timeout() {
+  childpid=$!
+  trap "kill -9 $childpid; rm -f $path" 1 2 3 14 15
+  wait $childpid
+}
+
+LANG=en_US.UTF-8
+export LANG
+if [ $fmt = "pdf" ]; then
+  /usr/bin/pdftotext $path - 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "doc" ]; then
+  (/usr/bin/catdoc $path; true) 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "ppt" ]; then
+  (/usr/bin/catppt $path; true) 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "xls" ]; then
+  (/usr/bin/xls2csv $path; true) 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "odt" -o $fmt = "ods" -o $fmt = "odp" ]; then
+  xmlunzip "content.xml"
+elif [ $fmt = "docx" ]; then
+  xmlunzip "document.xml"
+elif [ $fmt = "xlsx" ]; then
+  xmlunzip "sharedStrings.xml"
+elif [ $fmt = "pptx" ]; then
+  xmlunzip "slide*.xml"
+else
+  echo "Buggy decoder script: $fmt not handled" >&2
+  exit 1
+fi
+exit 0
diff --git a/src/plugins/fts/doveadm-dump-fts-expunge-log.c b/src/plugins/fts/doveadm-dump-fts-expunge-log.c
new file mode 100644
index 0000000..7438bca
--- /dev/null
+++ b/src/plugins/fts/doveadm-dump-fts-expunge-log.c
@@ -0,0 +1,116 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "hex-binary.h"
+#include "guid.h"
+#include "doveadm-dump.h"
+#include "doveadm-fts.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+struct fts_expunge_log_record {
+	uint32_t checksum;
+	uint32_t record_size;
+	guid_128_t guid;
+};
+
+static int dump_record(int fd, buffer_t *buf)
+{
+	struct fts_expunge_log_record rec;
+	off_t offset;
+	void *data;
+	const uint32_t *expunges, *uids;
+	ssize_t ret;
+	size_t data_size;
+	unsigned int i, uids_count;
+
+	offset = lseek(fd, 0, SEEK_CUR);
+
+	ret = read(fd, &rec, sizeof(rec));
+	if (ret == 0)
+		return 0;
+
+	if (ret != sizeof(rec))
+		i_fatal("rec read() %d != %d", (int)ret, (int)sizeof(rec));
+
+	if (rec.record_size < sizeof(rec) + sizeof(uint32_t) ||
+	    rec.record_size > INT_MAX) {
+		i_fatal("Invalid record_size=%u at offset %"PRIuUOFF_T,
+			rec.record_size, offset);
+	}
+	data_size = rec.record_size - sizeof(rec);
+	buffer_set_used_size(buf, 0);
+	data = buffer_append_space_unsafe(buf, data_size);
+	ret = read(fd, data, data_size);
+	if (ret != (ssize_t)data_size)
+		i_fatal("rec read() %d != %d", (int)ret, (int)data_size);
+
+	printf("#%"PRIuUOFF_T":\n", offset);
+	printf("  checksum  = %8x\n", rec.checksum);
+	printf("  size .... = %u\n", rec.record_size);
+	printf("  mailbox . = %s\n", guid_128_to_string(rec.guid));
+
+	expunges = CONST_PTR_OFFSET(data, data_size - sizeof(uint32_t));
+	printf("  expunges  = %u\n", *expunges);
+
+	printf("  uids .... = ");
+
+	uids = data;
+	uids_count = (rec.record_size - sizeof(rec) - sizeof(uint32_t)) /
+		sizeof(uint32_t);
+	for (i = 0; i < uids_count; i += 2) {
+		if (i != 0)
+			printf(",");
+		if (uids[i] == uids[i+1])
+			printf("%u", uids[i]);
+		else
+			printf("%u-%u", uids[i], uids[i+1]);
+	}
+	printf("\n");
+	return 1;
+}
+
+static void
+cmd_dump_fts_expunge_log(const char *path, const char *const *args ATTR_UNUSED)
+{
+	buffer_t *buf;
+	int fd, ret;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		i_fatal("open(%s) failed: %m", path);
+
+	buf = buffer_create_dynamic(default_pool, 1024);
+	do {
+		T_BEGIN {
+			ret = dump_record(fd, buf);
+		} T_END;
+	} while (ret > 0);
+	buffer_free(&buf);
+	i_close_fd(&fd);
+}
+
+static bool test_dump_fts_expunge_log(const char *path)
+{
+	const char *p;
+
+	if ((p = strrchr(path, '/')) != NULL)
+		p++;
+	else
+		p = path;
+	return strcmp(p, "dovecot-expunges.log") == 0;
+}
+
+static const struct doveadm_cmd_dump doveadm_cmd_dump_fts_expunge_log = {
+	"fts-expunge-log",
+	test_dump_fts_expunge_log,
+	cmd_dump_fts_expunge_log
+};
+
+void doveadm_dump_fts_expunge_log_init(void)
+{
+	doveadm_dump_register(&doveadm_cmd_dump_fts_expunge_log);
+}
diff --git a/src/plugins/fts/doveadm-fts.c b/src/plugins/fts/doveadm-fts.c
new file mode 100644
index 0000000..1b902a1
--- /dev/null
+++ b/src/plugins/fts/doveadm-fts.c
@@ -0,0 +1,470 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "imap-util.h"
+#include "mail-namespace.h"
+#include "mail-search.h"
+#include "mailbox-list-iter.h"
+#include "fts-tokenizer.h"
+#include "fts-filter.h"
+#include "fts-language.h"
+#include "fts-storage.h"
+#include "fts-search-args.h"
+#include "fts-user.h"
+#include "doveadm-print.h"
+#include "doveadm-mail.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-fts.h"
+
+const char *doveadm_fts_plugin_version = DOVECOT_ABI_VERSION;
+
+struct fts_tokenize_cmd_context {
+	struct doveadm_mail_cmd_context ctx;
+	const char *language;
+	const char *tokens;
+};
+
+static int
+cmd_search_box(struct doveadm_mail_cmd_context *ctx,
+	       const struct mailbox_info *info)
+{
+	struct mailbox *box;
+	struct fts_backend *backend;
+	struct fts_result result;
+	int ret = 0;
+
+	backend = fts_list_backend(info->ns->list);
+	if (backend == NULL) {
+		i_error("fts not enabled for %s", info->vname);
+		ctx->exit_code = EX_CONFIG;
+		return -1;
+	}
+
+	i_zero(&result);
+	i_array_init(&result.definite_uids, 16);
+	i_array_init(&result.maybe_uids, 16);
+	i_array_init(&result.scores, 16);
+
+	box = mailbox_alloc(info->ns->list, info->vname, 0);
+	if (fts_backend_lookup(backend, box, ctx->search_args->args,
+				      FTS_LOOKUP_FLAG_AND_ARGS, &result) < 0) {
+		i_error("fts lookup failed");
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
+		ret = -1;
+	} else {
+		printf("%s: ", info->vname);
+		if (array_count(&result.definite_uids) == 0)
+			printf("no results\n");
+		else T_BEGIN {
+			string_t *str = t_str_new(128);
+			imap_write_seq_range(str, &result.definite_uids);
+			printf("%s\n", str_c(str));
+		} T_END;
+		if (array_count(&result.maybe_uids) > 0) T_BEGIN {
+			string_t *str = t_str_new(128);
+			imap_write_seq_range(str, &result.maybe_uids);
+			printf(" - maybe: %s\n", str_c(str));
+		} T_END;
+		fts_backend_lookup_done(backend);
+	}
+	mailbox_free(&box);
+	array_free(&result.definite_uids);
+	array_free(&result.maybe_uids);
+	array_free(&result.scores);
+	return ret;
+}
+
+static int
+cmd_fts_lookup_run(struct doveadm_mail_cmd_context *ctx,
+		   struct mail_user *user)
+{
+	const enum mailbox_list_iter_flags iter_flags =
+		MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+		MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+	struct doveadm_mailbox_list_iter *iter;
+	const struct mailbox_info *info;
+	int ret = 0;
+
+	iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args,
+					      iter_flags);
+	while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+		if (cmd_search_box(ctx, info) < 0)
+			ret = -1;
+	} T_END;
+	if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+static void
+cmd_fts_lookup_init(struct doveadm_mail_cmd_context *ctx,
+		    const char *const args[])
+{
+	if (args[0] == NULL)
+		doveadm_mail_help_name("fts lookup");
+
+	ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_lookup_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_lookup_run;
+	ctx->v.init = cmd_fts_lookup_init;
+	return ctx;
+}
+
+static int
+cmd_fts_expand_run(struct doveadm_mail_cmd_context *ctx,
+		   struct mail_user *user)
+{
+	struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
+	struct mailbox *box;
+	struct fts_backend *backend;
+	string_t *str = t_str_new(128);
+
+	backend = fts_list_backend(ns->list);
+	if (backend == NULL) {
+		i_error("fts not enabled for INBOX");
+		ctx->exit_code = EX_CONFIG;
+		return -1;
+	}
+
+	box = mailbox_alloc(ns->list, "INBOX", 0);
+	mail_search_args_init(ctx->search_args, box, FALSE, NULL);
+
+	if (fts_search_args_expand(backend, ctx->search_args) < 0)
+		i_fatal("Couldn't expand search args");
+	mail_search_args_to_cmdline(str, ctx->search_args->args);
+	printf("%s\n", str_c(str));
+	mailbox_free(&box);
+	return 0;
+}
+
+static void
+cmd_fts_expand_init(struct doveadm_mail_cmd_context *ctx,
+		    const char *const args[])
+{
+	if (args[0] == NULL)
+		doveadm_mail_help_name("fts expand");
+
+	ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_expand_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_expand_run;
+	ctx->v.init = cmd_fts_expand_init;
+	return ctx;
+}
+
+static int
+cmd_fts_tokenize_run(struct doveadm_mail_cmd_context *_ctx,
+		     struct mail_user *user)
+{
+	struct fts_tokenize_cmd_context *ctx =
+		(struct fts_tokenize_cmd_context *)_ctx;
+	struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
+	struct fts_backend *backend;
+	struct fts_user_language *user_lang;
+	const struct fts_language *lang = NULL;
+	int ret, ret2;
+	bool final = FALSE;
+
+	backend = fts_list_backend(ns->list);
+	if (backend == NULL) {
+		i_error("fts not enabled for INBOX");
+		_ctx->exit_code = EX_CONFIG;
+		return -1;
+	}
+
+	if (ctx->language == NULL) {
+		struct fts_language_list *lang_list =
+			fts_user_get_language_list(user);
+		enum fts_language_result result;
+		const char *error;
+
+		result = fts_language_detect(lang_list,
+		    (const unsigned char *)ctx->tokens, strlen(ctx->tokens),
+                    &lang, &error);
+		if (lang == NULL)
+			lang = fts_language_list_get_first(lang_list);
+		switch (result) {
+		case FTS_LANGUAGE_RESULT_SHORT:
+			i_warning("Text too short, can't detect its language - assuming %s", lang->name);
+			break;
+		case FTS_LANGUAGE_RESULT_UNKNOWN:
+			i_warning("Can't detect its language - assuming %s", lang->name);
+			break;
+		case FTS_LANGUAGE_RESULT_OK:
+			break;
+		case FTS_LANGUAGE_RESULT_ERROR:
+			i_error("Language detection library initialization failed: %s", error);
+			_ctx->exit_code = EX_CONFIG;
+			return -1;
+		default:
+			i_unreached();
+		}
+	} else {
+		lang = fts_language_find(ctx->language);
+		if (lang == NULL) {
+			i_error("Unknown language: %s", ctx->language);
+			_ctx->exit_code = EX_USAGE;
+			return -1;
+		}
+	}
+	user_lang = fts_user_language_find(user, lang);
+	if (user_lang == NULL) {
+		i_error("Language not enabled for user: %s", ctx->language);
+		_ctx->exit_code = EX_USAGE;
+		return -1;
+	}
+
+	fts_tokenizer_reset(user_lang->index_tokenizer);
+	for (;;) {
+		const char *token, *error;
+
+		if (!final) {
+			ret = fts_tokenizer_next(user_lang->index_tokenizer,
+				(const unsigned char *)ctx->tokens, strlen(ctx->tokens),
+				&token, &error);
+		} else {
+			ret = fts_tokenizer_final(user_lang->index_tokenizer,
+						  &token, &error);
+		}
+		if (ret < 0)
+			break;
+		if (ret > 0 && user_lang->filter != NULL) {
+			ret2 = fts_filter_filter(user_lang->filter, &token, &error);
+			if (ret2 > 0)
+				doveadm_print(token);
+			else if (ret2 < 0)
+				i_error("Couldn't create indexable tokens: %s", error);
+		}
+		if (ret == 0) {
+			if (final)
+				break;
+			final = TRUE;
+		}
+	}
+	return 0;
+}
+
+static void
+cmd_fts_tokenize_init(struct doveadm_mail_cmd_context *_ctx,
+		      const char *const args[])
+{
+	struct fts_tokenize_cmd_context *ctx =
+		(struct fts_tokenize_cmd_context *)_ctx;
+
+	if (args[0] == NULL)
+		doveadm_mail_help_name("fts tokenize");
+
+	ctx->tokens = p_strdup(_ctx->pool, t_strarray_join(args, " "));
+
+	doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+	doveadm_print_header("token", "token", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+}
+
+static bool
+cmd_fts_tokenize_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+	struct fts_tokenize_cmd_context *ctx =
+		(struct fts_tokenize_cmd_context *)_ctx;
+
+	switch (c) {
+	case 'l':
+		ctx->language = p_strdup(_ctx->pool, optarg);
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_tokenize_alloc(void)
+{
+	struct fts_tokenize_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct fts_tokenize_cmd_context);
+	ctx->ctx.v.run = cmd_fts_tokenize_run;
+	ctx->ctx.v.init = cmd_fts_tokenize_init;
+	ctx->ctx.v.parse_arg = cmd_fts_tokenize_parse_arg;
+	ctx->ctx.getopt_args = "l";
+	return &ctx->ctx;
+}
+
+static int
+fts_namespace_find(struct mail_user *user, const char *ns_prefix,
+		   struct mail_namespace **ns_r)
+{
+	struct mail_namespace *ns;
+
+	if (ns_prefix == NULL)
+		ns = mail_namespace_find_inbox(user->namespaces);
+	else {
+		ns = mail_namespace_find_prefix(user->namespaces, ns_prefix);
+		if (ns == NULL) {
+			i_error("Namespace prefix not found: %s", ns_prefix);
+			return -1;
+		}
+	}
+
+	if (fts_list_backend(ns->list) == NULL) {
+		i_error("fts not enabled for user's namespace %s",
+			ns_prefix != NULL ? ns_prefix : "INBOX");
+		return -1;
+	}
+	*ns_r = ns;
+	return 0;
+}
+
+static int
+cmd_fts_optimize_run(struct doveadm_mail_cmd_context *ctx,
+		     struct mail_user *user)
+{
+	const char *ns_prefix = ctx->args[0];
+	struct mail_namespace *ns;
+	struct fts_backend *backend;
+
+	if (fts_namespace_find(user, ns_prefix, &ns) < 0) {
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND);
+		return -1;
+	}
+	backend = fts_list_backend(ns->list);
+	if (fts_backend_optimize(backend) < 0) {
+		i_error("fts optimize failed");
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+cmd_fts_optimize_init(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
+		      const char *const args[])
+{
+	if (str_array_length(args) > 1)
+		doveadm_mail_help_name("fts optimize");
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_optimize_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_optimize_run;
+	ctx->v.init = cmd_fts_optimize_init;
+	return ctx;
+}
+
+static int
+cmd_fts_rescan_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user)
+{
+	const char *ns_prefix = ctx->args[0];
+	struct mail_namespace *ns;
+	struct fts_backend *backend;
+
+	if (fts_namespace_find(user, ns_prefix, &ns) < 0) {
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND);
+		return -1;
+	}
+	backend = fts_list_backend(ns->list);
+	if (fts_backend_rescan(backend) < 0) {
+		i_error("fts rescan failed");
+		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+cmd_fts_rescan_init(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
+		    const char *const args[])
+{
+	if (str_array_length(args) > 1)
+		doveadm_mail_help_name("fts rescan");
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_fts_rescan_alloc(void)
+{
+	struct doveadm_mail_cmd_context *ctx;
+
+	ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+	ctx->v.run = cmd_fts_rescan_run;
+	ctx->v.init = cmd_fts_rescan_init;
+	return ctx;
+}
+
+static struct doveadm_cmd_ver2 fts_commands[] = {
+{
+	.name = "fts lookup",
+	.mail_cmd = cmd_fts_lookup_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts expand",
+	.mail_cmd = cmd_fts_expand_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts tokenize",
+	.mail_cmd = cmd_fts_tokenize_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<text>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('l', "language", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "text", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts optimize",
+	.mail_cmd = cmd_fts_optimize_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[<namespace>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "namespace", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "fts rescan",
+	.mail_cmd = cmd_fts_rescan_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[<namespace>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "namespace", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+};
+
+void doveadm_fts_plugin_init(struct module *module ATTR_UNUSED)
+{
+	unsigned int i;
+
+	for (i = 0; i < N_ELEMENTS(fts_commands); i++)
+		doveadm_cmd_register_ver2(&fts_commands[i]);
+	doveadm_dump_fts_expunge_log_init();
+}
+
+void doveadm_fts_plugin_deinit(void)
+{
+}
diff --git a/src/plugins/fts/doveadm-fts.h b/src/plugins/fts/doveadm-fts.h
new file mode 100644
index 0000000..d4307fe
--- /dev/null
+++ b/src/plugins/fts/doveadm-fts.h
@@ -0,0 +1,11 @@
+#ifndef DOVEADM_FTS_H
+#define DOVEADM_FTS_H
+
+struct module;
+
+void doveadm_dump_fts_expunge_log_init(void);
+
+void doveadm_fts_plugin_init(struct module *module);
+void doveadm_fts_plugin_deinit(void);
+
+#endif
diff --git a/src/plugins/fts/fts-api-private.h b/src/plugins/fts/fts-api-private.h
new file mode 100644
index 0000000..a070564
--- /dev/null
+++ b/src/plugins/fts/fts-api-private.h
@@ -0,0 +1,139 @@
+#ifndef FTS_API_PRIVATE_H
+#define FTS_API_PRIVATE_H
+
+#include "unichar.h"
+#include "fts-api.h"
+
+struct mail_user;
+struct mailbox_list;
+
+#define MAILBOX_GUID_HEX_LENGTH (GUID_128_SIZE*2)
+
+struct fts_backend_vfuncs {
+	struct fts_backend *(*alloc)(void);
+	int (*init)(struct fts_backend *backend, const char **error_r);
+	void (*deinit)(struct fts_backend *backend);
+
+	int (*get_last_uid)(struct fts_backend *backend, struct mailbox *box,
+			    uint32_t *last_uid_r);
+
+	struct fts_backend_update_context *
+		(*update_init)(struct fts_backend *backend);
+	int (*update_deinit)(struct fts_backend_update_context *ctx);
+
+	void (*update_set_mailbox)(struct fts_backend_update_context *ctx,
+				   struct mailbox *box);
+	void (*update_expunge)(struct fts_backend_update_context *ctx,
+			       uint32_t uid);
+
+	/* Start a build for specified key */
+	bool (*update_set_build_key)(struct fts_backend_update_context *ctx,
+				    const struct fts_backend_build_key *key);
+	/* Finish a build for specified key - guaranteed to be called */
+	void (*update_unset_build_key)(struct fts_backend_update_context *ctx);
+	/* Add data for current build key */
+	int (*update_build_more)(struct fts_backend_update_context *ctx,
+				 const unsigned char *data, size_t size);
+
+	int (*refresh)(struct fts_backend *backend);
+	int (*rescan)(struct fts_backend *backend);
+	int (*optimize)(struct fts_backend *backend);
+
+	bool (*can_lookup)(struct fts_backend *backend,
+			   const struct mail_search_arg *args);
+	int (*lookup)(struct fts_backend *backend, struct mailbox *box,
+		      struct mail_search_arg *args, enum fts_lookup_flags flags,
+		      struct fts_result *result);
+	int (*lookup_multi)(struct fts_backend *backend,
+			    struct mailbox *const boxes[],
+			    struct mail_search_arg *args,
+			    enum fts_lookup_flags flags,
+			    struct fts_multi_result *result);
+	void (*lookup_done)(struct fts_backend *backend);
+};
+
+enum fts_backend_flags {
+	/* Backend supports indexing binary MIME parts */
+	FTS_BACKEND_FLAG_BINARY_MIME_PARTS	= 0x01,
+	/* Send built text to backend normalized rather than
+	   preserving original case */
+	FTS_BACKEND_FLAG_NORMALIZE_INPUT	= 0x02,
+	/* Send only fully indexable words rather than randomly sized blocks */
+	FTS_BACKEND_FLAG_BUILD_FULL_WORDS	= 0x04,
+	/* Fuzzy search works */
+	FTS_BACKEND_FLAG_FUZZY_SEARCH		= 0x08,
+	/* Tokenize all the input. update_build_more() will be called a single
+	   directly indexable token at a time. Searching will modify the search
+	   args so that lookup() sees only tokens that can be directly
+	   searched. */
+	FTS_BACKEND_FLAG_TOKENIZED_INPUT	= 0x10
+};
+
+struct fts_header_filters {
+	pool_t pool;
+	ARRAY_TYPE(const_string) includes;
+	ARRAY_TYPE(const_string) excludes;
+	bool loaded:1;
+	bool exclude_is_default:1;
+};
+
+struct fts_backend {
+	const char *name;
+	enum fts_backend_flags flags;
+
+	struct fts_backend_vfuncs v;
+	struct mail_namespace *ns;
+	struct fts_header_filters header_filters;
+
+	bool updating:1;
+};
+
+struct fts_backend_update_context {
+	struct fts_backend *backend;
+	normalizer_func_t *normalizer;
+
+	struct mailbox *cur_box, *backend_box;
+
+	bool build_key_open:1;
+	bool failed:1;
+};
+
+struct fts_index_header {
+	uint32_t last_indexed_uid;
+
+	/* Checksum of settings. If the settings change, the index should
+	   be rebuilt. */
+	uint32_t settings_checksum;
+	uint32_t unused;
+};
+
+void fts_backend_register(const struct fts_backend *backend);
+void fts_backend_unregister(const char *name);
+
+bool fts_backend_default_can_lookup(struct fts_backend *backend,
+				    const struct mail_search_arg *args);
+
+void fts_filter_uids(ARRAY_TYPE(seq_range) *definite_dest,
+		     const ARRAY_TYPE(seq_range) *definite_filter,
+		     ARRAY_TYPE(seq_range) *maybe_dest,
+		     const ARRAY_TYPE(seq_range) *maybe_filter);
+
+/* Returns TRUE if ok, FALSE if no fts header */
+bool fts_index_get_header(struct mailbox *box, struct fts_index_header *hdr_r);
+int fts_index_set_header(struct mailbox *box,
+			 const struct fts_index_header *hdr);
+int ATTR_NOWARN_UNUSED_RESULT
+fts_index_set_last_uid(struct mailbox *box, uint32_t last_uid);
+int fts_backend_reset_last_uids(struct fts_backend *backend);
+int fts_index_have_compatible_settings(struct mailbox_list *list,
+				       uint32_t checksum);
+
+/* Returns TRUE if FTS backend should index the header for optimizing
+   separate lookups */
+bool fts_header_want_indexed(const char *hdr_name);
+/* Returns TRUE if header's values should be considered to have a language. */
+bool fts_header_has_language(const char *hdr_name);
+
+int fts_mailbox_get_guid(struct mailbox *box, const char **guid_r);
+
+#endif
diff --git a/src/plugins/fts/fts-api.c b/src/plugins/fts/fts-api.c
new file mode 100644
index 0000000..a6ea716
--- /dev/null
+++ b/src/plugins/fts/fts-api.c
@@ -0,0 +1,554 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hex-binary.h"
+#include "mail-index.h"
+#include "mail-namespace.h"
+#include "mail-storage-private.h"
+#include "mailbox-list-iter.h"
+#include "mail-search.h"
+#include "fts-api-private.h"
+
+struct event_category event_category_fts = {
+	.name = "fts",
+};
+
+static ARRAY(const struct fts_backend *) backends;
+
+void fts_backend_register(const struct fts_backend *backend)
+{
+	if (!array_is_created(&backends))
+		i_array_init(&backends, 4);
+	array_push_back(&backends, &backend);
+}
+
+void fts_backend_unregister(const char *name)
+{
+	const struct fts_backend *const *be;
+	unsigned int i, count;
+
+	be = array_get(&backends, &count);
+	for (i = 0; i < count; i++) {
+		if (strcmp(be[i]->name, name) == 0) {
+			array_delete(&backends, i, 1);
+			break;
+		}
+	}
+	if (i == count)
+		i_panic("fts_backend_unregister(%s): unknown backend", name);
+
+	if (count == 1)
+		array_free(&backends);
+}
+
+static const struct fts_backend *
+fts_backend_class_lookup(const char *backend_name)
+{
+	const struct fts_backend *const *be;
+	unsigned int i, count;
+
+	if (array_is_created(&backends)) {
+		be = array_get(&backends, &count);
+		for (i = 0; i < count; i++) {
+			if (strcmp(be[i]->name, backend_name) == 0)
+				return be[i];
+		}
+	}
+	return NULL;
+}
+
+static void
+fts_header_filters_init(struct fts_backend *backend)
+{
+	struct fts_header_filters *filters = &backend->header_filters;
+	pool_t pool = filters->pool = pool_alloconly_create(
+		MEMPOOL_GROWING"fts_header_filters", 256);
+
+	p_array_init(&filters->includes, pool, 8);
+	p_array_init(&filters->excludes, pool, 8);
+}
+
+static void
+fts_header_filters_deinit(struct fts_backend *backend)
+{
+	pool_unref(&backend->header_filters.pool);
+}
+
+int fts_backend_init(const char *backend_name, struct mail_namespace *ns,
+		     const char **error_r, struct fts_backend **backend_r)
+{
+	const struct fts_backend *be;
+	struct fts_backend *backend;
+
+	be = fts_backend_class_lookup(backend_name);
+	if (be == NULL) {
+		*error_r = "Unknown backend";
+		return -1;
+	}
+
+	backend = be->v.alloc();
+	backend->ns = ns;
+	if (backend->v.init(backend, error_r) < 0) {
+		i_free(backend);
+		return -1;
+	}
+
+	fts_header_filters_init(backend);
+	*backend_r = backend;
+	return 0;
+}
+
+void fts_backend_deinit(struct fts_backend **_backend)
+{
+	struct fts_backend *backend = *_backend;
+
+	fts_header_filters_deinit(backend);
+	*_backend = NULL;
+	backend->v.deinit(backend);
+}
+
+int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box,
+			     uint32_t *last_uid_r)
+{
+	struct fts_index_header hdr;
+
+	if (box->virtual_vfuncs != NULL) {
+		/* virtual mailboxes themselves don't have any indexes,
+		   so catch this call here */
+		if (!fts_index_get_header(box, &hdr))
+			*last_uid_r = 0;
+		else
+			*last_uid_r = hdr.last_indexed_uid;
+		return 0;
+	}
+
+	return backend->v.get_last_uid(backend, box, last_uid_r);
+}
+
+bool fts_backend_is_updating(struct fts_backend *backend)
+{
+	return backend->updating;
+}
+
+struct fts_backend_update_context *
+fts_backend_update_init(struct fts_backend *backend)
+{
+	struct fts_backend_update_context *ctx;
+
+	i_assert(!backend->updating);
+
+	backend->updating = TRUE;
+	ctx = backend->v.update_init(backend);
+	if ((backend->flags & FTS_BACKEND_FLAG_NORMALIZE_INPUT) != 0)
+		ctx->normalizer = backend->ns->user->default_normalizer;
+	return ctx;
+}
+
+static void fts_backend_set_cur_mailbox(struct fts_backend_update_context *ctx)
+{
+	fts_backend_update_unset_build_key(ctx);
+	if (ctx->backend_box != ctx->cur_box) {
+		ctx->backend->v.update_set_mailbox(ctx, ctx->cur_box);
+		ctx->backend_box = ctx->cur_box;
+	}
+}
+
+int fts_backend_update_deinit(struct fts_backend_update_context **_ctx)
+{
+	struct fts_backend_update_context *ctx = *_ctx;
+	struct fts_backend *backend = ctx->backend;
+	int ret;
+
+	*_ctx = NULL;
+
+	ctx->cur_box = NULL;
+	fts_backend_set_cur_mailbox(ctx);
+
+	ret = backend->v.update_deinit(ctx);
+	backend->updating = FALSE;
+	return ret;
+}
+
+void fts_backend_update_set_mailbox(struct fts_backend_update_context *ctx,
+				    struct mailbox *box)
+{
+	if (ctx->backend_box != NULL && box != ctx->backend_box) {
+		/* make sure we don't reference the backend box anymore */
+		ctx->backend->v.update_set_mailbox(ctx, NULL);
+		ctx->backend_box = NULL;
+	}
+	ctx->cur_box = box;
+}
+
+void fts_backend_update_expunge(struct fts_backend_update_context *ctx,
+				uint32_t uid)
+{
+	fts_backend_set_cur_mailbox(ctx);
+	ctx->backend->v.update_expunge(ctx, uid);
+}
+
+bool fts_backend_update_set_build_key(struct fts_backend_update_context *ctx,
+				      const struct fts_backend_build_key *key)
+{
+	fts_backend_set_cur_mailbox(ctx);
+
+	i_assert(ctx->cur_box != NULL);
+
+	if (!ctx->backend->v.update_set_build_key(ctx, key))
+		return FALSE;
+	ctx->build_key_open = TRUE;
+	return TRUE;
+}
+
+void fts_backend_update_unset_build_key(struct fts_backend_update_context *ctx)
+{
+	if (ctx->build_key_open) {
+		ctx->backend->v.update_unset_build_key(ctx);
+		ctx->build_key_open = FALSE;
+	}
+}
+
+int fts_backend_update_build_more(struct fts_backend_update_context *ctx,
+				  const unsigned char *data, size_t size)
+{
+	i_assert(ctx->build_key_open);
+
+	return ctx->backend->v.update_build_more(ctx, data, size);
+}
+
+int fts_backend_refresh(struct fts_backend *backend)
+{
+	return backend->v.refresh(backend);
+}
+
+int fts_backend_reset_last_uids(struct fts_backend *backend)
+{
+	struct mailbox_list_iterate_context *iter;
+	const struct mailbox_info *info;
+	struct mailbox *box;
+	int ret = 0;
+
+	iter = mailbox_list_iter_init(backend->ns->list, "*",
+				      MAILBOX_LIST_ITER_SKIP_ALIASES |
+				      MAILBOX_LIST_ITER_NO_AUTO_BOXES);
+	while ((info = mailbox_list_iter_next(iter)) != NULL) {
+		if ((info->flags &
+		     (MAILBOX_NONEXISTENT | MAILBOX_NOSELECT)) != 0)
+			continue;
+
+		box = mailbox_alloc(info->ns->list, info->vname, 0);
+		if (mailbox_open(box) == 0) {
+			if (fts_index_set_last_uid(box, 0) < 0)
+				ret = -1;
+		}
+		mailbox_free(&box);
+	}
+	if (mailbox_list_iter_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+int fts_backend_rescan(struct fts_backend *backend)
+{
+	struct mailbox *box;
+	bool virtual_storage;
+
+	box = mailbox_alloc(backend->ns->list, "", 0);
+	virtual_storage = box->virtual_vfuncs != NULL;
+	mailbox_free(&box);
+
+	if (virtual_storage) {
+		/* just reset the last-uids for a virtual storage. */
+		return fts_backend_reset_last_uids(backend);
+	}
+
+	return backend->v.rescan == NULL ? 0 :
+		backend->v.rescan(backend);
+}
+
+int fts_backend_optimize(struct fts_backend *backend)
+{
+	return backend->v.optimize == NULL ? 0 :
+		backend->v.optimize(backend);
+}
+
+static void
+fts_merge_maybies(ARRAY_TYPE(seq_range) *dest_maybe,
+		  const ARRAY_TYPE(seq_range) *dest_definite,
+		  const ARRAY_TYPE(seq_range) *src_maybe,
+		  const ARRAY_TYPE(seq_range) *src_definite)
+{
+	ARRAY_TYPE(seq_range) src_unwanted;
+	const struct seq_range *range;
+	struct seq_range new_range;
+	unsigned int i, count;
+	uint32_t seq;
+
+	/* add/leave to dest_maybe if at least one list has maybe,
+	   and no lists have none */
+
+	/* create unwanted sequences list from both sources */
+	t_array_init(&src_unwanted, 128);
+	new_range.seq1 = 0; new_range.seq2 = (uint32_t)-1;
+	array_push_back(&src_unwanted, &new_range);
+	seq_range_array_remove_seq_range(&src_unwanted, src_maybe);
+	seq_range_array_remove_seq_range(&src_unwanted, src_definite);
+
+	/* drop unwanted uids */
+	seq_range_array_remove_seq_range(dest_maybe, &src_unwanted);
+
+	/* add uids that are in dest_definite and src_maybe lists */
+	range = array_get(dest_definite, &count);
+	for (i = 0; i < count; i++) {
+		for (seq = range[i].seq1; seq <= range[i].seq2; seq++) {
+			if (seq_range_exists(src_maybe, seq))
+				seq_range_array_add(dest_maybe, seq);
+		}
+	}
+}
+
+void fts_filter_uids(ARRAY_TYPE(seq_range) *definite_dest,
+		     const ARRAY_TYPE(seq_range) *definite_filter,
+		     ARRAY_TYPE(seq_range) *maybe_dest,
+		     const ARRAY_TYPE(seq_range) *maybe_filter)
+{
+	T_BEGIN {
+		fts_merge_maybies(maybe_dest, definite_dest,
+				  maybe_filter, definite_filter);
+	} T_END;
+	/* keep only what exists in both lists. the rest is in
+	   maybies or not wanted */
+	seq_range_array_intersect(definite_dest, definite_filter);
+}
+
+bool fts_backend_default_can_lookup(struct fts_backend *backend,
+				    const struct mail_search_arg *args)
+{
+	for (; args != NULL; args = args->next) {
+		switch (args->type) {
+		case SEARCH_OR:
+		case SEARCH_SUB:
+		case SEARCH_INTHREAD:
+			if (fts_backend_default_can_lookup(backend,
+							   args->value.subargs))
+				return TRUE;
+			break;
+		case SEARCH_HEADER:
+		case SEARCH_HEADER_ADDRESS:
+		case SEARCH_HEADER_COMPRESS_LWSP:
+		case SEARCH_BODY:
+		case SEARCH_TEXT:
+			if (!args->no_fts)
+				return TRUE;
+			break;
+		default:
+			break;
+		}
+	}
+	return FALSE;
+}
+
+bool fts_backend_can_lookup(struct fts_backend *backend,
+			    const struct mail_search_arg *args)
+{
+	return backend->v.can_lookup(backend, args);
+}
+
+static int fts_score_map_sort(const struct fts_score_map *m1,
+			      const struct fts_score_map *m2)
+{
+	if (m1->uid < m2->uid)
+		return -1;
+	if (m1->uid > m2->uid)
+		return 1;
+	return 0;
+}
+
+int fts_backend_lookup(struct fts_backend *backend, struct mailbox *box,
+		       struct mail_search_arg *args,
+		       enum fts_lookup_flags flags,
+		       struct fts_result *result)
+{
+	array_clear(&result->definite_uids);
+	array_clear(&result->maybe_uids);
+	array_clear(&result->scores);
+
+	if (backend->v.lookup(backend, box, args, flags, result) < 0)
+		return -1;
+
+	if (!result->scores_sorted && array_is_created(&result->scores)) {
+		array_sort(&result->scores, fts_score_map_sort);
+		result->scores_sorted = TRUE;
+	}
+	return 0;
+}
+
+int fts_backend_lookup_multi(struct fts_backend *backend,
+			     struct mailbox *const boxes[],
+			     struct mail_search_arg *args,
+			     enum fts_lookup_flags flags,
+			     struct fts_multi_result *result)
+{
+	unsigned int i;
+
+	i_assert(boxes[0] != NULL);
+
+	if (backend->v.lookup_multi != NULL) {
+		if (backend->v.lookup_multi(backend, boxes, args,
+					    flags, result) < 0)
+			return -1;
+		if (result->box_results == NULL) {
+			result->box_results = p_new(result->pool,
+						    struct fts_result, 1);
+		}
+		return 0;
+	}
+
+	for (i = 0; boxes[i] != NULL; i++) ;
+	result->box_results = p_new(result->pool, struct fts_result, i+1);
+
+	for (i = 0; boxes[i] != NULL; i++) {
+		struct fts_result *box_result = &result->box_results[i];
+
+		p_array_init(&box_result->definite_uids, result->pool, 32);
+		p_array_init(&box_result->maybe_uids, result->pool, 32);
+		p_array_init(&box_result->scores, result->pool, 32);
+		if (backend->v.lookup(backend, boxes[i], args,
+				      flags, box_result) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+void fts_backend_lookup_done(struct fts_backend *backend)
+{
+	if (backend->v.lookup_done != NULL)
+		backend->v.lookup_done(backend);
+}
+
+static uint32_t fts_index_get_ext_id(struct mailbox *box)
+{
+	return mail_index_ext_register(box->index, "fts",
+				       sizeof(struct fts_index_header),
+				       0, 0);
+}
+
+bool fts_index_get_header(struct mailbox *box, struct fts_index_header *hdr_r)
+{
+	struct mail_index_view *view;
+	const void *data;
+	size_t data_size;
+	bool ret;
+
+	mail_index_refresh(box->index);
+	view = mail_index_view_open(box->index);
+	mail_index_get_header_ext(view, fts_index_get_ext_id(box),
+				  &data, &data_size);
+	if (data_size < sizeof(*hdr_r)) {
+		i_zero(hdr_r);
+		ret = FALSE;
+	} else {
+		memcpy(hdr_r, data, sizeof(*hdr_r));
+		ret = TRUE;
+	}
+	mail_index_view_close(&view);
+	return ret;
+}
+
+int fts_index_set_header(struct mailbox *box,
+			 const struct fts_index_header *hdr)
+{
+	struct mail_index_transaction *trans;
+	uint32_t ext_id = fts_index_get_ext_id(box);
+
+	trans = mail_index_transaction_begin(box->view, 0);
+	mail_index_update_header_ext(trans, ext_id, 0, hdr, sizeof(*hdr));
+	return mail_index_transaction_commit(&trans);
+}
+
+int fts_index_set_last_uid(struct mailbox *box, uint32_t last_uid)
+{
+	struct fts_index_header hdr;
+
+	(void)fts_index_get_header(box, &hdr);
+	hdr.last_indexed_uid = last_uid;
+	return fts_index_set_header(box, &hdr);
+}
+
+int fts_index_have_compatible_settings(struct mailbox_list *list,
+				       uint32_t checksum)
+{
+	struct mail_namespace *ns = mailbox_list_get_namespace(list);
+	struct mailbox *box;
+	struct fts_index_header hdr;
+	const char *vname;
+	size_t len;
+	int ret;
+
+	if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0)
+		vname = "INBOX";
+	else {
+		len = strlen(ns->prefix);
+		if (len > 0 && ns->prefix[len-1] == mail_namespace_get_sep(ns))
+			len--;
+		vname = t_strndup(ns->prefix, len);
+	}
+
+	box = mailbox_alloc(list, vname, 0);
+	if (mailbox_sync(box, (enum mailbox_sync_flags)0) < 0) {
+		i_error("fts: Failed to sync mailbox %s: %s", vname,
+			mailbox_get_last_internal_error(box, NULL));
+		ret = -1;
+	} else {
+		ret = fts_index_get_header(box, &hdr) &&
+			hdr.settings_checksum == checksum ? 1 : 0;
+	}
+	mailbox_free(&box);
+	return ret;
+}
+
+static const char *indexed_headers[] = {
+	"From", "To", "Cc", "Bcc", "Subject"
+};
+
+bool fts_header_want_indexed(const char *hdr_name)
+{
+	unsigned int i;
+
+	for (i = 0; i < N_ELEMENTS(indexed_headers); i++) {
+		if (strcasecmp(hdr_name, indexed_headers[i]) == 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+bool fts_header_has_language(const char *hdr_name)
+{
+	/* FIXME: should email address headers be detected as different
+	   languages? That mainly contains people's names.. */
+	/*if (message_header_is_address(hdr_name))
+		return TRUE;*/
+
+	/* Subject definitely contains language-specific data that can be
+	   detected. Comment and Keywords headers also could contain, although
+	   just about nobody uses those headers.
+
+	   For now we assume that other headers contain non-language specific
+	   data that we don't want to filter in special ways. For example
+	   it is good to be able to search for Message-IDs. */
+	return strcasecmp(hdr_name, "Subject") == 0 ||
+		strcasecmp(hdr_name, "Comments") == 0 ||
+		strcasecmp(hdr_name, "Keywords") == 0;
+}
+
+int fts_mailbox_get_guid(struct mailbox *box, const char **guid_r)
+{
+	struct mailbox_metadata metadata;
+
+	if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0)
+		return -1;
+
+	*guid_r = guid_128_to_string(metadata.guid);
+	return 0;
+}
diff --git a/src/plugins/fts/fts-api.h b/src/plugins/fts/fts-api.h
new file mode 100644
index 0000000..11a331f
--- /dev/null
+++ b/src/plugins/fts/fts-api.h
@@ -0,0 +1,173 @@
+#ifndef FTS_API_H
+#define FTS_API_H
+
+struct mail;
+struct mailbox;
+struct mail_namespace;
+struct mail_search_arg;
+
+struct fts_backend;
+
+#include "seq-range-array.h"
+
+enum fts_lookup_flags {
+	/* Specifies if the args should be ANDed or ORed together. */
+	FTS_LOOKUP_FLAG_AND_ARGS	= 0x01,
+	/* Require exact matching for non-fuzzy search args by returning all
+	   such matches as maybe_uids instead of definite_uids */
+	FTS_LOOKUP_FLAG_NO_AUTO_FUZZY	= 0x02
+};
+
+enum fts_backend_build_key_type {
+	/* Header */
+	FTS_BACKEND_BUILD_KEY_HDR,
+	/* MIME part header */
+	FTS_BACKEND_BUILD_KEY_MIME_HDR,
+	/* MIME body part */
+	FTS_BACKEND_BUILD_KEY_BODY_PART,
+	/* Binary MIME body part, if backend supports binary data */
+	FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY
+};
+
+struct fts_backend_build_key {
+	uint32_t uid;
+	enum fts_backend_build_key_type type;
+	struct message_part *part;
+
+	/* for _KEY_HDR: */
+	const char *hdr_name;
+
+	/* for _KEY_BODY_PART and _KEY_BODY_PART_BINARY: */
+
+	/* Contains a valid parsed "type/subtype" string. For messages without
+	   (valid) Content-Type: header, it's set to "text/plain". */
+	const char *body_content_type;
+	/* Content-Disposition: header without parsing/validation if it exists,
+	   otherwise NULL. */
+	const char *body_content_disposition;
+};
+
+struct fts_score_map {
+	uint32_t uid;
+	float score;
+};
+ARRAY_DEFINE_TYPE(fts_score_map, struct fts_score_map);
+
+/* the structure is meant to be implemented by plugins that want to carry
+   some state over from a call to next ones within an fts_search_context
+   session.
+
+   The pointer to this structure is initially granted to be NULL and it
+   remains such unless the plugin itself activates it.
+
+   Any memory management for the pointer and its contents is expected to
+   be performed by the plugin itself, possibly but not necessarily using
+   the result pool propagated to plugin call by struct fts_result.pool and
+   struct fts_multi_result.pool. */
+
+struct fts_search_state;
+
+struct fts_result {
+	pool_t pool;
+	struct fts_search_state *search_state;
+
+	struct mailbox *box;
+
+	ARRAY_TYPE(seq_range) definite_uids;
+	/* The maybe_uids is useful with backends that can only filter out
+	   messages, but can't definitively say if the search matched a
+	   message. */
+	ARRAY_TYPE(seq_range) maybe_uids;
+	ARRAY_TYPE(fts_score_map) scores;
+	bool scores_sorted;
+};
+
+struct fts_multi_result {
+	pool_t pool;
+	struct fts_search_state *search_state;
+
+	/* box=NULL-terminated array of mailboxes and matching UIDs,
+	   all allocated from the given pool. */
+	struct fts_result *box_results;
+};
+
+extern struct event_category event_category_fts;
+
+int fts_backend_init(const char *backend_name, struct mail_namespace *ns,
+		     const char **error_r, struct fts_backend **backend_r);
+void fts_backend_deinit(struct fts_backend **backend);
+
+/* Get the last_uid for the mailbox. */
+int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box,
+			     uint32_t *last_uid_r);
+
+/* Returns TRUE if there exists an update context. */
+bool fts_backend_is_updating(struct fts_backend *backend);
+
+/* Start an index update. */
+struct fts_backend_update_context *
+fts_backend_update_init(struct fts_backend *backend);
+/* Finish an index update. Returns 0 if ok, -1 if some updates failed.
+   If updates failed, the index is in unspecified state. */
+int fts_backend_update_deinit(struct fts_backend_update_context **ctx);
+
+/* Switch to updating the specified mailbox. box may also be set to NULL to
+   make sure the previous mailbox won't tried to be accessed anymore. */
+void fts_backend_update_set_mailbox(struct fts_backend_update_context *ctx,
+				    struct mailbox *box);
+/* Expunge the specified mail. */
+void fts_backend_update_expunge(struct fts_backend_update_context *ctx,
+				uint32_t uid);
+
+/* Switch to building index for specified key. If backend doesn't want to
+   index this key, it can return FALSE and caller will skip to next key. */
+bool fts_backend_update_set_build_key(struct fts_backend_update_context *ctx,
+				      const struct fts_backend_build_key *key);
+/* Make sure that if _build_more() is called, we'll assert-crash. */
+void fts_backend_update_unset_build_key(struct fts_backend_update_context *ctx);
+/* Add more content to the index for the currently specified build key.
+   Non-BODY_PART_BINARY data must contain only full valid UTF-8 characters,
+   but it doesn't need to be NUL-terminated. size contains the data size in
+   bytes, not characters. This function may be called many times and the data
+   block sizes may be small. Backend returns 0 if ok, -1 if build should be
+   aborted. */
+int fts_backend_update_build_more(struct fts_backend_update_context *ctx,
+				  const unsigned char *data, size_t size);
+
+/* Refresh index to make sure we see latest changes from lookups.
+   Returns 0 if ok, -1 if error. */
+int fts_backend_refresh(struct fts_backend *backend);
+/* Go through the entire index and make sure all mails are indexed,
+   and delete any extra mails in the index. */
+int fts_backend_rescan(struct fts_backend *backend);
+/* Optimize the index. This can be a somewhat heavy operation. */
+int fts_backend_optimize(struct fts_backend *backend);
+
+/* Returns TRUE if fts_backend_lookup() should even be tried for the
+   given args. */
+bool fts_backend_can_lookup(struct fts_backend *backend,
+			    const struct mail_search_arg *args);
+/* Do a FTS lookup for the given search args. Backends can support different
+   kinds of search arguments, so match_always=TRUE must be set to all search
+   args that were actually used to produce the search results. The other args
+   are handled by the regular search code. The backends MUST ignore all args
+   that have subargs (SEARCH_OR, SEARCH_SUB), since they are looked up
+   separately.
+
+   The arrays in result must be initialized by caller. */
+int fts_backend_lookup(struct fts_backend *backend, struct mailbox *box,
+		       struct mail_search_arg *args,
+		       enum fts_lookup_flags flags,
+		       struct fts_result *result);
+
+/* Search from multiple mailboxes. result->pool must be initialized. */
+int fts_backend_lookup_multi(struct fts_backend *backend,
+			     struct mailbox *const boxes[],
+			     struct mail_search_arg *args,
+			     enum fts_lookup_flags flags,
+			     struct fts_multi_result *result);
+/* Called after the lookups are done. The next lookup will be preceded by a
+   refresh. */
+void fts_backend_lookup_done(struct fts_backend *backend);
+
+#endif
diff --git a/src/plugins/fts/fts-build-mail.c b/src/plugins/fts/fts-build-mail.c
new file mode 100644
index 0000000..73d4f4b
--- /dev/null
+++ b/src/plugins/fts/fts-build-mail.c
@@ -0,0 +1,719 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "buffer.h"
+#include "str.h"
+#include "rfc822-parser.h"
+#include "message-address.h"
+#include "message-parser.h"
+#include "message-decoder.h"
+#include "mail-storage.h"
+#include "index-mail.h"
+#include "fts-parser.h"
+#include "fts-user.h"
+#include "fts-language.h"
+#include "fts-tokenizer.h"
+#include "fts-filter.h"
+#include "fts-api-private.h"
+#include "fts-build-mail.h"
+
+/* there are other characters as well, but this doesn't have to be exact */
+#define IS_WORD_WHITESPACE(c) \
+	((c) == ' ' || (c) == '\t' || (c) == '\n')
+/* if we see a word larger than this, just go ahead and split it from
+   wherever */
+#define MAX_WORD_SIZE 1024
+
+struct fts_mail_build_context {
+	struct mail *mail;
+	struct fts_backend_update_context *update_ctx;
+
+	char *content_type, *content_disposition;
+	struct fts_parser *body_parser;
+
+	buffer_t *word_buf, *pending_input;
+	struct fts_user_language *cur_user_lang;
+};
+
+static int fts_build_data(struct fts_mail_build_context *ctx,
+			  const unsigned char *data, size_t size, bool last);
+
+static void fts_build_parse_content_type(struct fts_mail_build_context *ctx,
+					 const struct message_header_line *hdr)
+{
+	struct rfc822_parser_context parser;
+	string_t *content_type;
+
+	if (ctx->content_type != NULL)
+		return;
+
+	rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL);
+	rfc822_skip_lwsp(&parser);
+
+	T_BEGIN {
+		content_type = t_str_new(64);
+		(void)rfc822_parse_content_type(&parser, content_type);
+		ctx->content_type = str_lcase(i_strdup(str_c(content_type)));
+	} T_END;
+	rfc822_parser_deinit(&parser);
+}
+
+static void
+fts_build_parse_content_disposition(struct fts_mail_build_context *ctx,
+				    const struct message_header_line *hdr)
+{
+	/* just pass it as-is to backend. */
+	i_free(ctx->content_disposition);
+	ctx->content_disposition =
+		i_strndup(hdr->full_value, hdr->full_value_len);
+}
+
+static void fts_parse_mail_header(struct fts_mail_build_context *ctx,
+				  const struct message_block *raw_block)
+{
+	const struct message_header_line *hdr = raw_block->hdr;
+
+	if (strcasecmp(hdr->name, "Content-Type") == 0)
+		fts_build_parse_content_type(ctx, hdr);
+	else if (strcasecmp(hdr->name, "Content-Disposition") == 0)
+		fts_build_parse_content_disposition(ctx, hdr);
+}
+
+static int
+fts_build_unstructured_header(struct fts_mail_build_context *ctx,
+			      const struct message_header_line *hdr)
+{
+	const unsigned char *data = hdr->full_value;
+	unsigned char *buf = NULL;
+	unsigned int i;
+	int ret;
+
+	/* @UNSAFE: if there are any NULs, replace them with spaces */
+	for (i = 0; i < hdr->full_value_len; i++) {
+		if (hdr->full_value[i] == '\0') {
+			if (buf == NULL) {
+				buf = i_memdup(hdr->full_value,
+					       hdr->full_value_len);
+				data = buf;
+			}
+			buf[i] = ' ';
+		}
+	}
+	ret = fts_build_data(ctx, data, hdr->full_value_len, TRUE);
+	i_free(buf);
+	return ret;
+}
+
+static void fts_mail_build_ctx_set_lang(struct fts_mail_build_context *ctx,
+					struct fts_user_language *user_lang)
+{
+	i_assert(user_lang != NULL);
+
+	ctx->cur_user_lang = user_lang;
+	/* reset tokenizer between fields - just to be sure no state
+	   leaks between fields (especially if previous indexing had
+	   failed) */
+	fts_tokenizer_reset(user_lang->index_tokenizer);
+}
+
+static void
+fts_build_tokenized_hdr_update_lang(struct fts_mail_build_context *ctx,
+				    const struct message_header_line *hdr)
+{
+	/* Headers that don't contain any human language will only be
+	   translated to lowercase - no stemming or other filtering. There's
+	   unfortunately no pefect way of detecting which headers contain
+	   human languages, so we check with fts_header_has_language if the
+	   header is something that's supposed to containing human text. */
+	if (fts_header_has_language(hdr->name))
+		ctx->cur_user_lang = NULL;
+	else {
+		fts_mail_build_ctx_set_lang(ctx,
+			fts_user_get_data_lang(ctx->update_ctx->backend->ns->user));
+	}
+}
+
+static int fts_build_mail_header(struct fts_mail_build_context *ctx,
+				 const struct message_block *block)
+{
+	const struct message_header_line *hdr = block->hdr;
+	struct fts_backend_build_key key;
+	int ret;
+
+	if (hdr->eoh)
+		return 0;
+
+	/* hdr->full_value is always set because we get the block from
+	   message_decoder */
+	i_zero(&key);
+	key.uid = ctx->mail->uid;
+	key.type = block->part->physical_pos == 0 ?
+		FTS_BACKEND_BUILD_KEY_HDR : FTS_BACKEND_BUILD_KEY_MIME_HDR;
+	key.part = block->part;
+	key.hdr_name = hdr->name;
+
+	if ((ctx->update_ctx->backend->flags &
+	     FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0)
+		fts_build_tokenized_hdr_update_lang(ctx, hdr);
+
+	if (!fts_backend_update_set_build_key(ctx->update_ctx, &key))
+		return 0;
+
+	if (!message_header_is_address(hdr->name)) {
+		/* regular unstructured header */
+		ret = fts_build_unstructured_header(ctx, hdr);
+	} else T_BEGIN {
+		/* message address. normalize it to give better
+		   search results. */
+		struct message_address *addr;
+		string_t *str;
+
+		addr = message_address_parse(pool_datastack_create(),
+					     hdr->full_value,
+					     hdr->full_value_len,
+					     UINT_MAX, 0);
+		str = t_str_new(hdr->full_value_len);
+		message_address_write(str, addr);
+
+		ret = fts_build_data(ctx, str_data(str), str_len(str), TRUE);
+	} T_END;
+
+	if ((ctx->update_ctx->backend->flags &
+	     FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) {
+		/* index the header name itself using data-language. */
+		struct fts_user_language *prev_lang = ctx->cur_user_lang;
+
+		fts_mail_build_ctx_set_lang(ctx,
+			fts_user_get_data_lang(ctx->update_ctx->backend->ns->user));
+		key.hdr_name = "";
+		if (fts_backend_update_set_build_key(ctx->update_ctx, &key)) {
+			if (fts_build_data(ctx, (const void *)hdr->name,
+					   strlen(hdr->name), TRUE) < 0)
+				ret = -1;
+		}
+		fts_mail_build_ctx_set_lang(ctx, prev_lang);
+	}
+	return ret;
+}
+
+static bool
+fts_build_body_begin(struct fts_mail_build_context *ctx,
+		     struct message_part *part, bool *binary_body_r)
+{
+	struct mail_storage *storage;
+	struct fts_parser_context parser_context;
+	struct fts_backend_build_key key;
+
+	i_assert(ctx->body_parser == NULL);
+
+	*binary_body_r = FALSE;
+	i_zero(&key);
+	key.uid = ctx->mail->uid;
+	key.part = part;
+
+	i_zero(&parser_context);
+	parser_context.content_type = ctx->content_type != NULL ?
+		ctx->content_type : "text/plain";
+	if (str_begins(parser_context.content_type, "multipart/")) {
+		/* multiparts are never indexed, only their contents */
+		return FALSE;
+	}
+	storage = mailbox_get_storage(ctx->mail->box);
+	parser_context.user = mail_storage_get_user(storage);
+	parser_context.content_disposition = ctx->content_disposition;
+
+	if (fts_parser_init(&parser_context, &ctx->body_parser)) {
+		/* extract text using the the returned parser */
+		*binary_body_r = TRUE;
+		key.type = FTS_BACKEND_BUILD_KEY_BODY_PART;
+	} else if (str_begins(parser_context.content_type, "text/") ||
+		   str_begins(parser_context.content_type, "message/")) {
+		/* text body parts */
+		key.type = FTS_BACKEND_BUILD_KEY_BODY_PART;
+		ctx->body_parser = fts_parser_text_init();
+	} else {
+		/* possibly binary */
+		if ((ctx->update_ctx->backend->flags &
+		     FTS_BACKEND_FLAG_BINARY_MIME_PARTS) == 0)
+			return FALSE;
+		*binary_body_r = TRUE;
+		key.type = FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY;
+	}
+	key.body_content_type = parser_context.content_type;
+	key.body_content_disposition = ctx->content_disposition;
+	ctx->cur_user_lang = NULL;
+	if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) {
+		if (ctx->body_parser != NULL)
+			(void)fts_parser_deinit(&ctx->body_parser, NULL);
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static int
+fts_build_add_tokens_with_filter(struct fts_mail_build_context *ctx,
+				 const unsigned char *data, size_t size)
+{
+	struct fts_tokenizer *tokenizer = ctx->cur_user_lang->index_tokenizer;
+	struct fts_filter *filter = ctx->cur_user_lang->filter;
+	const char *token, *error;
+	int ret = 1, ret2;
+
+	while (ret > 0) T_BEGIN {
+		ret = ret2 = fts_tokenizer_next(tokenizer, data, size, &token, &error);
+		if (ret2 > 0 && filter != NULL)
+			ret2 = fts_filter_filter(filter, &token, &error);
+		if (ret2 < 0) {
+			mail_set_critical(ctx->mail,
+				"fts: Couldn't create indexable tokens: %s",
+				error);
+		}
+		if (ret2 > 0) {
+			if (fts_backend_update_build_more(ctx->update_ctx,
+							  (const void *)token,
+							  strlen(token)) < 0) {
+				mail_storage_set_internal_error(ctx->mail->box->storage);
+				ret = -1;
+			}
+		}
+	} T_END;
+	return ret;
+}
+
+static int
+fts_detect_language(struct fts_mail_build_context *ctx,
+		    const unsigned char *data, size_t size, bool last,
+		    const struct fts_language **lang_r)
+{
+	struct mail_user *user = ctx->update_ctx->backend->ns->user;
+	struct fts_language_list *lang_list = fts_user_get_language_list(user);
+	const struct fts_language *lang;
+	const char *error;
+
+	switch (fts_language_detect(lang_list, data, size, &lang, &error)) {
+	case FTS_LANGUAGE_RESULT_SHORT:
+		/* save the input so far and try again later */
+		buffer_append(ctx->pending_input, data, size);
+		if (last) {
+			/* we've run out of data. use the default language. */
+			*lang_r = fts_language_list_get_first(lang_list);
+			return 1;
+		}
+		return 0;
+	case FTS_LANGUAGE_RESULT_UNKNOWN:
+		/* use the default language */
+		*lang_r = fts_language_list_get_first(lang_list);
+		return 1;
+	case FTS_LANGUAGE_RESULT_OK:
+		*lang_r = lang;
+		return 1;
+	case FTS_LANGUAGE_RESULT_ERROR:
+		/* internal language detection library failure
+		   (e.g. invalid config). don't index anything. */
+		mail_set_critical(ctx->mail,
+			"Language detection library initialization failed: %s",
+			error);
+		return -1;
+	default:
+		i_unreached();
+	}
+}
+
+static int
+fts_build_tokenized(struct fts_mail_build_context *ctx,
+		    const unsigned char *data, size_t size, bool last)
+{
+	struct mail_user *user = ctx->update_ctx->backend->ns->user;
+	const struct fts_language *lang;
+	int ret;
+
+	if (ctx->cur_user_lang != NULL) {
+		/* we already have a language */
+	} else if ((ret = fts_detect_language(ctx, data, size, last, &lang)) < 0) {
+		return -1;
+	} else if (ret == 0) {
+		/* wait for more data */
+		return 0;
+	} else {
+		fts_mail_build_ctx_set_lang(ctx, fts_user_language_find(user, lang));
+
+		if (ctx->pending_input->used > 0) {
+			if (fts_build_add_tokens_with_filter(ctx,
+					ctx->pending_input->data,
+					ctx->pending_input->used) < 0)
+				return -1;
+			buffer_set_used_size(ctx->pending_input, 0);
+		}
+	}
+	if (fts_build_add_tokens_with_filter(ctx, data, size) < 0)
+		return -1;
+	if (last) {
+		if (fts_build_add_tokens_with_filter(ctx, NULL, 0) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int
+fts_build_full_words(struct fts_mail_build_context *ctx,
+		     const unsigned char *data, size_t size, bool last)
+{
+	size_t i;
+
+	/* we'll need to send only full words to the backend */
+
+	if (ctx->word_buf != NULL && ctx->word_buf->used > 0) {
+		/* continuing previous word */
+		for (i = 0; i < size; i++) {
+			if (IS_WORD_WHITESPACE(data[i]))
+				break;
+		}
+		buffer_append(ctx->word_buf, data, i);
+		data += i;
+		size -= i;
+		if (size == 0 && ctx->word_buf->used < MAX_WORD_SIZE && !last) {
+			/* word is still not finished */
+			return 0;
+		}
+		/* we have a full word, index it */
+		if (fts_backend_update_build_more(ctx->update_ctx,
+						  ctx->word_buf->data,
+						  ctx->word_buf->used) < 0) {
+			mail_storage_set_internal_error(ctx->mail->box->storage);
+			return -1;
+		}
+		buffer_set_used_size(ctx->word_buf, 0);
+	}
+
+	/* find the boundary for last word */
+	if (last)
+		i = size;
+	else {
+		for (i = size; i > 0; i--) {
+			if (IS_WORD_WHITESPACE(data[i-1]))
+				break;
+		}
+	}
+
+	if (fts_backend_update_build_more(ctx->update_ctx, data, i) < 0) {
+		mail_storage_set_internal_error(ctx->mail->box->storage);
+		return -1;
+	}
+
+	if (i < size) {
+		if (ctx->word_buf == NULL) {
+			ctx->word_buf =
+				buffer_create_dynamic(default_pool, 128);
+		}
+		buffer_append(ctx->word_buf, data + i, size - i);
+	}
+	return 0;
+}
+
+static int fts_build_data(struct fts_mail_build_context *ctx,
+			  const unsigned char *data, size_t size, bool last)
+{
+	if ((ctx->update_ctx->backend->flags &
+	     FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) {
+		return fts_build_tokenized(ctx, data, size, last);
+	} else if ((ctx->update_ctx->backend->flags &
+		    FTS_BACKEND_FLAG_BUILD_FULL_WORDS) != 0) {
+		return fts_build_full_words(ctx, data, size, last);
+	} else {
+		if (fts_backend_update_build_more(ctx->update_ctx, data, size) < 0) {
+			mail_storage_set_internal_error(ctx->mail->box->storage);
+			return -1;
+		}
+		return 0;
+	}
+}
+
+static int fts_build_body_block(struct fts_mail_build_context *ctx,
+				const struct message_block *block, bool last)
+{
+	i_assert(block->hdr == NULL);
+
+	return fts_build_data(ctx, block->data, block->size, last);
+}
+
+static int fts_body_parser_finish(struct fts_mail_build_context *ctx,
+				  const char **retriable_err_msg_r,
+				  bool *may_need_retry_r)
+{
+	struct message_block block;
+	const char *retriable_error;
+	int ret = 0;
+	int deinit_ret;
+	*may_need_retry_r = FALSE;
+
+	do {
+		i_zero(&block);
+		fts_parser_more(ctx->body_parser, &block);
+		if (fts_build_body_block(ctx, &block, FALSE) < 0) {
+			ret = -1;
+			break;
+		}
+	} while (block.size > 0);
+
+	deinit_ret = fts_parser_deinit(&ctx->body_parser, &retriable_error);
+	if (ret < 0) {
+		/* indexing already failed - we don't want to retry
+		   in any case */
+		return -1;
+	}
+
+	if (deinit_ret == 0) {
+		/* retry the parsing */
+		*may_need_retry_r = TRUE;
+		*retriable_err_msg_r = retriable_error;
+		return -1;
+	}
+	if (deinit_ret < 0) {
+		mail_storage_set_internal_error(ctx->mail->box->storage);
+		return -1;
+	}
+	return 0;
+}
+
+static void
+load_header_filter(const char *key, struct fts_backend *backend,
+		   ARRAY_TYPE(const_string) list, bool *matches_all_r)
+{
+	const char *str = mail_user_plugin_getenv(backend->ns->user, key);
+
+	*matches_all_r = FALSE;
+	if (str == NULL || *str == '\0')
+		return;
+
+	char **entries = p_strsplit_spaces(backend->header_filters.pool, str, " ");
+	for (char **entry = entries; *entry != NULL; ++entry) {
+		const char *value = str_lcase(*entry);
+		array_push_back(&list, &value);
+		if (*value == '*') {
+			*matches_all_r = TRUE;
+			break;
+		}
+	}
+	array_sort(&list, i_strcmp_p);
+}
+
+static struct fts_header_filters *
+load_header_filters(struct fts_backend *backend)
+{
+	struct fts_header_filters *filters = &backend->header_filters;
+	if (!filters->loaded) {
+		bool match_all;
+
+		/* match_all return ignored in includes */
+		load_header_filter("fts_header_includes", backend,
+				   filters->includes, &match_all);
+
+		load_header_filter("fts_header_excludes", backend,
+				   filters->excludes, &match_all);
+		filters->loaded = TRUE;
+		filters->exclude_is_default = match_all;
+	}
+	return filters;
+}
+
+/* This performs comparison between two strings, where the second one can end
+ * with the wildcard '*'. When the match reaches a '*' on the pitem side, zero
+ * (match) is returned regardles of the remaining characters.
+ *
+ * The function obeys the same lexicographic order as i_strcmp_p() and
+ * strcmp(), which is the reason for the casts to unsigned before comparing.
+ */
+static int ATTR_PURE
+header_prefix_cmp(const char *const *pkey, const char *const *pitem)
+{
+	const char *key = *pkey;
+	const char *item = *pitem;
+
+	while (*key == *item && *key != '\0') key++, item++;
+	return item[0] == '*' && item[1] == '\0' ? 0 :
+	       (unsigned char)*key - (unsigned char)*item;
+}
+
+static bool
+is_header_indexable(const char *header_name, struct fts_backend *backend)
+{
+	bool indexable;
+	T_BEGIN {
+		struct fts_header_filters *filters = load_header_filters(backend);
+		const char *hdr = t_str_lcase(header_name);
+
+		if (array_bsearch(&filters->includes, &hdr, header_prefix_cmp) != NULL)
+			indexable = TRUE;
+		else if (filters->exclude_is_default ||
+		         array_bsearch(&filters->excludes, &hdr, header_prefix_cmp) != NULL)
+			indexable = FALSE;
+		else
+			indexable = TRUE;
+	} T_END;
+	return indexable;
+}
+
+static int
+fts_build_mail_real(struct fts_backend_update_context *update_ctx,
+		    struct mail *mail,
+		    const char **retriable_err_msg_r,
+		    bool *may_need_retry_r)
+{
+	const struct message_parser_settings parser_set = {
+		.hdr_flags = MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE,
+	};
+	struct fts_mail_build_context ctx;
+	struct istream *input;
+	struct message_parser_ctx *parser;
+	struct message_decoder_context *decoder;
+	struct message_block raw_block, block;
+	struct message_part *prev_part, *parts;
+	bool skip_body = FALSE, body_part = FALSE, body_added = FALSE;
+	bool binary_body;
+	const char *error;
+	int ret;
+
+	*may_need_retry_r = FALSE;
+	if (mail_get_stream_because(mail, NULL, NULL, "fts indexing", &input) < 0) {
+		if (mail->expunged)
+			return 0;
+		mail_set_critical(mail, "Failed to read stream: %s",
+			mailbox_get_last_internal_error(mail->box, NULL));
+		return -1;
+	}
+
+	i_zero(&ctx);
+	ctx.update_ctx = update_ctx;
+	ctx.mail = mail;
+	if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0)
+		ctx.pending_input = buffer_create_dynamic(default_pool, 128);
+
+	prev_part = NULL;
+	parser = message_parser_init(pool_datastack_create(), input, &parser_set);
+
+	decoder = message_decoder_init(update_ctx->normalizer, 0);
+	for (;;) {
+		ret = message_parser_parse_next_block(parser, &raw_block);
+		i_assert(ret != 0);
+		if (ret < 0) {
+			if (input->stream_errno == 0)
+				ret = 0;
+			else {
+				mail_set_critical(mail, "read(%s) failed: %s",
+					i_stream_get_name(input),
+					i_stream_get_error(input));
+			}
+			break;
+		}
+
+		if (raw_block.part != prev_part) {
+			/* body part changed. we're now parsing the end of
+			   boundary, possibly followed by message epilogue */
+			if (ctx.body_parser != NULL) {
+				if (fts_body_parser_finish(&ctx, retriable_err_msg_r,
+							   may_need_retry_r) < 0) {
+					ret = -1;
+					break;
+				}
+			}
+			message_decoder_set_return_binary(decoder, FALSE);
+			fts_backend_update_unset_build_key(update_ctx);
+			prev_part = raw_block.part;
+			i_free_and_null(ctx.content_type);
+			i_free_and_null(ctx.content_disposition);
+
+			if (raw_block.size != 0) {
+				/* multipart. skip until beginning of next
+				   part's headers */
+				skip_body = TRUE;
+			}
+		}
+
+		if (raw_block.hdr != NULL) {
+			/* always handle headers */
+		} else if (raw_block.size == 0) {
+			/* end of headers */
+			skip_body = !fts_build_body_begin(&ctx, raw_block.part,
+							  &binary_body);
+			if (binary_body)
+				message_decoder_set_return_binary(decoder, TRUE);
+			body_part = TRUE;
+		} else {
+			if (skip_body)
+				continue;
+		}
+
+		if (!message_decoder_decode_next_block(decoder, &raw_block,
+						       &block))
+			continue;
+
+		if (block.hdr != NULL) {
+			fts_parse_mail_header(&ctx, &raw_block);
+			if (is_header_indexable(block.hdr->name, update_ctx->backend) &&
+			    fts_build_mail_header(&ctx, &block) < 0) {
+				ret = -1;
+				break;
+			}
+		} else if (block.size == 0) {
+			/* end of headers */
+		} else {
+			i_assert(body_part);
+			if (ctx.body_parser != NULL)
+				fts_parser_more(ctx.body_parser, &block);
+			if (fts_build_body_block(&ctx, &block, FALSE) < 0) {
+				ret = -1;
+				break;
+			}
+			body_added = TRUE;
+		}
+	}
+	if (ctx.body_parser != NULL) {
+		if (ret == 0)
+			ret = fts_body_parser_finish(&ctx, retriable_err_msg_r,
+						     may_need_retry_r);
+		else
+			(void)fts_parser_deinit(&ctx.body_parser, NULL);
+	}
+	if (ret == 0 && body_part && !skip_body && !body_added) {
+		/* make sure body is added even when it doesn't exist */
+		block.data = NULL; block.size = 0;
+		ret = fts_build_body_block(&ctx, &block, TRUE);
+	}
+	if (message_parser_deinit_from_parts(&parser, &parts, &error) < 0)
+		index_mail_set_message_parts_corrupted(mail, error);
+	message_decoder_deinit(&decoder);
+	i_free(ctx.content_type);
+	i_free(ctx.content_disposition);
+	buffer_free(&ctx.word_buf);
+	buffer_free(&ctx.pending_input);
+	return ret < 0 ? -1 : 1;
+}
+
+int fts_build_mail(struct fts_backend_update_context *update_ctx,
+		   struct mail *mail)
+{
+	int ret;
+	/* Number of attempts to be taken if retry is needed */
+	unsigned int attempts = 2;
+	const char *retriable_err_msg;
+	bool may_need_retry;
+
+	T_BEGIN {
+		while ((ret = fts_build_mail_real(update_ctx, mail,
+						  &retriable_err_msg,
+						  &may_need_retry)) < 0 &&
+		       may_need_retry) {
+			if (--attempts == 0) {
+				/* Log this as info instead of as error,
+				   because e.g. Tika doesn't differentiate
+				   between temporary errors and invalid
+				   document input. */
+				i_info("%s - ignoring", retriable_err_msg);
+				ret = 0;
+				break;
+			}
+		}
+	} T_END;
+	return ret;
+}
diff --git a/src/plugins/fts/fts-build-mail.h b/src/plugins/fts/fts-build-mail.h
new file mode 100644
index 0000000..aed4413
--- /dev/null
+++ b/src/plugins/fts/fts-build-mail.h
@@ -0,0 +1,9 @@
+#ifndef FTS_BUILD_MAIL_H
+#define FTS_BUILD_MAIL_H
+
+/* Build indexes for the given mail. Returns 0 on success, -1 on error.
+   The error is set to mail's storage. */
+int fts_build_mail(struct fts_backend_update_context *update_ctx,
+		   struct mail *mail);
+
+#endif
diff --git a/src/plugins/fts/fts-expunge-log.c b/src/plugins/fts/fts-expunge-log.c
new file mode 100644
index 0000000..d39ceea
--- /dev/null
+++ b/src/plugins/fts/fts-expunge-log.c
@@ -0,0 +1,617 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "crc32.h"
+#include "hash.h"
+#include "istream.h"
+#include "write-full.h"
+#include "seq-range-array.h"
+#include "mail-storage.h"
+#include "fts-expunge-log.h"
+
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+struct fts_expunge_log_record {
+	/* CRC32 of this entire record (except this checksum) */
+	uint32_t checksum;
+	/* Size of this entire record */
+	uint32_t record_size;
+
+	/* Mailbox GUID */
+	guid_128_t guid;
+	/* { uid1, uid2 } pairs */
+	/* uint32_t expunge_uid_ranges[]; */
+
+	/* Total number of messages expunged so far in this log */
+	/* uint32_t expunge_count; */
+};
+
+struct fts_expunge_log {
+	char *path;
+
+	int fd;
+	struct stat st;
+};
+
+struct fts_expunge_log_mailbox {
+	guid_128_t guid;
+	ARRAY_TYPE(seq_range) uids;
+	unsigned uids_count;
+};
+
+struct fts_expunge_log_append_ctx {
+	struct fts_expunge_log *log;
+	pool_t pool;
+
+	HASH_TABLE(uint8_t *, struct fts_expunge_log_mailbox *) mailboxes;
+	struct fts_expunge_log_mailbox *prev_mailbox;
+
+	bool failed;
+};
+
+struct fts_expunge_log_read_ctx {
+	struct fts_expunge_log *log;
+
+	struct istream *input;
+	buffer_t buffer;
+	struct fts_expunge_log_read_record read_rec;
+
+	bool failed;
+	bool corrupted;
+	bool unlink;
+};
+
+struct fts_expunge_log *fts_expunge_log_init(const char *path)
+{
+	struct fts_expunge_log *log;
+
+	log = i_new(struct fts_expunge_log, 1);
+	log->path = i_strdup(path);
+	log->fd = -1;
+	return log;
+}
+
+void fts_expunge_log_deinit(struct fts_expunge_log **_log)
+{
+	struct fts_expunge_log *log = *_log;
+
+	*_log = NULL;
+	i_close_fd(&log->fd);
+	i_free(log->path);
+	i_free(log);
+}
+
+static int fts_expunge_log_open(struct fts_expunge_log *log, bool create)
+{
+	int fd;
+
+	i_assert(log->fd == -1);
+
+	/* FIXME: use proper permissions */
+	fd = open(log->path, O_RDWR | O_APPEND | (create ? O_CREAT : 0), 0600);
+	if (fd == -1) {
+		if (errno == ENOENT && !create)
+			return 0;
+
+		i_error("open(%s) failed: %m", log->path);
+		return -1;
+	}
+	if (fstat(fd, &log->st) < 0) {
+		i_error("fstat(%s) failed: %m", log->path);
+		i_close_fd(&fd);
+		return -1;
+	}
+	log->fd = fd;
+	return 1;
+}
+
+static int
+fts_expunge_log_reopen_if_needed(struct fts_expunge_log *log, bool create)
+{
+	struct stat st;
+
+	if (log->fd == -1)
+		return fts_expunge_log_open(log, create);
+
+	if (stat(log->path, &st) == 0) {
+		if (st.st_ino == log->st.st_ino &&
+		    CMP_DEV_T(st.st_dev, log->st.st_dev)) {
+			/* same file */
+			return 0;
+		}
+		/* file changed */
+	} else if (errno == ENOENT) {
+		/* recreate the file */
+	} else {
+		i_error("stat(%s) failed: %m", log->path);
+		return -1;
+	}
+	if (close(log->fd) < 0)
+		i_error("close(%s) failed: %m", log->path);
+	log->fd = -1;
+	return fts_expunge_log_open(log, create);
+}
+
+static int
+fts_expunge_log_read_expunge_count(struct fts_expunge_log *log,
+				   uint32_t *expunge_count_r)
+{
+	ssize_t ret;
+
+	i_assert(log->fd != -1);
+
+	if (fstat(log->fd, &log->st) < 0) {
+		i_error("fstat(%s) failed: %m", log->path);
+		return -1;
+	}
+	if ((uoff_t)log->st.st_size < sizeof(*expunge_count_r)) {
+		*expunge_count_r = 0;
+		return 0;
+	}
+	/* we'll assume that write()s atomically grow the file size, as
+	   O_APPEND almost guarantees. even if not, having a race condition
+	   isn't the end of the world. the expunge count is simply read wrong
+	   and fts optimize is performed earlier or later than intended. */
+	ret = pread(log->fd, expunge_count_r, sizeof(*expunge_count_r),
+		    log->st.st_size - 4);
+	if (ret < 0) {
+		i_error("pread(%s) failed: %m", log->path);
+		return -1;
+	}
+	if (ret != sizeof(*expunge_count_r)) {
+		i_error("pread(%s) read only %d of %d bytes", log->path,
+			(int)ret, (int)sizeof(*expunge_count_r));
+		return -1;
+	}
+	return 0;
+}
+
+struct fts_expunge_log_append_ctx *
+fts_expunge_log_append_begin(struct fts_expunge_log *log)
+{
+	struct fts_expunge_log_append_ctx *ctx;
+	pool_t pool;
+
+	pool = pool_alloconly_create("fts expunge log append", 1024);
+	ctx = p_new(pool, struct fts_expunge_log_append_ctx, 1);
+	ctx->log = log;
+	ctx->pool = pool;
+	hash_table_create(&ctx->mailboxes, pool, 0, guid_128_hash, guid_128_cmp);
+
+	if (log != NULL && fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
+		ctx->failed = TRUE;
+	return ctx;
+}
+
+static struct fts_expunge_log_mailbox *
+fts_expunge_log_mailbox_alloc(struct fts_expunge_log_append_ctx *ctx,
+			      const guid_128_t mailbox_guid)
+{
+	uint8_t *guid_p;
+	struct fts_expunge_log_mailbox *mailbox;
+
+	mailbox = p_new(ctx->pool, struct fts_expunge_log_mailbox, 1);
+	guid_128_copy(mailbox->guid, mailbox_guid);
+	p_array_init(&mailbox->uids, ctx->pool, 16);
+
+	guid_p = mailbox->guid;
+	hash_table_insert(ctx->mailboxes, guid_p, mailbox);
+	return mailbox;
+}
+
+static struct fts_expunge_log_mailbox *
+fts_expunge_log_append_mailbox(struct fts_expunge_log_append_ctx *ctx,
+			       const guid_128_t mailbox_guid)
+{
+	const uint8_t *guid_p = mailbox_guid;
+	struct fts_expunge_log_mailbox *mailbox;
+
+	if (ctx->prev_mailbox != NULL &&
+	    guid_128_equals(mailbox_guid, ctx->prev_mailbox->guid))
+		mailbox = ctx->prev_mailbox;
+	else {
+		mailbox = hash_table_lookup(ctx->mailboxes, guid_p);
+		if (mailbox == NULL)
+			mailbox = fts_expunge_log_mailbox_alloc(ctx, mailbox_guid);
+		ctx->prev_mailbox = mailbox;
+	}
+	return mailbox;
+}
+void fts_expunge_log_append_next(struct fts_expunge_log_append_ctx *ctx,
+				 const guid_128_t mailbox_guid,
+				 uint32_t uid)
+{
+	struct fts_expunge_log_mailbox *mailbox;
+
+	mailbox = fts_expunge_log_append_mailbox(ctx, mailbox_guid);
+	if (!seq_range_array_add(&mailbox->uids, uid))
+		mailbox->uids_count++;
+}
+void fts_expunge_log_append_range(struct fts_expunge_log_append_ctx *ctx,
+				  const guid_128_t mailbox_guid,
+				  const struct seq_range *uids)
+{
+	struct fts_expunge_log_mailbox *mailbox;
+
+	mailbox = fts_expunge_log_append_mailbox(ctx, mailbox_guid);
+	mailbox->uids_count += seq_range_array_add_range_count(&mailbox->uids,
+							       uids->seq1, uids->seq2);
+	/* To be honest, an unbacked log doesn't need to maintain the uids_count,
+	   but we don't know here if we're supporting an unbacked log or not, so we
+	   have to maintain the value, just in case.
+	   At the moment, the only caller of this function is for unbacked logs. */
+}
+void fts_expunge_log_append_record(struct fts_expunge_log_append_ctx *ctx,
+				   const struct fts_expunge_log_read_record *record)
+{
+	const struct seq_range *range;
+	/* FIXME: Optimise with a merge */
+	array_foreach(&record->uids, range)
+		fts_expunge_log_append_range(ctx, record->mailbox_guid, range);
+}
+static void fts_expunge_log_append_mailbox_record(struct fts_expunge_log_append_ctx *ctx,
+						  struct fts_expunge_log_mailbox *mailbox)
+{
+	const struct seq_range *range;
+	/* FIXME: Optimise with a merge */
+	array_foreach(&mailbox->uids, range)
+		fts_expunge_log_append_range(ctx, mailbox->guid, range);
+}
+
+static void
+fts_expunge_log_export(struct fts_expunge_log_append_ctx *ctx,
+		       uint32_t expunge_count, buffer_t *output)
+{
+	struct hash_iterate_context *iter;
+	uint8_t *guid_p;
+	struct fts_expunge_log_mailbox *mailbox;
+	struct fts_expunge_log_record *rec;
+	size_t rec_offset;
+
+	iter = hash_table_iterate_init(ctx->mailboxes);
+	while (hash_table_iterate(iter, ctx->mailboxes, &guid_p, &mailbox)) {
+		rec_offset = output->used;
+		rec = buffer_append_space_unsafe(output, sizeof(*rec));
+		memcpy(rec->guid, mailbox->guid, sizeof(rec->guid));
+
+		/* uint32_t expunge_uid_ranges[]; */
+		buffer_append(output, array_front(&mailbox->uids),
+			      array_count(&mailbox->uids) *
+			      sizeof(struct seq_range));
+		/* uint32_t expunge_count; */
+		expunge_count += mailbox->uids_count;
+		buffer_append(output, &expunge_count, sizeof(expunge_count));
+
+		/* update the header now that we know the record contents */
+		rec = buffer_get_space_unsafe(output, rec_offset,
+					      output->used - rec_offset);
+		rec->record_size = output->used - rec_offset;
+		rec->checksum = crc32_data(&rec->record_size,
+					   rec->record_size -
+					   sizeof(rec->checksum));
+	}
+	hash_table_iterate_deinit(&iter);
+}
+
+static int
+fts_expunge_log_write(struct fts_expunge_log_append_ctx *ctx)
+{
+	struct fts_expunge_log *log = ctx->log;
+	buffer_t *buf;
+	uint32_t expunge_count, *e;
+	int ret;
+
+	/* Unbacked expunge logs cannot be written, by definition */
+	i_assert(log != NULL);
+
+	/* try to append to the latest file */
+	if (fts_expunge_log_reopen_if_needed(log, TRUE) < 0)
+		return -1;
+
+	if (fts_expunge_log_read_expunge_count(log, &expunge_count) < 0)
+		return -1;
+
+	buf = buffer_create_dynamic(default_pool, 1024);
+	fts_expunge_log_export(ctx, expunge_count, buf);
+	/* the file was opened with O_APPEND, so this write() should be
+	   appended atomically without any need for locking. */
+	for (;;) {
+		if (write_full(log->fd, buf->data, buf->used) < 0) {
+			i_error("write(%s) failed: %m", log->path);
+			if (ftruncate(log->fd, log->st.st_size) < 0)
+				i_error("ftruncate(%s) failed: %m", log->path);
+		}
+		if ((ret = fts_expunge_log_reopen_if_needed(log, TRUE)) <= 0)
+			break;
+		/* the log was unlinked, so we'll need to write again to
+		   the new file. the expunge_count needs to be reset to zero
+		   from here. */
+		e = buffer_get_space_unsafe(buf, buf->used - sizeof(uint32_t),
+					    sizeof(uint32_t));
+		i_assert(*e > expunge_count);
+		*e -= expunge_count;
+		expunge_count = 0;
+	}
+	buffer_free(&buf);
+
+	if (ret == 0) {
+		/* finish by closing the log. this forces NFS to flush the
+		   changes to disk without our having to explicitly play with
+		   fsync() */
+		if (close(log->fd) < 0) {
+			/* FIXME: we should ftruncate() in case there
+			   were partial writes.. */
+			i_error("close(%s) failed: %m", log->path);
+			ret = -1;
+		}
+		log->fd = -1;
+	}
+	return ret;
+}
+
+static int fts_expunge_log_append_finalize(struct fts_expunge_log_append_ctx **_ctx,
+					   bool commit)
+{
+	struct fts_expunge_log_append_ctx *ctx = *_ctx;
+	int ret = ctx->failed ? -1 : 0;
+
+	*_ctx = NULL;
+	if (commit && ret == 0)
+		ret = fts_expunge_log_write(ctx);
+
+	hash_table_destroy(&ctx->mailboxes);
+	pool_unref(&ctx->pool);
+	return ret;
+}
+
+int fts_expunge_log_uid_count(struct fts_expunge_log *log,
+			      unsigned int *expunges_r)
+{
+	int ret;
+
+	if ((ret = fts_expunge_log_reopen_if_needed(log, FALSE)) <= 0) {
+		*expunges_r = 0;
+		return ret;
+	}
+
+	return fts_expunge_log_read_expunge_count(log, expunges_r);
+}
+
+int fts_expunge_log_append_commit(struct fts_expunge_log_append_ctx **_ctx)
+{
+	return fts_expunge_log_append_finalize(_ctx, TRUE);
+}
+
+int fts_expunge_log_append_abort(struct fts_expunge_log_append_ctx **_ctx)
+{
+	return fts_expunge_log_append_finalize(_ctx, FALSE);
+}
+
+struct fts_expunge_log_read_ctx *
+fts_expunge_log_read_begin(struct fts_expunge_log *log)
+{
+	struct fts_expunge_log_read_ctx *ctx;
+
+	ctx = i_new(struct fts_expunge_log_read_ctx, 1);
+	ctx->log = log;
+	if (fts_expunge_log_reopen_if_needed(log, FALSE) < 0)
+		ctx->failed = TRUE;
+	else if (log->fd != -1)
+		ctx->input = i_stream_create_fd(log->fd, SIZE_MAX);
+	ctx->unlink = TRUE;
+	return ctx;
+}
+
+static bool
+fts_expunge_log_record_size_is_valid(const struct fts_expunge_log_record *rec,
+				     unsigned int *uids_size_r)
+{
+	if (rec->record_size < sizeof(*rec) + sizeof(uint32_t)*3)
+		return FALSE;
+	*uids_size_r = rec->record_size - sizeof(*rec) - sizeof(uint32_t);
+	return *uids_size_r % sizeof(uint32_t)*2 == 0;
+}
+
+static void
+fts_expunge_log_read_failure(struct fts_expunge_log_read_ctx *ctx,
+			     unsigned int wanted_size)
+{
+	size_t size;
+
+	if (ctx->input->stream_errno != 0) {
+		ctx->failed = TRUE;
+		i_error("read(%s) failed: %s", ctx->log->path,
+			i_stream_get_error(ctx->input));
+	} else {
+		size = i_stream_get_data_size(ctx->input);
+		ctx->corrupted = TRUE;
+		i_error("Corrupted fts expunge log %s: "
+			"Unexpected EOF (read %zu / %u bytes)",
+			ctx->log->path, size, wanted_size);
+	}
+}
+
+const struct fts_expunge_log_read_record *
+fts_expunge_log_read_next(struct fts_expunge_log_read_ctx *ctx)
+{
+	const unsigned char *data;
+	const struct fts_expunge_log_record *rec;
+	unsigned int uids_size;
+	size_t size;
+	uint32_t checksum;
+
+	if (ctx->input == NULL)
+		return NULL;
+
+	/* initial read to try to get the record */
+	(void)i_stream_read_bytes(ctx->input, &data, &size, IO_BLOCK_SIZE);
+	if (size == 0 && ctx->input->stream_errno == 0) {
+		/* expected EOF - mark the file as read by unlinking it */
+		if (ctx->unlink)
+			i_unlink_if_exists(ctx->log->path);
+
+		/* try reading again, in case something new was written */
+		i_stream_sync(ctx->input);
+		(void)i_stream_read_bytes(ctx->input, &data, &size,
+					  IO_BLOCK_SIZE);
+	}
+	if (size < sizeof(*rec)) {
+		if (size == 0 && ctx->input->stream_errno == 0) {
+			/* expected EOF */
+			return NULL;
+		}
+		fts_expunge_log_read_failure(ctx, sizeof(*rec));
+		return NULL;
+	}
+	rec = (const void *)data;
+
+	if (!fts_expunge_log_record_size_is_valid(rec, &uids_size)) {
+		ctx->corrupted = TRUE;
+		i_error("Corrupted fts expunge log %s: "
+			"Invalid record size: %u",
+			ctx->log->path, rec->record_size);
+		return NULL;
+	}
+
+	/* read the entire record */
+	while (size < rec->record_size) {
+		if (i_stream_read_bytes(ctx->input, &data, &size, rec->record_size) < 0) {
+			fts_expunge_log_read_failure(ctx, rec->record_size);
+			return NULL;
+		}
+		rec = (const void *)data;
+	}
+
+	/* verify that the record checksum is valid */
+	checksum = crc32_data(&rec->record_size,
+			      rec->record_size - sizeof(rec->checksum));
+	if (checksum != rec->checksum) {
+		ctx->corrupted = TRUE;
+		i_error("Corrupted fts expunge log %s: "
+			"Record checksum mismatch: %u != %u",
+			ctx->log->path, checksum, rec->checksum);
+		return NULL;
+	}
+
+	memcpy(ctx->read_rec.mailbox_guid, rec->guid,
+	       sizeof(ctx->read_rec.mailbox_guid));
+	/* create the UIDs array by pointing it directly into input
+	   stream's buffer */
+	buffer_create_from_const_data(&ctx->buffer, rec + 1, uids_size);
+	array_create_from_buffer(&ctx->read_rec.uids, &ctx->buffer,
+				 sizeof(struct seq_range));
+
+	i_stream_skip(ctx->input, rec->record_size);
+	return &ctx->read_rec;
+}
+
+int fts_expunge_log_read_end(struct fts_expunge_log_read_ctx **_ctx)
+{
+	struct fts_expunge_log_read_ctx *ctx = *_ctx;
+	int ret = ctx->failed ? -1 : (ctx->corrupted ? 0 : 1);
+
+	*_ctx = NULL;
+
+	if (ctx->corrupted) {
+		if (ctx->unlink)
+			i_unlink_if_exists(ctx->log->path);
+	}
+
+	i_stream_unref(&ctx->input);
+	i_free(ctx);
+	return ret;
+}
+
+int fts_expunge_log_flatten(const char *path,
+			    struct fts_expunge_log_append_ctx **flattened_r)
+{
+	struct fts_expunge_log *read;
+	struct fts_expunge_log_read_ctx *read_ctx;
+	const struct fts_expunge_log_read_record *record;
+	struct fts_expunge_log_append_ctx *append;
+	int ret;
+
+	i_assert(path != NULL && flattened_r != NULL);
+	read = fts_expunge_log_init(path);
+
+	read_ctx = fts_expunge_log_read_begin(read);
+	read_ctx->unlink = FALSE;
+
+	append = fts_expunge_log_append_begin(NULL);
+	while((record = fts_expunge_log_read_next(read_ctx)) != NULL) {
+		fts_expunge_log_append_record(append, record);
+	}
+
+	if ((ret = fts_expunge_log_read_end(&read_ctx)) > 0)
+		*flattened_r = append;
+	fts_expunge_log_deinit(&read);
+
+	return ret;
+}
+bool fts_expunge_log_contains(const struct fts_expunge_log_append_ctx *ctx,
+			      const guid_128_t mailbox_guid, uint32_t uid)
+{
+	const struct fts_expunge_log_mailbox *mailbox;
+	const uint8_t *guid_p = mailbox_guid;
+
+	mailbox = hash_table_lookup(ctx->mailboxes, guid_p);
+	if (mailbox == NULL)
+		return FALSE;
+	return seq_range_exists(&mailbox->uids, uid);	
+}
+int fts_expunge_log_append_remove(struct fts_expunge_log_append_ctx *from,
+				   const struct fts_expunge_log_read_record *record)
+{
+	const uint8_t *guid_p = record->mailbox_guid;
+	struct fts_expunge_log_mailbox *mailbox = hash_table_lookup(from->mailboxes, guid_p);
+	if (mailbox == NULL)
+		return 0; /* may only remove things that exist */
+
+	mailbox->uids_count -= seq_range_array_remove_seq_range(&mailbox->uids, &record->uids);
+	return 1;
+}
+int fts_expunge_log_subtract(struct fts_expunge_log_append_ctx *from,
+			     struct fts_expunge_log *subtract)
+{
+	unsigned int failures = 0;
+	struct fts_expunge_log_read_ctx *read_ctx = fts_expunge_log_read_begin(subtract);
+	read_ctx->unlink = FALSE;
+
+	const struct fts_expunge_log_read_record *record;
+	while ((record = fts_expunge_log_read_next(read_ctx)) != NULL) {
+		if (fts_expunge_log_append_remove(from, record) <= 0)
+			failures++;
+	}
+	if (failures > 0)
+		i_warning("fts: Expunge log subtract ignored %u nonexistent mailbox GUIDs",
+			  failures);
+	return fts_expunge_log_read_end(&read_ctx);
+}
+/* It could be argued that somehow adding a log (file) to the append context
+   and then calling the _write() helper would be easier. But then there's the
+   _commit() vs. _abort() cleanup that would need to be addressed. Just creating
+   a copy is simpler. */
+int fts_expunge_log_flat_write(const struct fts_expunge_log_append_ctx *read_log,
+			       const char *path)
+{
+	int ret;
+	struct fts_expunge_log *nlog = fts_expunge_log_init(path);
+	struct fts_expunge_log_append_ctx *nappend = fts_expunge_log_append_begin(nlog);
+
+	struct hash_iterate_context *iter;
+	uint8_t *guid_p;
+	struct fts_expunge_log_mailbox *mailbox;
+
+	iter = hash_table_iterate_init(read_log->mailboxes);
+	while (hash_table_iterate(iter, read_log->mailboxes, &guid_p, &mailbox))
+		fts_expunge_log_append_mailbox_record(nappend, mailbox);
+
+	hash_table_iterate_deinit(&iter);
+	ret = fts_expunge_log_append_commit(&nappend);
+	fts_expunge_log_deinit(&nlog);
+
+	return ret;
+}
diff --git a/src/plugins/fts/fts-expunge-log.h b/src/plugins/fts/fts-expunge-log.h
new file mode 100644
index 0000000..cc15f29
--- /dev/null
+++ b/src/plugins/fts/fts-expunge-log.h
@@ -0,0 +1,58 @@
+#ifndef FTS_EXPUNGE_LOG
+#define FTS_EXPUNGE_LOG
+
+#include "seq-range-array.h"
+#include "guid.h"
+
+struct fts_expunge_log_read_record {
+	guid_128_t mailbox_guid;
+	ARRAY_TYPE(seq_range) uids;
+};
+
+struct fts_expunge_log *fts_expunge_log_init(const char *path);
+void fts_expunge_log_deinit(struct fts_expunge_log **log);
+
+struct fts_expunge_log_append_ctx *
+fts_expunge_log_append_begin(struct fts_expunge_log *log);
+void fts_expunge_log_append_next(struct fts_expunge_log_append_ctx *ctx,
+				 const guid_128_t mailbox_guid,
+				 uint32_t uid);
+void fts_expunge_log_append_range(struct fts_expunge_log_append_ctx *ctx,
+				  const guid_128_t mailbox_guid,
+				  const struct seq_range *uids);
+void fts_expunge_log_append_record(struct fts_expunge_log_append_ctx *ctx,
+				   const struct fts_expunge_log_read_record *record);
+/* In-memory flattened structures may have records removed from them,
+   file-backed ones may not. Non-existence of UIDs is not an error,
+   non-existence of mailbox GUID causes an error return of 0. */
+int fts_expunge_log_append_remove(struct fts_expunge_log_append_ctx *ctx,
+				  const struct fts_expunge_log_read_record *record);
+int fts_expunge_log_append_commit(struct fts_expunge_log_append_ctx **ctx);
+/* Do not commit non-backed structures, abort them after use. */
+int fts_expunge_log_append_abort(struct fts_expunge_log_append_ctx **ctx);
+
+int fts_expunge_log_uid_count(struct fts_expunge_log *log,
+			      unsigned int *expunges_r);
+
+struct fts_expunge_log_read_ctx *
+fts_expunge_log_read_begin(struct fts_expunge_log *log);
+const struct fts_expunge_log_read_record *
+fts_expunge_log_read_next(struct fts_expunge_log_read_ctx *ctx);
+/* Returns 1 if all ok, 0 if there was corruption, -1 if I/O error.
+   If end() is called before reading all records, the log isn't unlinked. */
+int fts_expunge_log_read_end(struct fts_expunge_log_read_ctx **ctx);
+
+/* Read an entire log file, and flatten it into one hash of arrays.
+   The struct it returns cannot be written, as it has no backing store */
+int fts_expunge_log_flatten(const char *path,
+			    struct fts_expunge_log_append_ctx **flattened_r);
+bool fts_expunge_log_contains(const struct fts_expunge_log_append_ctx *ctx,
+			      const guid_128_t mailbox_guid, uint32_t uid);
+/* Modify in-place a flattened log. If non-existent mailbox GUIDs are
+   encountered, a warning will be logged. */
+int fts_expunge_log_subtract(struct fts_expunge_log_append_ctx *from,
+			     struct fts_expunge_log *subtract);
+/* Write a modified flattened log as a new file. */
+int fts_expunge_log_flat_write(const struct fts_expunge_log_append_ctx *flattened,
+			       const char *path);
+#endif
diff --git a/src/plugins/fts/fts-indexer.c b/src/plugins/fts/fts-indexer.c
new file mode 100644
index 0000000..aca23c9
--- /dev/null
+++ b/src/plugins/fts/fts-indexer.c
@@ -0,0 +1,300 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "connection.h"
+#include "write-full.h"
+#include "istream.h"
+#include "ostream.h"
+#include "strescape.h"
+#include "time-util.h"
+#include "settings-parser.h"
+#include "mail-user.h"
+#include "mail-storage-private.h"
+#include "fts-api.h"
+#include "fts-indexer.h"
+
+#define INDEXER_NOTIFY_INTERVAL_SECS 10
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_WAIT_MSECS 250
+
+struct fts_indexer_context {
+	struct connection conn;
+
+	struct mailbox *box;
+	struct ioloop *ioloop;
+
+	struct timeval search_start_time, last_notify;
+	unsigned int percentage;
+	struct connection_list *connection_list;
+
+	bool notified:1;
+	bool failed:1;
+	bool completed:1;
+};
+
+static void fts_indexer_notify(struct fts_indexer_context *ctx)
+{
+	unsigned long long elapsed_msecs, est_total_msecs;
+	unsigned int eta_secs;
+
+	if (ioloop_time - ctx->last_notify.tv_sec < INDEXER_NOTIFY_INTERVAL_SECS)
+		return;
+	ctx->last_notify = ioloop_timeval;
+
+	if (ctx->box->storage->callbacks.notify_ok == NULL ||
+	    ctx->percentage == 0)
+		return;
+
+	elapsed_msecs = timeval_diff_msecs(&ioloop_timeval,
+					   &ctx->search_start_time);
+	est_total_msecs = elapsed_msecs * 100 / ctx->percentage;
+	eta_secs = (est_total_msecs - elapsed_msecs) / 1000;
+
+	T_BEGIN {
+		const char *text;
+
+		text = t_strdup_printf("Indexed %d%% of the mailbox, "
+				       "ETA %d:%02d", ctx->percentage,
+				       eta_secs/60, eta_secs%60);
+		ctx->box->storage->callbacks.
+			notify_ok(ctx->box, text,
+				  ctx->box->storage->callback_context);
+		ctx->notified = TRUE;
+	} T_END;
+}
+
+static int fts_indexer_more_int(struct fts_indexer_context *ctx)
+{
+	struct ioloop *prev_ioloop = current_ioloop;
+	struct timeout *to;
+
+	if (ctx->failed)
+		return -1;
+	if (ctx->completed)
+		return 1;
+
+	/* wait for a while for the reply. FIXME: once search API supports
+	   asynchronous waits, get rid of this wait and use the mail IO loop */
+	io_loop_set_current(ctx->ioloop);
+	to = timeout_add_short(INDEXER_WAIT_MSECS, io_loop_stop, ctx->ioloop);
+	io_loop_run(ctx->ioloop);
+	timeout_remove(&to);
+	io_loop_set_current(prev_ioloop);
+
+	if (ctx->failed)
+		return -1;
+	if (ctx->completed)
+		return 1;
+	return 0;
+}
+
+int fts_indexer_more(struct fts_indexer_context *ctx)
+{
+	int ret;
+
+	if ((ret = fts_indexer_more_int(ctx)) < 0) {
+		/* If failed is already set, the code has had a chance to
+		 * set an internal error already, i.e. MAIL_ERROR_INUSE. */
+		if (!ctx->failed)
+			mail_storage_set_internal_error(ctx->box->storage);
+		ctx->failed = TRUE;
+		return -1;
+	}
+
+	if (ret == 0)
+		fts_indexer_notify(ctx);
+
+	return ret;
+}
+
+static void fts_indexer_destroy(struct connection *conn)
+{
+	struct fts_indexer_context *ctx =
+		container_of(conn, struct fts_indexer_context, conn);
+	connection_deinit(conn);
+	if (!ctx->completed)
+		ctx->failed = TRUE;
+	ctx->completed = TRUE;
+}
+
+int fts_indexer_deinit(struct fts_indexer_context **_ctx)
+{
+	struct fts_indexer_context *ctx = *_ctx;
+	i_assert(ctx != NULL);
+	*_ctx = NULL;
+	if (!ctx->completed)
+		ctx->failed = TRUE;
+	int ret = ctx->failed ? -1 : 0;
+	if (ctx->notified) {
+		/* we notified at least once */
+		ctx->box->storage->callbacks.
+			notify_ok(ctx->box, "Mailbox indexing finished",
+				  ctx->box->storage->callback_context);
+	}
+	connection_list_deinit(&ctx->connection_list);
+	io_loop_set_current(ctx->ioloop);
+	io_loop_destroy(&ctx->ioloop);
+	i_free(ctx);
+	return ret;
+}
+
+static int
+fts_indexer_input_args(struct connection *conn, const char *const *args)
+{
+	struct fts_indexer_context *ctx =
+		container_of(conn, struct fts_indexer_context, conn);
+	int percentage;
+	if (args[1] == NULL) {
+		e_error(conn->event, "indexer sent invalid reply");
+		return -1;
+	}
+	if (strcmp(args[0], "1") != 0) {
+		e_error(conn->event, "indexer sent invalid reply");
+		return -1;
+	}
+	if (strcmp(args[1], "OK") == 0)
+		return 1;
+	if (str_to_int(args[1], &percentage) < 0) {
+		e_error(conn->event, "indexer sent invalid progress: %s", args[1]);
+		ctx->failed = TRUE;
+		return -1;
+	}
+	if (percentage < 0) {
+		e_error(ctx->box->event, "indexer failed to index mailbox");
+		ctx->failed = TRUE;
+		return -1;
+	}
+	ctx->percentage = percentage;
+	if (ctx->percentage == 100)
+		ctx->completed = TRUE;
+	return 1;
+}
+
+static void fts_indexer_client_connected(struct connection *conn, bool success)
+{
+	struct fts_indexer_context *ctx =
+		container_of(conn, struct fts_indexer_context, conn);
+	if (!success) {
+		ctx->completed = TRUE;
+		ctx->failed = TRUE;
+		return;
+	}
+	ctx->failed = ctx->completed = FALSE;
+	const char *cmd = t_strdup_printf("PREPEND\t1\t%s\t%s\t0\t%s\n",
+			      str_tabescape(ctx->box->storage->user->username),
+			      str_tabescape(ctx->box->vname),
+			      str_tabescape(ctx->box->storage->user->session_id));
+	o_stream_nsend_str(conn->output, cmd);
+}
+
+static void fts_indexer_idle_timeout(struct connection *conn)
+{
+	struct fts_indexer_context *ctx =
+		container_of(conn, struct fts_indexer_context, conn);
+	mail_storage_set_error(ctx->box->storage, MAIL_ERROR_INUSE,
+			       "Timeout while waiting for indexing to finish");
+	ctx->failed = TRUE;
+	connection_disconnect(conn);
+}
+
+static const struct connection_settings indexer_client_set =
+{
+	.service_name_in = "indexer",
+	.service_name_out = "indexer",
+	.major_version = 1,
+	.minor_version = 0,
+	.client_connect_timeout_msecs = 2000,
+	.input_max_size = SIZE_MAX,
+	.output_max_size = IO_BLOCK_SIZE,
+	.client = TRUE,
+};
+
+static const struct connection_vfuncs indexer_client_vfuncs =
+{
+	.destroy = fts_indexer_destroy,
+	.client_connected = fts_indexer_client_connected,
+	.input_args = fts_indexer_input_args,
+	.idle_timeout = fts_indexer_idle_timeout,
+};
+
+int fts_indexer_init(struct fts_backend *backend, struct mailbox *box,
+		     struct fts_indexer_context **ctx_r)
+{
+	struct ioloop *prev_ioloop = current_ioloop;
+	struct fts_indexer_context *ctx;
+	struct mailbox_status status;
+	uint32_t last_uid, seq1, seq2;
+	const char *path, *value, *error;
+	unsigned int timeout_secs = 0;
+	int ret;
+
+	value = mail_user_plugin_getenv(box->storage->user, "fts_index_timeout");
+	if (value != NULL) {
+		if (settings_get_time(value, &timeout_secs, &error) < 0) {
+			e_error(box->storage->user->event,
+				"Invalid fts_index_timeout setting: %s",
+				error);
+			return -1;
+		}
+	}
+
+	if (fts_backend_get_last_uid(backend, box, &last_uid) < 0)
+		return -1;
+
+	mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
+	if (status.uidnext == last_uid+1) {
+		/* everything is already indexed */
+		return 0;
+	}
+
+	mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
+	if (seq1 == 0) {
+		/* no new messages (last messages in mailbox were expunged) */
+		return 0;
+	}
+
+	path = t_strconcat(box->storage->user->set->base_dir,
+			   "/"INDEXER_SOCKET_NAME, NULL);
+
+	ctx = i_new(struct fts_indexer_context, 1);
+	ctx->box = box;
+	ctx->search_start_time = ioloop_timeval;
+	ctx->conn.event_parent = box->event;
+	ctx->ioloop = io_loop_create();
+	ctx->connection_list = connection_list_init(&indexer_client_set,
+						    &indexer_client_vfuncs);
+	ctx->conn.input_idle_timeout_secs = timeout_secs;
+	connection_init_client_unix(ctx->connection_list, &ctx->conn,
+				    path);
+	ret = connection_client_connect(&ctx->conn);
+	io_loop_set_current(prev_ioloop);
+	*ctx_r = ctx;
+	return ctx->failed || ret < 0 ? -1 : 1;
+}
+
+#define INDEXER_HANDSHAKE "1\t0\tindexer\tindexer\n"
+
+int fts_indexer_cmd(struct mail_user *user, const char *cmd,
+	            const char **path_r)
+{
+	const char *path;
+	int fd;
+
+	path = t_strconcat(user->set->base_dir,
+	                   "/"INDEXER_SOCKET_NAME, NULL);
+	fd = net_connect_unix_with_retries(path, 1000);
+	if (fd == -1) {
+	        i_error("net_connect_unix(%s) failed: %m", path);
+	        return -1;
+	}
+
+	cmd = t_strconcat(INDEXER_HANDSHAKE, cmd, NULL);
+	if (write_full(fd, cmd, strlen(cmd)) < 0) {
+	        i_error("write(%s) failed: %m", path);
+	        i_close_fd(&fd);
+	        return -1;
+	}
+	*path_r = path;
+	return fd;
+}
diff --git a/src/plugins/fts/fts-indexer.h b/src/plugins/fts/fts-indexer.h
new file mode 100644
index 0000000..7ccbc7e
--- /dev/null
+++ b/src/plugins/fts/fts-indexer.h
@@ -0,0 +1,22 @@
+#ifndef FTS_BUILD_H
+#define FTS_BUILD_H
+
+struct fts_backend;
+struct fts_indexer_context;
+
+/* Initialize indexing the given mailbox via indexer service. Returns 1 if
+   indexing started, 0 if there was no need to index or -1 if error. */
+int fts_indexer_init(struct fts_backend *backend, struct mailbox *box,
+		     struct fts_indexer_context **ctx_r);
+/* Returns 0 if ok, -1 if error. */
+int fts_indexer_deinit(struct fts_indexer_context **ctx);
+
+/* Build more. Returns 1 if finished, 0 if this function needs to be called
+   again, -1 if error. */
+int fts_indexer_more(struct fts_indexer_context *ctx);
+
+/* Returns fd, which you can either read from or close. */
+int fts_indexer_cmd(struct mail_user *user, const char *cmd,
+		    const char **path_r);
+
+#endif
diff --git a/src/plugins/fts/fts-parser-html.c b/src/plugins/fts/fts-parser-html.c
new file mode 100644
index 0000000..aa2078d
--- /dev/null
+++ b/src/plugins/fts/fts-parser-html.c
@@ -0,0 +1,64 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "message-parser.h"
+#include "mail-html2text.h"
+#include "fts-parser.h"
+
+struct html_fts_parser {
+	struct fts_parser parser;
+	struct mail_html2text *html2text;
+	buffer_t *output;
+};
+
+static struct fts_parser *
+fts_parser_html_try_init(struct fts_parser_context *parser_context)
+{
+	struct html_fts_parser *parser;
+
+	if (!mail_html2text_content_type_match(parser_context->content_type))
+		return NULL;
+
+	parser = i_new(struct html_fts_parser, 1);
+	parser->parser.v = fts_parser_html;
+	parser->html2text = mail_html2text_init(0);
+	parser->output = buffer_create_dynamic(default_pool, 4096);
+	return &parser->parser;
+}
+
+static void fts_parser_html_more(struct fts_parser *_parser,
+				 struct message_block *block)
+{
+	struct html_fts_parser *parser = (struct html_fts_parser *)_parser;
+
+	if (block->size == 0) {
+		/* finished */
+		return;
+	}
+
+	buffer_set_used_size(parser->output, 0);
+	mail_html2text_more(parser->html2text, block->data, block->size,
+			    parser->output);
+
+	block->data = parser->output->data;
+	block->size = parser->output->used;
+}
+
+static int fts_parser_html_deinit(struct fts_parser *_parser,
+				  const char **retriable_err_msg_r ATTR_UNUSED)
+{
+	struct html_fts_parser *parser = (struct html_fts_parser *)_parser;
+
+	mail_html2text_deinit(&parser->html2text);
+	buffer_free(&parser->output);
+	i_free(parser);
+	return 1;
+}
+
+struct fts_parser_vfuncs fts_parser_html = {
+	fts_parser_html_try_init,
+	fts_parser_html_more,
+	fts_parser_html_deinit,
+	NULL
+};
diff --git a/src/plugins/fts/fts-parser-script.c b/src/plugins/fts/fts-parser-script.c
new file mode 100644
index 0000000..eefbe07
--- /dev/null
+++ b/src/plugins/fts/fts-parser-script.c
@@ -0,0 +1,277 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "net.h"
+#include "istream.h"
+#include "write-full.h"
+#include "module-context.h"
+#include "rfc822-parser.h"
+#include "rfc2231-parser.h"
+#include "message-parser.h"
+#include "mail-user.h"
+#include "fts-parser.h"
+
+#define SCRIPT_USER_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, fts_parser_script_user_module)
+
+#define SCRIPT_HANDSHAKE "VERSION\tscript\t4\t0\nalarm=10\nnoreply\n"
+
+struct content {
+	const char *content_type;
+	const char *const *extensions;
+};
+
+struct fts_parser_script_user {
+	union mail_user_module_context module_ctx;
+
+	ARRAY(struct content) content;
+};
+
+struct script_fts_parser {
+	struct fts_parser parser;
+
+	int fd;
+	char *path;
+
+	unsigned char outbuf[IO_BLOCK_SIZE];
+	bool failed;
+	bool shutdown;
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(fts_parser_script_user_module,
+				  &mail_user_module_register);
+
+static int script_connect(struct mail_user *user, const char **path_r)
+{
+	const char *path;
+	int fd;
+
+	path = mail_user_plugin_getenv(user, "fts_decoder");
+	if (path == NULL)
+		return -1;
+
+	if (*path != '/')
+		path = t_strconcat(user->set->base_dir, "/", path, NULL);
+	fd = net_connect_unix_with_retries(path, 1000);
+	if (fd == -1)
+		i_error("net_connect_unix(%s) failed: %m", path);
+	else
+		net_set_nonblock(fd, FALSE);
+	*path_r = path;
+	return fd;
+}
+
+static int script_contents_read(struct mail_user *user)
+{
+	struct fts_parser_script_user *suser = SCRIPT_USER_CONTEXT(user);
+	const char *path, *cmd, *line;
+	char **args;
+	struct istream *input;
+	struct content *content;
+	bool eof_seen = FALSE;
+	int fd, ret = 0;
+	i_assert(suser != NULL);
+
+	fd = script_connect(user, &path);
+	if (fd == -1)
+		return -1;
+
+	cmd = t_strdup_printf(SCRIPT_HANDSHAKE"\n");
+	if (write_full(fd, cmd, strlen(cmd)) < 0) {
+		i_error("write(%s) failed: %m", path);
+		i_close_fd(&fd);
+		return -1;
+	}
+	input = i_stream_create_fd_autoclose(&fd, 1024);
+	while ((line = i_stream_read_next_line(input)) != NULL) {
+		/* <content-type> <extension> [<extension> ...] */
+		args = p_strsplit_spaces(user->pool, line, " ");
+		if (args[0] == NULL) {
+			eof_seen = TRUE;
+			break;
+		}
+		if (args[0][0] == '\0' || args[1] == NULL) {
+			i_error("parser script sent invalid input: %s", line);
+			continue;
+		}
+
+		content = array_append_space(&suser->content);
+		content->content_type = str_lcase(args[0]);
+		content->extensions = (const void *)(args+1);
+	}
+	if (input->stream_errno != 0) {
+		i_error("parser script read(%s) failed: %s", path,
+			i_stream_get_error(input));
+		ret = -1;
+	} else if (!eof_seen) {
+		if (input->v_offset == 0)
+			i_error("parser script didn't send any data");
+		else
+			i_error("parser script didn't send empty EOF line");
+	}
+	i_stream_destroy(&input);
+	return ret;
+}
+
+static bool script_support_content(struct mail_user *user,
+				   const char **content_type,
+				   const char *filename)
+{
+	struct fts_parser_script_user *suser = SCRIPT_USER_CONTEXT(user);
+	const struct content *content;
+	const char *extension;
+
+	if (suser == NULL) {
+		suser = p_new(user->pool, struct fts_parser_script_user, 1);
+		p_array_init(&suser->content, user->pool, 32);
+		MODULE_CONTEXT_SET(user, fts_parser_script_user_module, suser);
+	}
+	if (array_count(&suser->content) == 0) {
+		if (script_contents_read(user) < 0)
+			return FALSE;
+	}
+
+	if (strcmp(*content_type, "application/octet-stream") == 0) {
+		if (filename == NULL)
+			return FALSE;
+		extension = strrchr(filename, '.');
+		if (extension == NULL)
+			return FALSE;
+		extension = extension + 1;
+
+		array_foreach(&suser->content, content) {
+			if (content->extensions != NULL &&
+			    str_array_icase_find(content->extensions, extension)) {
+				*content_type = content->content_type;
+				return TRUE;
+			}
+		}
+	} else {
+		array_foreach(&suser->content, content) {
+			if (strcmp(content->content_type, *content_type) == 0)
+				return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static void parse_content_disposition(const char *content_disposition,
+				      const char **filename_r)
+{
+	struct rfc822_parser_context parser;
+	const char *const *results, *filename2;
+	string_t *str;
+
+	*filename_r = NULL;
+
+	if (content_disposition == NULL)
+		return;
+
+	rfc822_parser_init(&parser, (const unsigned char *)content_disposition,
+			   strlen(content_disposition), NULL);
+	rfc822_skip_lwsp(&parser);
+
+	/* type; param; param; .. */
+	str = t_str_new(32);
+	if (rfc822_parse_mime_token(&parser, str) < 0) {
+		rfc822_parser_deinit(&parser);
+		return;
+	}
+
+	rfc2231_parse(&parser, &results);
+	filename2 = NULL;
+	for (; *results != NULL; results += 2) {
+		if (strcasecmp(results[0], "filename") == 0) {
+			*filename_r = results[1];
+			break;
+		}
+		if (strcasecmp(results[0], "filename*") == 0)
+			filename2 = results[1];
+	}
+	if (*filename_r == NULL) {
+		/* RFC 2231 style non-ascii filename. we don't really care
+		   much about the filename actually, just about its extension */
+		*filename_r = filename2;
+	}
+	rfc822_parser_deinit(&parser);
+}
+
+static struct fts_parser *
+fts_parser_script_try_init(struct fts_parser_context *parser_context)
+{
+	struct script_fts_parser *parser;
+	const char *filename, *path, *cmd;
+	int fd;
+
+	parse_content_disposition(parser_context->content_disposition, &filename);
+	if (!script_support_content(parser_context->user, &parser_context->content_type, filename))
+		return NULL;
+
+	fd = script_connect(parser_context->user, &path);
+	if (fd == -1)
+		return NULL;
+	cmd = t_strdup_printf(SCRIPT_HANDSHAKE"%s\n\n", parser_context->content_type);
+	if (write_full(fd, cmd, strlen(cmd)) < 0) {
+		i_error("write(%s) failed: %m", path);
+		i_close_fd(&fd);
+		return NULL;
+	}
+
+	parser = i_new(struct script_fts_parser, 1);
+	parser->parser.v = fts_parser_script;
+	parser->path = i_strdup(path);
+	parser->fd = fd;
+	return &parser->parser;
+}
+
+static void fts_parser_script_more(struct fts_parser *_parser,
+				   struct message_block *block)
+{
+	struct script_fts_parser *parser = (struct script_fts_parser *)_parser;
+	ssize_t ret;
+
+	if (block->size > 0) {
+		/* first we'll send everything to the script */
+		if (!parser->failed &&
+		    write_full(parser->fd, block->data, block->size) < 0) {
+			i_error("write(%s) failed: %m", parser->path);
+			parser->failed = TRUE;
+		}
+		block->size = 0;
+	} else {
+		if (!parser->shutdown) {
+			if (shutdown(parser->fd, SHUT_WR) < 0)
+				i_error("shutdown(%s) failed: %m", parser->path);
+			parser->shutdown = TRUE;
+		}
+		/* read the result from the script */
+		ret = read(parser->fd, parser->outbuf, sizeof(parser->outbuf));
+		if (ret < 0)
+			i_error("read(%s) failed: %m", parser->path);
+		else {
+			block->data = parser->outbuf;
+			block->size = ret;
+		}
+	}
+}
+
+static int fts_parser_script_deinit(struct fts_parser *_parser,
+				    const char **retriable_err_msg_r ATTR_UNUSED)
+{
+	struct script_fts_parser *parser = (struct script_fts_parser *)_parser;
+	int ret = parser->failed ? -1 : 1;
+
+	if (close(parser->fd) < 0)
+		i_error("close(%s) failed: %m", parser->path);
+	i_free(parser->path);
+	i_free(parser);
+	return ret;
+}
+
+struct fts_parser_vfuncs fts_parser_script = {
+	fts_parser_script_try_init,
+	fts_parser_script_more,
+	fts_parser_script_deinit,
+	NULL
+};
diff --git a/src/plugins/fts/fts-parser-tika.c b/src/plugins/fts/fts-parser-tika.c
new file mode 100644
index 0000000..bb6379c
--- /dev/null
+++ b/src/plugins/fts/fts-parser-tika.c
@@ -0,0 +1,278 @@
+/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "module-context.h"
+#include "iostream-ssl.h"
+#include "http-url.h"
+#include "http-client.h"
+#include "message-parser.h"
+#include "mail-user.h"
+#include "fts-parser.h"
+
+#define TIKA_USER_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, fts_parser_tika_user_module)
+
+struct fts_parser_tika_user {
+	union mail_user_module_context module_ctx;
+	struct http_url *http_url;
+};
+
+struct tika_fts_parser {
+	struct fts_parser parser;
+	struct mail_user *user;
+	struct http_client_request *http_req;
+
+	struct ioloop *ioloop;
+	struct io *io;
+	struct istream *payload;
+
+	bool failed;
+};
+
+static struct http_client *tika_http_client = NULL;
+static MODULE_CONTEXT_DEFINE_INIT(fts_parser_tika_user_module,
+				  &mail_user_module_register);
+
+static int
+tika_get_http_client_url(struct mail_user *user, struct http_url **http_url_r)
+{
+	struct fts_parser_tika_user *tuser = TIKA_USER_CONTEXT(user);
+	struct http_client_settings http_set;
+	struct ssl_iostream_settings ssl_set;
+	const char *url, *error;
+
+	url = mail_user_plugin_getenv(user, "fts_tika");
+	if (url == NULL) {
+		/* fts_tika disabled */
+		return -1;
+	}
+
+	if (tuser != NULL) {
+		*http_url_r = tuser->http_url;
+		return *http_url_r == NULL ? -1 : 0;
+	}
+
+	tuser = p_new(user->pool, struct fts_parser_tika_user, 1);
+	MODULE_CONTEXT_SET(user, fts_parser_tika_user_module, tuser);
+
+	if (http_url_parse(url, NULL, 0, user->pool,
+			   &tuser->http_url, &error) < 0) {
+		i_error("fts_tika: Failed to parse HTTP url %s: %s", url, error);
+		return -1;
+	}
+
+	if (tika_http_client == NULL) {
+		mail_user_init_ssl_client_settings(user, &ssl_set);
+
+		i_zero(&http_set);
+		http_set.max_idle_time_msecs = 100;
+		http_set.max_parallel_connections = 1;
+		http_set.max_pipelined_requests = 1;
+		http_set.max_redirects = 1;
+		http_set.max_attempts = 3;
+		http_set.connect_timeout_msecs = 5*1000;
+		http_set.request_timeout_msecs = 60*1000;
+		http_set.ssl = &ssl_set;
+		http_set.debug = user->mail_debug;
+		http_set.event_parent = user->event;
+
+		/* FIXME: We should initialize a shared client instead. However,
+		          this is currently not possible due to an obscure bug
+		          in the blocking HTTP payload API, which causes
+		          conflicts with other HTTP applications like FTS Solr.
+		          Using a private client will provide a quick fix for
+		          now. */
+		tika_http_client = http_client_init_private(&http_set);
+	}
+	*http_url_r = tuser->http_url;
+	return 0;
+}
+
+static void
+fts_tika_parser_response(const struct http_response *response,
+			 struct tika_fts_parser *parser)
+{
+	i_assert(parser->payload == NULL);
+
+	switch (response->status) {
+	case 200:
+		/* read response */
+		if (response->payload == NULL)
+			parser->payload = i_stream_create_from_data("", 0);
+		else {
+			i_stream_ref(response->payload);
+			parser->payload = response->payload;
+		}
+		break;
+	case 204: /* empty response */
+	case 415: /* Unsupported Media Type */
+	case 422: /* Unprocessable Entity */
+		e_debug(parser->user->event, "fts_tika: PUT %s failed: %s",
+			mail_user_plugin_getenv(parser->user, "fts_tika"),
+			http_response_get_message(response));
+		parser->payload = i_stream_create_from_data("", 0);
+		break;
+	default:
+		if (response->status / 100 == 5) {
+			/* Server Error - the problem could be anything (in Tika or
+			   HTTP server or proxy) and might be retriable, but Tika has
+			   trouble processing some documents and throws up this error
+			   every time for those documents. */
+			parser->parser.may_need_retry = TRUE;
+			i_free(parser->parser.retriable_error_msg);
+			parser->parser.retriable_error_msg =
+				i_strdup_printf("fts_tika: PUT %s failed: %s",
+						mail_user_plugin_getenv(parser->user, "fts_tika"),
+						http_response_get_message(response));
+			parser->payload = i_stream_create_from_data("", 0);
+		} else {
+			i_error("fts_tika: PUT %s failed: %s",
+				mail_user_plugin_getenv(parser->user, "fts_tika"),
+				http_response_get_message(response));
+			parser->failed = TRUE;
+		}
+		break;
+	}
+	parser->http_req = NULL;
+	io_loop_stop(current_ioloop);
+}
+
+static struct fts_parser *
+fts_parser_tika_try_init(struct fts_parser_context *parser_context)
+{
+	struct tika_fts_parser *parser;
+	struct http_url *http_url;
+	struct http_client_request *http_req;
+
+	if (tika_get_http_client_url(parser_context->user, &http_url) < 0)
+		return NULL;
+	if (http_url->path == NULL)
+		http_url->path = "/";
+
+	parser = i_new(struct tika_fts_parser, 1);
+	parser->parser.v = fts_parser_tika;
+	parser->user = parser_context->user;
+
+	http_req = http_client_request(tika_http_client, "PUT",
+			http_url->host.name,
+			t_strconcat(http_url->path, http_url->enc_query, NULL),
+			fts_tika_parser_response, parser);
+	http_client_request_set_port(http_req, http_url->port);
+	http_client_request_set_ssl(http_req, http_url->have_ssl);
+	if (parser_context->content_type != NULL)
+		http_client_request_add_header(http_req, "Content-Type",
+					       parser_context->content_type);
+	if (parser_context->content_disposition != NULL)
+		http_client_request_add_header(http_req, "Content-Disposition",
+					       parser_context->content_disposition);
+	http_client_request_add_header(http_req, "Accept", "text/plain");
+
+	parser->http_req = http_req;
+	return &parser->parser;
+}
+
+static void fts_parser_tika_more(struct fts_parser *_parser,
+				 struct message_block *block)
+{
+	struct tika_fts_parser *parser = (struct tika_fts_parser *)_parser;
+	struct ioloop *prev_ioloop = current_ioloop;
+	const unsigned char *data;
+	size_t size;
+	ssize_t ret;
+
+	if (block->size > 0) {
+		/* first we'll send everything to Tika */
+		if (!parser->failed &&
+		    http_client_request_send_payload(&parser->http_req,
+						     block->data,
+						     block->size) < 0)
+			parser->failed = TRUE;
+		block->size = 0;
+		return;
+	}
+
+	if (parser->payload == NULL) {
+		/* read the result from Tika */
+		if (!parser->failed &&
+		    http_client_request_finish_payload(&parser->http_req) < 0)
+			parser->failed = TRUE;
+		if (!parser->failed && parser->payload == NULL)
+			http_client_wait(tika_http_client);
+		if (parser->failed)
+			return;
+		i_assert(parser->payload != NULL);
+	}
+	/* continue returning data from Tika. we'll create a new ioloop just
+	   for reading this one payload. */
+	while ((ret = i_stream_read_more(parser->payload, &data, &size)) == 0) {
+		if (parser->failed)
+			break;
+		/* wait for more input from Tika */
+		if (parser->ioloop == NULL) {
+			parser->ioloop = io_loop_create();
+			parser->io = io_add_istream(parser->payload, io_loop_stop,
+						    current_ioloop);
+		} else {
+			io_loop_set_current(parser->ioloop);
+		}
+		io_loop_run(current_ioloop);
+	}
+	/* switch back to original ioloop. */
+	io_loop_set_current(prev_ioloop);
+
+	if (parser->failed)
+		;
+	else if (size > 0) {
+		i_assert(ret > 0);
+		block->data = data;
+		block->size = size;
+		i_stream_skip(parser->payload, size);
+	} else {
+		/* finished */
+		i_assert(ret == -1);
+		if (parser->payload->stream_errno != 0) {
+			i_error("read(%s) failed: %s",
+				i_stream_get_name(parser->payload),
+				i_stream_get_error(parser->payload));
+			parser->failed = TRUE;
+		}
+	}
+}
+
+static int fts_parser_tika_deinit(struct fts_parser *_parser, const char **retriable_err_msg_r)
+{
+	struct tika_fts_parser *parser = (struct tika_fts_parser *)_parser;
+	int ret = _parser->may_need_retry ? 0: (parser->failed ? -1 : 1);
+
+	i_assert(ret != 0 || _parser->retriable_error_msg != NULL);
+	if (retriable_err_msg_r != NULL)
+		*retriable_err_msg_r = t_strdup(_parser->retriable_error_msg);
+	i_free(_parser->retriable_error_msg);
+
+	/* remove io before unrefing payload - otherwise lib-http adds another
+	   timeout to ioloop unnecessarily */
+	i_stream_unref(&parser->payload);
+	io_remove(&parser->io);
+	http_client_request_abort(&parser->http_req);
+	if (parser->ioloop != NULL) {
+		io_loop_set_current(parser->ioloop);
+		io_loop_destroy(&parser->ioloop);
+	}
+	i_free(parser);
+	return ret;
+}
+
+static void fts_parser_tika_unload(void)
+{
+	if (tika_http_client != NULL)
+		http_client_deinit(&tika_http_client);
+}
+
+struct fts_parser_vfuncs fts_parser_tika = {
+	fts_parser_tika_try_init,
+	fts_parser_tika_more,
+	fts_parser_tika_deinit,
+	fts_parser_tika_unload
+};
diff --git a/src/plugins/fts/fts-parser.c b/src/plugins/fts/fts-parser.c
new file mode 100644
index 0000000..c0eac80
--- /dev/null
+++ b/src/plugins/fts/fts-parser.c
@@ -0,0 +1,127 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "unichar.h"
+#include "message-parser.h"
+#include "fts-parser.h"
+
+static const struct fts_parser_vfuncs *parsers[] = {
+	&fts_parser_html,
+	&fts_parser_script,
+	&fts_parser_tika
+};
+
+static const char *plaintext_content_types[] = {
+	"text/plain",
+	"message/delivery-status",
+	"message/disposition-notification",
+	"application/pgp-signature",
+	NULL
+};
+
+bool fts_parser_init(struct fts_parser_context *parser_context,
+		     struct fts_parser **parser_r)
+{
+	unsigned int i;
+	i_assert(parser_context->user != NULL);
+	i_assert(parser_context->content_type != NULL);
+
+	if (str_array_find(plaintext_content_types, parser_context->content_type)) {
+		/* we probably don't want/need to allow parsers to handle
+		   plaintext? */
+		return FALSE;
+	}
+
+	for (i = 0; i < N_ELEMENTS(parsers); i++) {
+		*parser_r = parsers[i]->try_init(parser_context);
+		if (*parser_r != NULL)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+struct fts_parser *fts_parser_text_init(void)
+{
+	return i_new(struct fts_parser, 1);
+}
+
+static bool data_has_nuls(const unsigned char *data, size_t size)
+{
+	size_t i;
+
+	for (i = 0; i < size; i++) {
+		if (data[i] == '\0')
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static void replace_nul_bytes(buffer_t *buf)
+{
+	unsigned char *data;
+	size_t i, size;
+
+	data = buffer_get_modifiable_data(buf, &size);
+	for (i = 0; i < size; i++) {
+		if (data[i] == '\0')
+			data[i] = ' ';
+	}
+}
+
+void fts_parser_more(struct fts_parser *parser, struct message_block *block)
+{
+	if (parser->v.more != NULL)
+		parser->v.more(parser, block);
+
+	if (!uni_utf8_data_is_valid(block->data, block->size) ||
+	    data_has_nuls(block->data, block->size)) {
+		/* output isn't valid UTF-8. make it. */
+		if (parser->utf8_output == NULL) {
+			parser->utf8_output =
+				buffer_create_dynamic(default_pool, 4096);
+		} else {
+			buffer_set_used_size(parser->utf8_output, 0);
+		}
+		if (uni_utf8_get_valid_data(block->data, block->size,
+					    parser->utf8_output)) {
+			/* valid UTF-8, but there were NULs */
+			buffer_append(parser->utf8_output, block->data,
+				      block->size);
+		}
+		replace_nul_bytes(parser->utf8_output);
+		block->data = parser->utf8_output->data;
+		block->size = parser->utf8_output->used;
+	}
+}
+
+int fts_parser_deinit(struct fts_parser **_parser, const char **retriable_err_msg_r)
+{
+	struct fts_parser *parser = *_parser;
+	int ret = 1;
+
+	*_parser = NULL;
+
+	buffer_free(&parser->utf8_output);
+	if (parser->v.deinit != NULL) {
+		const char *error = NULL;
+		ret = parser->v.deinit(parser, &error);
+		if (ret == 0) {
+			i_assert(error != NULL);
+			if (retriable_err_msg_r != NULL)
+				*retriable_err_msg_r = error;
+		}
+	} else
+		i_free(parser);
+	return ret;
+}
+
+void fts_parsers_unload(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < N_ELEMENTS(parsers); i++) {
+		if (parsers[i]->unload != NULL)
+			parsers[i]->unload();
+	}
+}
diff --git a/src/plugins/fts/fts-parser.h b/src/plugins/fts/fts-parser.h
new file mode 100644
index 0000000..0eb716e
--- /dev/null
+++ b/src/plugins/fts/fts-parser.h
@@ -0,0 +1,48 @@
+#ifndef FTS_PARSER_H
+#define FTS_PARSER_H
+
+struct message_block;
+struct mail_user;
+
+struct fts_parser_context {
+	/* Can't be NULL */
+	struct mail_user *user;
+	/* Can't be NULL */
+	const char *content_type;
+	const char *content_disposition;
+};
+
+struct fts_parser_vfuncs {
+	struct fts_parser *(*try_init)(struct fts_parser_context *parser_context);
+	void (*more)(struct fts_parser *parser, struct message_block *block);
+	int (*deinit)(struct fts_parser *parser, const char **retriable_err_msg_r);
+	void (*unload)(void);
+};
+
+struct fts_parser {
+	struct fts_parser_vfuncs v;
+	buffer_t *utf8_output;
+	bool may_need_retry;
+	char *retriable_error_msg;
+};
+
+extern struct fts_parser_vfuncs fts_parser_html;
+extern struct fts_parser_vfuncs fts_parser_script;
+extern struct fts_parser_vfuncs fts_parser_tika;
+
+bool fts_parser_init(struct fts_parser_context *parser_context,
+		     struct fts_parser **parser_r);
+struct fts_parser *fts_parser_text_init(void);
+
+/* The parser is initially called with message body blocks. Once message is
+   finished, it's still called with incoming size=0 while the parser increases
+   it to non-zero. */
+void fts_parser_more(struct fts_parser *parser, struct message_block *block);
+/* Returns 1 if ok, 0 if the parsing should be retried, -1 if error.
+   If 0 is returned, the retriable_err_msg_r is set, which should be logged
+   as error if no retrying is performed. */
+int fts_parser_deinit(struct fts_parser **parser, const char **retriable_err_msg_r);
+
+void fts_parsers_unload(void);
+
+#endif
diff --git a/src/plugins/fts/fts-plugin.c b/src/plugins/fts/fts-plugin.c
new file mode 100644
index 0000000..1902cb6
--- /dev/null
+++ b/src/plugins/fts/fts-plugin.c
@@ -0,0 +1,33 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-storage-hooks.h"
+#include "fts-filter.h"
+#include "fts-tokenizer.h"
+#include "fts-parser.h"
+#include "fts-storage.h"
+#include "fts-user.h"
+#include "fts-plugin.h"
+#include "fts-library.h"
+
+const char *fts_plugin_version = DOVECOT_ABI_VERSION;
+
+static struct mail_storage_hooks fts_mail_storage_hooks = {
+	.mail_namespaces_added = fts_mail_namespaces_added,
+	.mailbox_list_created = fts_mailbox_list_created,
+	.mailbox_allocated = fts_mailbox_allocated,
+	.mail_allocated = fts_mail_allocated
+};
+
+void fts_plugin_init(struct module *module)
+{
+	fts_library_init();
+	mail_storage_hooks_add(module, &fts_mail_storage_hooks);
+}
+
+void fts_plugin_deinit(void)
+{
+	fts_library_deinit();
+	fts_parsers_unload();
+	mail_storage_hooks_remove(&fts_mail_storage_hooks);
+}
diff --git a/src/plugins/fts/fts-plugin.h b/src/plugins/fts/fts-plugin.h
new file mode 100644
index 0000000..aeec68c
--- /dev/null
+++ b/src/plugins/fts/fts-plugin.h
@@ -0,0 +1,7 @@
+#ifndef FTS_PLUGIN_H
+#define FTS_PLUGIN_H
+
+void fts_plugin_init(struct module *module);
+void fts_plugin_deinit(void);
+
+#endif
diff --git a/src/plugins/fts/fts-search-args.c b/src/plugins/fts/fts-search-args.c
new file mode 100644
index 0000000..b58b238
--- /dev/null
+++ b/src/plugins/fts/fts-search-args.c
@@ -0,0 +1,258 @@
+/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-namespace.h"
+#include "mail-search.h"
+#include "fts-api-private.h"
+#include "fts-tokenizer.h"
+#include "fts-filter.h"
+#include "fts-user.h"
+#include "fts-search-args.h"
+
+static void strings_deduplicate(ARRAY_TYPE(const_string) *arr)
+{
+	const char *const *strings;
+	unsigned int i, count;
+
+	strings = array_get(arr, &count);
+	for (i = 1; i < count; ) {
+		if (strcmp(strings[i-1], strings[i]) == 0) {
+			array_delete(arr, i, 1);
+			strings = array_get(arr, &count);
+		} else {
+			i++;
+		}
+	}
+}
+
+static struct mail_search_arg *
+fts_search_arg_create_or(const struct mail_search_arg *orig_arg, pool_t pool,
+			 const ARRAY_TYPE(const_string) *tokens)
+{
+	struct mail_search_arg *arg, *or_arg, **argp;
+	const char *token;
+
+	/* create the OR arg first as the parent */
+	or_arg = p_new(pool, struct mail_search_arg, 1);
+	or_arg->type = SEARCH_OR;
+
+	/* now create all the child args for the OR */
+	argp = &or_arg->value.subargs;
+	array_foreach_elem(tokens, token) {
+		arg = p_new(pool, struct mail_search_arg, 1);
+		*arg = *orig_arg;
+		arg->match_not = FALSE; /* we copied this to the root OR */
+		arg->next = NULL;
+		arg->value.str = p_strdup(pool, token);
+
+		*argp = arg;
+		argp = &arg->next;
+	}
+	return or_arg;
+}
+
+static int
+fts_backend_dovecot_expand_tokens(struct fts_filter *filter,
+				  pool_t pool,
+				  struct mail_search_arg *parent_arg,
+				  const struct mail_search_arg *orig_arg,
+				  const char *orig_token, const char *token,
+				  const char **error_r)
+{
+	struct mail_search_arg *arg;
+	ARRAY_TYPE(const_string) tokens;
+	const char *token2, *error;
+	int ret;
+
+	t_array_init(&tokens, 4);
+	/* first add the word exactly as it without any tokenization */
+	array_push_back(&tokens, &orig_token);
+	/* then add it tokenized, but without filtering */
+	array_push_back(&tokens, &token);
+
+	/* add the word filtered */
+	if (filter != NULL) {
+		token2 = t_strdup(token);
+		ret = fts_filter_filter(filter, &token2, &error);
+		if (ret > 0) {
+			token2 = t_strdup(token2);
+			array_push_back(&tokens, &token2);
+		} else if (ret < 0) {
+			*error_r = t_strdup_printf("Couldn't filter search token: %s", error);
+			return -1;
+		} else {
+			/* The filter dropped the token, which means it was
+			   never even indexed. Ignore this word entirely in the
+			   search query. */
+			return 0;
+		}
+	}
+	array_sort(&tokens, i_strcmp_p);
+	strings_deduplicate(&tokens);
+
+	arg = fts_search_arg_create_or(orig_arg, pool, &tokens);
+	arg->next = parent_arg->value.subargs;
+	parent_arg->value.subargs = arg;
+	return 0;
+}
+
+static int
+fts_backend_dovecot_tokenize_lang(struct fts_user_language *user_lang,
+				  pool_t pool, struct mail_search_arg *or_arg,
+				  struct mail_search_arg *orig_arg,
+				  const char *orig_token, const char **error_r)
+{
+	size_t orig_token_len = strlen(orig_token);
+	struct mail_search_arg *and_arg, *orig_or_args = or_arg->value.subargs;
+	const char *token, *error;
+	int ret;
+
+	/* we want all the tokens found from the string to be found, so create
+	   a parent AND and place all the filtered token alternatives under
+	   it */
+	and_arg = p_new(pool, struct mail_search_arg, 1);
+	and_arg->type = SEARCH_SUB;
+	and_arg->next = orig_or_args;
+	or_arg->value.subargs = and_arg;
+
+	/* reset tokenizer between search args in case there's any state left
+	   from some previous failure */
+	fts_tokenizer_reset(user_lang->search_tokenizer);
+	while ((ret = fts_tokenizer_next(user_lang->search_tokenizer,
+					 (const void *)orig_token,
+					 orig_token_len, &token, &error)) > 0) {
+		if (fts_backend_dovecot_expand_tokens(user_lang->filter, pool,
+						      and_arg, orig_arg, orig_token,
+						      token, error_r) < 0)
+			return -1;
+	}
+	while (ret >= 0 &&
+	       (ret = fts_tokenizer_final(user_lang->search_tokenizer, &token, &error)) > 0) {
+		if (fts_backend_dovecot_expand_tokens(user_lang->filter, pool,
+						      and_arg, orig_arg, orig_token,
+						      token, error_r) < 0)
+			return -1;
+	}
+	if (ret < 0) {
+		*error_r = t_strdup_printf("Couldn't tokenize search args: %s", error);
+		return -1;
+	}
+	if (and_arg->value.subargs == NULL) {
+		/* nothing was actually expanded, remove the empty and_arg */
+		or_arg->value.subargs = orig_or_args;
+	}
+	return 0;
+}
+
+static int fts_search_arg_expand(struct fts_backend *backend, pool_t pool,
+				 struct mail_search_arg **argp)
+{
+	const ARRAY_TYPE(fts_user_language) *languages;
+	struct fts_user_language *lang;
+	struct mail_search_arg *or_arg, *orig_arg = *argp;
+	const char *error, *orig_token = orig_arg->value.str;
+
+	if (((*argp)->type == SEARCH_HEADER ||
+	     (*argp)->type == SEARCH_HEADER_ADDRESS ||
+	     (*argp)->type == SEARCH_HEADER_COMPRESS_LWSP) &&
+	    !fts_header_has_language((*argp)->hdr_field_name)) {
+		/* use only the data-language */
+		languages = fts_user_get_data_languages(backend->ns->user);
+	} else {
+		languages = fts_user_get_all_languages(backend->ns->user);
+	}
+
+	/* OR together all the different expansions for different languages.
+	   it's enough for one of them to match. */
+	or_arg = p_new(pool, struct mail_search_arg, 1);
+	or_arg->type = SEARCH_OR;
+	or_arg->match_not = orig_arg->match_not;
+	or_arg->next = orig_arg->next;
+
+	array_foreach_elem(languages, lang) {
+		if (fts_backend_dovecot_tokenize_lang(lang, pool, or_arg,
+						      orig_arg, orig_token, &error) < 0) {
+			i_error("fts: %s", error);
+			return -1;
+		}
+	}
+
+	if (or_arg->value.subargs == NULL) {
+		/* we couldn't parse any tokens from the input */
+		or_arg->type = SEARCH_ALL;
+		or_arg->match_not = !or_arg->match_not;
+	}
+	*argp = or_arg;
+	return 0;
+}
+
+static int
+fts_search_args_expand_tree(struct fts_backend *backend, pool_t pool,
+			    struct mail_search_arg **argp)
+{
+	int ret;
+
+	for (; *argp != NULL; argp = &(*argp)->next) {
+		switch ((*argp)->type) {
+		case SEARCH_OR:
+		case SEARCH_SUB:
+		case SEARCH_INTHREAD:
+			if (fts_search_args_expand_tree(backend, pool,
+							&(*argp)->value.subargs) < 0)
+				return -1;
+			break;
+		case SEARCH_HEADER:
+		case SEARCH_HEADER_ADDRESS:
+		case SEARCH_HEADER_COMPRESS_LWSP:
+			if ((*argp)->value.str[0] == '\0') {
+				/* we're testing for the existence of
+				   the header */
+				break;
+			}
+			/* fall through */
+		case SEARCH_BODY:
+		case SEARCH_TEXT:
+			T_BEGIN {
+				ret = fts_search_arg_expand(backend, pool, argp);
+			} T_END;
+			if (ret < 0)
+				return -1;
+			break;
+		default:
+			break;
+		}
+	}
+	return 0;
+}
+
+int fts_search_args_expand(struct fts_backend *backend,
+			   struct mail_search_args *args)
+{
+	struct mail_search_arg *args_dup, *orig_args = args->args;
+
+	/* don't keep re-expanding every time the search args are used.
+	   this is especially important to avoid an assert-crash in
+	   index_search_result_update_flags(). */
+	if (args->fts_expanded)
+		return 0;
+	args->fts_expanded = TRUE;
+
+	/* duplicate the args, so if expansion fails we haven't changed
+	   anything */
+	args_dup = mail_search_arg_dup(args->pool, args->args);
+
+	if (fts_search_args_expand_tree(backend, args->pool, &args_dup) < 0)
+		return -1;
+
+	/* we'll need to re-simplify the args if we changed anything */
+	args->simplified = FALSE;
+	args->args = args_dup;
+	mail_search_args_simplify(args);
+
+	/* duplicated args aren't initialized */
+	i_assert(args->init_refcount > 0);
+	mail_search_arg_init(args, args_dup);
+	mail_search_arg_deinit(orig_args);
+	return 0;
+}
diff --git a/src/plugins/fts/fts-search-args.h b/src/plugins/fts/fts-search-args.h
new file mode 100644
index 0000000..9fb8923
--- /dev/null
+++ b/src/plugins/fts/fts-search-args.h
@@ -0,0 +1,7 @@
+#ifndef FTS_SEARCH_ARGS_H
+#define FTS_SEARCH_ARGS_H
+
+int fts_search_args_expand(struct fts_backend *backend,
+			   struct mail_search_args *args);
+
+#endif
diff --git a/src/plugins/fts/fts-search-serialize.c b/src/plugins/fts/fts-search-serialize.c
new file mode 100644
index 0000000..e30d4ce
--- /dev/null
+++ b/src/plugins/fts/fts-search-serialize.c
@@ -0,0 +1,99 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "mail-search.h"
+#include "fts-search-serialize.h"
+
+#define HAVE_SUBARGS(arg) \
+	((arg)->type == SEARCH_SUB || (arg)->type == SEARCH_OR)
+
+void fts_search_serialize(buffer_t *buf, const struct mail_search_arg *args)
+{
+	char chr;
+
+	for (; args != NULL; args = args->next) {
+		chr = (args->match_always ? 1 : 0) |
+			(args->nonmatch_always ? 2 : 0);
+		buffer_append_c(buf, chr);
+
+		if (HAVE_SUBARGS(args))
+			fts_search_serialize(buf, args->value.subargs);
+	}
+}
+
+static void fts_search_deserialize_idx(struct mail_search_arg *args,
+				       const buffer_t *buf, unsigned int *idx)
+{
+	const char *data = buf->data;
+
+	for (; args != NULL; args = args->next) {
+		i_assert(*idx < buf->used);
+
+		args->match_always = (data[*idx] & 1) != 0;
+		args->nonmatch_always = (data[*idx] & 2) != 0;
+		args->result = args->match_always ? 1 :
+			(args->nonmatch_always ? 0 : -1);
+		*idx += 1;
+
+		if (HAVE_SUBARGS(args)) {
+			fts_search_deserialize_idx(args->value.subargs,
+						   buf, idx);
+		}
+	}
+}
+
+void fts_search_deserialize(struct mail_search_arg *args,
+			    const buffer_t *buf)
+{
+	unsigned int idx = 0;
+
+	fts_search_deserialize_idx(args, buf, &idx);
+	i_assert(idx == buf->used);
+}
+
+static void
+fts_search_deserialize_add_idx(struct mail_search_arg *args,
+			       const buffer_t *buf, unsigned int *idx,
+			       bool matches)
+{
+	const char *data = buf->data;
+
+	for (; args != NULL; args = args->next) {
+		i_assert(*idx < buf->used);
+
+		if (data[*idx] != 0) {
+			if (matches) {
+				args->match_always = TRUE;
+				args->result = 1;
+			} else {
+				args->nonmatch_always = TRUE;
+				args->result = 0;
+			}
+		}
+		*idx += 1;
+
+		if (HAVE_SUBARGS(args)) {
+			fts_search_deserialize_add_idx(args->value.subargs,
+						       buf, idx, matches);
+		}
+	}
+}
+
+void fts_search_deserialize_add_matches(struct mail_search_arg *args,
+					const buffer_t *buf)
+{
+	unsigned int idx = 0;
+
+	fts_search_deserialize_add_idx(args, buf, &idx, TRUE);
+	i_assert(idx == buf->used);
+}
+
+void fts_search_deserialize_add_nonmatches(struct mail_search_arg *args,
+					   const buffer_t *buf)
+{
+	unsigned int idx = 0;
+
+	fts_search_deserialize_add_idx(args, buf, &idx, FALSE);
+	i_assert(idx == buf->used);
+}
diff --git a/src/plugins/fts/fts-search-serialize.h b/src/plugins/fts/fts-search-serialize.h
new file mode 100644
index 0000000..c1a7d88
--- /dev/null
+++ b/src/plugins/fts/fts-search-serialize.h
@@ -0,0 +1,16 @@
+#ifndef FTS_SEARCH_SERIALIZE_H
+#define FTS_SEARCH_SERIALIZE_H
+
+/* serialize [non]match_always fields (clearing buffer) */
+void fts_search_serialize(buffer_t *buf, const struct mail_search_arg *args);
+/* add/remove [non]match_always fields in search args */
+void fts_search_deserialize(struct mail_search_arg *args,
+			    const buffer_t *buf);
+/* add match_always=TRUE fields to search args */
+void fts_search_deserialize_add_matches(struct mail_search_arg *args,
+					const buffer_t *buf);
+/* add nonmatch_always=TRUE fields to search args */
+void fts_search_deserialize_add_nonmatches(struct mail_search_arg *args,
+					   const buffer_t *buf);
+
+#endif
diff --git a/src/plugins/fts/fts-search.c b/src/plugins/fts/fts-search.c
new file mode 100644
index 0000000..895ea59
--- /dev/null
+++ b/src/plugins/fts/fts-search.c
@@ -0,0 +1,385 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "seq-range-array.h"
+#include "mail-search.h"
+#include "fts-api-private.h"
+#include "fts-search-args.h"
+#include "fts-search-serialize.h"
+#include "fts-storage.h"
+#include "hash.h"
+
+static void
+uid_range_to_seqs(struct fts_search_context *fctx,
+		  const ARRAY_TYPE(seq_range) *uid_range,
+		  ARRAY_TYPE(seq_range) *seq_range)
+{
+	const struct seq_range *range;
+	unsigned int i, count;
+	uint32_t seq1, seq2;
+
+	range = array_get(uid_range, &count);
+	if (!array_is_created(seq_range))
+		p_array_init(seq_range, fctx->result_pool, count);
+	for (i = 0; i < count; i++) {
+		if (range[i].seq1 > range[i].seq2)
+			continue;
+		mailbox_get_seq_range(fctx->box, range[i].seq1, range[i].seq2,
+				      &seq1, &seq2);
+		if (seq1 != 0)
+			seq_range_array_add_range(seq_range, seq1, seq2);
+	}
+}
+
+static int fts_search_lookup_level_single(struct fts_search_context *fctx,
+					  struct mail_search_arg *args,
+					  bool and_args)
+{
+	enum fts_lookup_flags flags = fctx->flags |
+		(and_args ? FTS_LOOKUP_FLAG_AND_ARGS : 0);
+	struct fts_search_level *level;
+	struct fts_result result;
+
+	i_zero(&result);
+	result.search_state = fctx->search_state;
+	result.pool = fctx->result_pool;
+	p_array_init(&result.definite_uids, fctx->result_pool, 32);
+	p_array_init(&result.maybe_uids, fctx->result_pool, 32);
+	p_array_init(&result.scores, fctx->result_pool, 32);
+
+	mail_search_args_reset(args, TRUE);
+	if (fts_backend_lookup(fctx->backend, fctx->box, args, flags,
+			       &result) < 0)
+		return -1;
+
+	fctx->search_state = result.search_state;
+	level = array_append_space(&fctx->levels);
+	level->args_matches = buffer_create_dynamic(fctx->result_pool, 16);
+	fts_search_serialize(level->args_matches, args);
+
+	uid_range_to_seqs(fctx, &result.definite_uids, &level->definite_seqs);
+	uid_range_to_seqs(fctx, &result.maybe_uids, &level->maybe_seqs);
+	level->score_map = result.scores;
+	return 0;
+}
+
+static void
+level_scores_add_vuids(struct mailbox *box,
+		       struct fts_search_level *level, struct fts_result *br)
+{
+	const struct fts_score_map *scores;
+	unsigned int i, count;
+	ARRAY_TYPE(seq_range) backend_uids;
+	ARRAY_TYPE(uint32_t) vuids_arr;
+	const uint32_t *vuids;
+	struct fts_score_map *score;
+
+	scores = array_get(&br->scores, &count);
+	t_array_init(&vuids_arr, count);
+	t_array_init(&backend_uids, 64);
+	for (i = 0; i < count; i++)
+		seq_range_array_add(&backend_uids, scores[i].uid);
+	box->virtual_vfuncs->get_virtual_uid_map(box, br->box,
+						 &backend_uids, &vuids_arr);
+
+	i_assert(array_count(&vuids_arr) == array_count(&br->scores));
+	vuids = array_get(&vuids_arr, &count);
+	for (i = 0; i < count; i++) {
+		score = array_append_space(&level->score_map);
+		score->uid = vuids[i];
+		score->score = scores[i].score;
+	}
+}
+
+static int
+mailbox_cmp_fts_backend(struct mailbox *const *m1, struct mailbox *const *m2)
+{
+	struct fts_backend *b1, *b2;
+
+	b1 = fts_mailbox_backend(*m1);
+	b2 = fts_mailbox_backend(*m2);
+	if (b1 < b2)
+		return -1;
+	if (b1 > b2)
+		return 1;
+	return 0;
+}
+
+static int
+multi_add_lookup_result(struct fts_search_context *fctx,
+			struct fts_search_level *level,
+			struct mail_search_arg *args,
+			struct fts_multi_result *result)
+{
+	ARRAY_TYPE(seq_range) vuids;
+	size_t orig_size;
+	unsigned int i;
+
+	orig_size = level->args_matches->used;
+	fts_search_serialize(level->args_matches, args);
+	if (orig_size > 0) {
+		if (level->args_matches->used != orig_size * 2 ||
+		    memcmp(level->args_matches->data,
+			   CONST_PTR_OFFSET(level->args_matches->data,
+					    orig_size), orig_size) != 0)
+			i_panic("incompatible fts backends for namespaces");
+		buffer_set_used_size(level->args_matches, orig_size);
+	}
+
+	t_array_init(&vuids, 64);
+	for (i = 0; result->box_results[i].box != NULL; i++) {
+		struct fts_result *br = &result->box_results[i];
+
+		array_clear(&vuids);
+		if (array_is_created(&br->definite_uids)) {
+			fctx->box->virtual_vfuncs->get_virtual_uids(fctx->box,
+				br->box, &br->definite_uids, &vuids);
+		}
+		uid_range_to_seqs(fctx, &vuids, &level->definite_seqs);
+
+		array_clear(&vuids);
+		if (array_is_created(&br->maybe_uids)) {
+			fctx->box->virtual_vfuncs->get_virtual_uids(fctx->box,
+				br->box, &br->maybe_uids, &vuids);
+		}
+		uid_range_to_seqs(fctx, &vuids, &level->maybe_seqs);
+
+		if (array_is_created(&br->scores))
+			level_scores_add_vuids(fctx->box, level, br);
+	}
+	return 0;
+}
+
+static int fts_search_lookup_level_multi(struct fts_search_context *fctx,
+					 struct mail_search_arg *args,
+					 bool and_args)
+{
+	enum fts_lookup_flags flags = fctx->flags |
+		(and_args ? FTS_LOOKUP_FLAG_AND_ARGS : 0);
+	ARRAY_TYPE(mailboxes) mailboxes_arr, tmp_mailboxes;
+	struct mailbox *const *mailboxes;
+	struct fts_backend *backend;
+	struct fts_search_level *level;
+	struct fts_multi_result result;
+	unsigned int i, j, mailbox_count;
+
+	p_array_init(&mailboxes_arr, fctx->result_pool, 8);
+	fctx->box->virtual_vfuncs->get_virtual_backend_boxes(fctx->box,
+		&mailboxes_arr, TRUE);
+	array_sort(&mailboxes_arr, mailbox_cmp_fts_backend);
+
+	i_zero(&result);
+	result.search_state = fctx->search_state;
+	result.pool = fctx->result_pool;
+
+	level = array_append_space(&fctx->levels);
+	level->args_matches = buffer_create_dynamic(fctx->result_pool, 16);
+	p_array_init(&level->score_map, fctx->result_pool, 1);
+
+	mailboxes = array_get(&mailboxes_arr, &mailbox_count);
+	t_array_init(&tmp_mailboxes, mailbox_count);
+	for (i = 0; i < mailbox_count; i = j) {
+		array_clear(&tmp_mailboxes);
+		array_push_back(&tmp_mailboxes, &mailboxes[i]);
+
+		backend = fts_mailbox_backend(mailboxes[i]);
+		for (j = i + 1; j < mailbox_count; j++) {
+			if (fts_mailbox_backend(mailboxes[j]) != backend)
+				break;
+			array_push_back(&tmp_mailboxes, &mailboxes[j]);
+		}
+		array_append_zero(&tmp_mailboxes);
+
+		mail_search_args_reset(args, TRUE);
+		if (fts_backend_lookup_multi(backend,
+					     array_front(&tmp_mailboxes),
+					     args, flags, &result) < 0)
+			return -1;
+
+		if (multi_add_lookup_result(fctx, level, args, &result) < 0)
+			return -1;
+	}
+	fctx->search_state = result.search_state;
+	return 0;
+}
+
+static int fts_search_lookup_level(struct fts_search_context *fctx,
+				   struct mail_search_arg *args,
+				   bool and_args)
+{
+	int ret;
+
+	T_BEGIN {
+		ret = !fctx->virtual_mailbox ?
+			fts_search_lookup_level_single(fctx, args, and_args) :
+			fts_search_lookup_level_multi(fctx, args, and_args);
+	} T_END;
+	if (ret < 0)
+		return -1;
+
+	for (; args != NULL; args = args->next) {
+		if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
+			continue;
+
+		if (fts_search_lookup_level(fctx, args->value.subargs,
+					    args->type == SEARCH_SUB) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static void
+fts_search_merge_scores_and(ARRAY_TYPE(fts_score_map) *dest,
+			    const ARRAY_TYPE(fts_score_map) *src)
+{
+	struct fts_score_map *dest_map;
+	const struct fts_score_map *src_map;
+	unsigned int desti, srci, dest_count, src_count;
+
+	dest_map = array_get_modifiable(dest, &dest_count);
+	src_map = array_get(src, &src_count);
+
+	/* arg_scores are summed to current scores. we could drop UIDs that
+	   don't exist in both, but that's just extra work so don't bother */
+	for (desti = srci = 0; desti < dest_count && srci < src_count;) {
+		if (dest_map[desti].uid < src_map[srci].uid)
+			desti++;
+		else if (dest_map[desti].uid > src_map[srci].uid)
+			srci++;
+		else {
+			if (dest_map[desti].score < src_map[srci].score)
+				dest_map[desti].score = src_map[srci].score;
+			desti++; srci++;
+		}
+	}
+}
+
+static void
+fts_search_merge_scores_or(ARRAY_TYPE(fts_score_map) *dest,
+			   const ARRAY_TYPE(fts_score_map) *src)
+{
+	ARRAY_TYPE(fts_score_map) src2;
+	const struct fts_score_map *src_map, *src2_map;
+	unsigned int srci, src2i, src_count, src2_count;
+
+	t_array_init(&src2, array_count(dest));
+	array_append_array(&src2, dest);
+	array_clear(dest);
+
+	src_map = array_get(src, &src_count);
+	src2_map = array_get(&src2, &src2_count);
+
+	/* add any missing UIDs to current scores. if any existing UIDs have
+	   lower scores than in arg_scores, increase them. */
+	for (srci = src2i = 0; srci < src_count || src2i < src2_count;) {
+		if (src2i == src2_count ||
+		    src_map[srci].uid < src2_map[src2i].uid) {
+			array_push_back(dest, &src_map[srci]);
+			srci++;
+		} else if (srci == src_count ||
+			   src_map[srci].uid > src2_map[src2i].uid) {
+			array_push_back(dest, &src2_map[src2i]);
+			src2i++;
+		} else {
+			i_assert(src_map[srci].uid == src2_map[src2i].uid);
+			if (src_map[srci].score > src2_map[src2i].score)
+				array_push_back(dest, &src_map[srci]);
+			else
+				array_push_back(dest, &src2_map[src2i]);
+			srci++; src2i++;
+		}
+	}
+}
+
+static void
+fts_search_merge_scores_level(struct fts_search_context *fctx,
+			      struct mail_search_arg *args, unsigned int *idx,
+			      bool and_args, ARRAY_TYPE(fts_score_map) *scores)
+{
+	const struct fts_search_level *level;
+	ARRAY_TYPE(fts_score_map) arg_scores;
+
+	i_assert(array_count(scores) == 0);
+
+	/*
+	   The (simplified) args can look like:
+
+	   A and B and (C or D) and (E or F) and ...
+	   A or B or (C and D) or (E and F) or ...
+
+	   The A op B part's scores are in level->scores. The child args'
+	   scores are in the sub levels' scores.
+	*/
+
+	level = array_idx(&fctx->levels, *idx);
+	array_append_array(scores, &level->score_map);
+
+	t_array_init(&arg_scores, 64);
+	for (; args != NULL; args = args->next) {
+		if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
+			continue;
+
+		*idx += 1;
+		array_clear(&arg_scores);
+		fts_search_merge_scores_level(fctx, args->value.subargs, idx,
+					      args->type == SEARCH_OR,
+					      &arg_scores);
+
+		if (and_args)
+			fts_search_merge_scores_and(scores, &arg_scores);
+		else
+			fts_search_merge_scores_or(scores, &arg_scores);
+	}
+}
+
+static void fts_search_merge_scores(struct fts_search_context *fctx)
+{
+	unsigned int idx = 0;
+
+	fts_search_merge_scores_level(fctx, fctx->args->args, &idx,
+				      TRUE, &fctx->scores->score_map);
+}
+
+static void fts_search_try_lookup(struct fts_search_context *fctx)
+{
+	uint32_t last_uid, seq1, seq2;
+
+	i_assert(array_count(&fctx->levels) == 0);
+	i_assert(fctx->args->simplified);
+
+	if (fts_backend_refresh(fctx->backend) < 0)
+		return;
+	if (fts_backend_get_last_uid(fctx->backend, fctx->box, &last_uid) < 0)
+		return;
+	mailbox_get_seq_range(fctx->box, last_uid+1, (uint32_t)-1,
+			      &seq1, &seq2);
+	fctx->first_unindexed_seq = seq1 != 0 ? seq1 : (uint32_t)-1;
+
+	if (fctx->virtual_mailbox) {
+		hash_table_clear(fctx->last_indexed_virtual_uids, TRUE);
+		fctx->next_unindexed_seq = fctx->first_unindexed_seq;
+	}
+
+	if ((fctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) {
+		if (fts_search_args_expand(fctx->backend, fctx->args) < 0)
+			return;
+	}
+	fts_search_serialize(fctx->orig_matches, fctx->args->args);
+
+	if (fts_search_lookup_level(fctx, fctx->args->args, TRUE) == 0) {
+		fctx->fts_lookup_success = TRUE;
+		fts_search_merge_scores(fctx);
+	}
+
+	fts_search_deserialize(fctx->args->args, fctx->orig_matches);
+	fts_backend_lookup_done(fctx->backend);
+}
+
+void fts_search_lookup(struct fts_search_context *fctx)
+{
+	struct event_reason *reason = event_reason_begin("fts:lookup");
+	fts_search_try_lookup(fctx);
+	event_reason_end(&reason);
+}
diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c
new file mode 100644
index 0000000..101d52a
--- /dev/null
+++ b/src/plugins/fts/fts-storage.c
@@ -0,0 +1,981 @@
+/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "strescape.h"
+#include "write-full.h"
+#include "mail-search-build.h"
+#include "mail-storage-private.h"
+#include "mailbox-list-private.h"
+#include "fts-api-private.h"
+#include "fts-tokenizer.h"
+#include "fts-indexer.h"
+#include "fts-build-mail.h"
+#include "fts-search-serialize.h"
+#include "fts-plugin.h"
+#include "fts-user.h"
+#include "fts-storage.h"
+#include "hash.h"
+
+
+#define FTS_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, fts_storage_module)
+#define FTS_CONTEXT_REQUIRE(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, fts_storage_module)
+#define FTS_MAIL_CONTEXT(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, fts_mail_module)
+#define FTS_LIST_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, fts_mailbox_list_module)
+#define FTS_LIST_CONTEXT_REQUIRE(obj) \
+	MODULE_CONTEXT_REQUIRE(obj, fts_mailbox_list_module)
+
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
+struct fts_mailbox_list {
+	union mailbox_list_module_context module_ctx;
+	struct fts_backend *backend;
+
+	const char *backend_name;
+	struct fts_backend_update_context *update_ctx;
+	unsigned int update_ctx_refcount;
+
+	bool failed:1;
+};
+
+struct fts_mailbox {
+	union mailbox_module_context module_ctx;
+	struct fts_backend_update_context *sync_update_ctx;
+	bool fts_mailbox_excluded;
+};
+
+struct fts_transaction_context {
+	union mailbox_transaction_module_context module_ctx;
+
+	struct fts_scores *scores;
+	uint32_t next_index_seq;
+	uint32_t highest_virtual_uid;
+	unsigned int precache_extra_count;
+
+	bool indexing:1;
+	bool precached:1;
+	bool mails_saved:1;
+	const char *failure_reason;
+};
+
+struct fts_mail {
+	union mail_module_context module_ctx;
+	char score[30];
+
+	bool virtual_mail:1;
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
+				  &mail_storage_module_register);
+static MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
+static MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module,
+				  &mailbox_list_module_register);
+
+static int fts_mailbox_get_last_cached_seq(struct mailbox *box, uint32_t *seq_r)
+{
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(box->list);
+	uint32_t seq1, seq2, last_uid;
+
+	if (fts_backend_get_last_uid(flist->backend, box, &last_uid) < 0) {
+		mail_storage_set_internal_error(box->storage);
+		return -1;
+	}
+
+	if (last_uid == 0)
+		*seq_r = 0;
+	else {
+		mailbox_get_seq_range(box, 1, last_uid, &seq1, &seq2);
+		*seq_r = seq2;
+	}
+	return 0;
+}
+
+static int
+fts_mailbox_get_status(struct mailbox *box, enum mailbox_status_items items,
+		       struct mailbox_status *status_r)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(box);
+	uint32_t seq;
+
+	if (fbox->module_ctx.super.get_status(box, items, status_r) < 0)
+		return -1;
+
+	if ((items & STATUS_LAST_CACHED_SEQ) != 0) {
+		if (fts_mailbox_get_last_cached_seq(box, &seq) < 0)
+			return -1;
+
+		/* Always use the FTS's last_cached_seq. This is because we
+		   don't want to reindex all mails to FTS if .cache file is
+		   deleted. */
+		status_r->last_cached_seq = seq;
+	}
+	return 0;
+}
+
+
+static void fts_scores_unref(struct fts_scores **_scores)
+{
+	struct fts_scores *scores = *_scores;
+
+	*_scores = NULL;
+	if (--scores->refcount == 0) {
+		array_free(&scores->score_map);
+		i_free(scores);
+	}
+}
+
+static void fts_try_build_init(struct mail_search_context *ctx,
+			       struct fts_search_context *fctx)
+{
+	int ret;
+
+	i_assert(!fts_backend_is_updating(fctx->backend));
+
+	ret = fts_indexer_init(fctx->backend, ctx->transaction->box,
+			       &fctx->indexer_ctx);
+	if (ret < 0)
+		return;
+
+	if (ret == 0) {
+		/* the index was up to date */
+		fts_search_lookup(fctx);
+	} else {
+		/* hide "searching" notifications while building index */
+		ctx->progress_hidden = TRUE;
+	}
+}
+
+static bool fts_want_build_args(const struct mail_search_arg *args)
+{
+	/* we want to update index only when searching from message body.
+	   it's not worth the wait for searching only from headers, which
+	   could be in cache file already */
+	for (; args != NULL; args = args->next) {
+		switch (args->type) {
+		case SEARCH_OR:
+		case SEARCH_SUB:
+		case SEARCH_INTHREAD:
+			if (fts_want_build_args(args->value.subargs))
+				return TRUE;
+			break;
+		case SEARCH_BODY:
+		case SEARCH_TEXT:
+			if (!args->no_fts)
+				return TRUE;
+			break;
+		default:
+			break;
+		}
+	}
+	return FALSE;
+}
+
+static bool fts_args_have_fuzzy(const struct mail_search_arg *args)
+{
+	for (; args != NULL; args = args->next) {
+		if (args->fuzzy)
+			return TRUE;
+		switch (args->type) {
+		case SEARCH_OR:
+		case SEARCH_SUB:
+		case SEARCH_INTHREAD:
+			if (fts_args_have_fuzzy(args->value.subargs))
+				return TRUE;
+			break;
+		default:
+			break;
+		}
+	}
+	return FALSE;
+}
+
+static enum fts_enforced fts_enforced_parse(const char *str)
+{
+	if (str == NULL || strcmp(str, "no") == 0)
+		return FTS_ENFORCED_NO;
+	else if (strcmp(str, "body") == 0)
+		return FTS_ENFORCED_BODY;
+	else
+		return FTS_ENFORCED_YES;
+}
+
+static struct mail_search_context *
+fts_mailbox_search_init(struct mailbox_transaction_context *t,
+			struct mail_search_args *args,
+			const enum mail_sort_type *sort_program,
+			enum mail_fetch_field wanted_fields,
+			struct mailbox_header_lookup_ctx *wanted_headers)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(t);
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(t->box);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(t->box->list);
+	struct mail_search_context *ctx;
+	struct fts_search_context *fctx;
+
+	ctx = fbox->module_ctx.super.search_init(t, args, sort_program,
+						 wanted_fields, wanted_headers);
+
+	if (!fts_backend_can_lookup(flist->backend, args->args))
+		return ctx;
+
+	fctx = i_new(struct fts_search_context, 1);
+	fctx->box = t->box;
+	fctx->backend = flist->backend;
+	fctx->t = t;
+	fctx->args = args;
+	fctx->result_pool = pool_alloconly_create("fts results", 1024*64);
+	fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
+	fctx->virtual_mailbox = t->box->virtual_vfuncs != NULL;
+	if (fctx->virtual_mailbox) {
+		hash_table_create(&fctx->last_indexed_virtual_uids,
+				  default_pool, 0, str_hash, strcmp);
+	}
+	fctx->enforced = fts_enforced_parse(
+		mail_user_plugin_getenv(t->box->storage->user, "fts_enforced"));
+	i_array_init(&fctx->levels, 8);
+	fctx->scores = i_new(struct fts_scores, 1);
+	fctx->scores->refcount = 1;
+	i_array_init(&fctx->scores->score_map, 64);
+	MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
+
+	/* FIXME: we'll assume that all the args are fuzzy. not good,
+	   but would require much more work to fix it. */
+	if (!fts_args_have_fuzzy(args->args) &&
+	    mail_user_plugin_getenv_bool(t->box->storage->user,
+				    "fts_no_autofuzzy"))
+		fctx->flags |= FTS_LOOKUP_FLAG_NO_AUTO_FUZZY;
+	/* transaction contains the last search's scores. they can be
+	   queried later with mail_get_special() */
+	if (ft->scores != NULL)
+		fts_scores_unref(&ft->scores);
+	ft->scores = fctx->scores;
+	ft->scores->refcount++;
+
+	if (fctx->enforced == FTS_ENFORCED_YES ||
+	    fts_want_build_args(args->args))
+		fts_try_build_init(ctx, fctx);
+	else
+		fts_search_lookup(fctx);
+	return ctx;
+}
+
+static bool fts_mailbox_build_continue(struct mail_search_context *ctx)
+{
+	struct fts_search_context *fctx = FTS_CONTEXT_REQUIRE(ctx);
+	int ret;
+
+	ret = fts_indexer_more(fctx->indexer_ctx);
+	if (ret == 0)
+		return FALSE;
+
+	/* indexing finished */
+	ctx->progress_hidden = FALSE;
+	if (fts_indexer_deinit(&fctx->indexer_ctx) < 0)
+		ret = -1;
+	if (ret > 0)
+		fts_search_lookup(fctx);
+	if (ret < 0) {
+		/* if indexing timed out, it probably means that
+		   the mailbox is still being indexed, but it's a large
+		   mailbox and it takes a while. in this situation
+		   we'll simply abort the search.
+
+		   if indexing failed for any other reason, just
+		   fallback to searching the slow way. */
+		fctx->indexing_timed_out =
+			mailbox_get_last_mail_error(fctx->box) == MAIL_ERROR_INUSE;
+	}
+	return TRUE;
+}
+
+static bool
+fts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
+				 struct mail **mail_r, bool *tryagain_r)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box);
+	struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+
+	if (fctx != NULL && fctx->indexer_ctx != NULL) {
+		/* this command is still building the indexes */
+		if (!fts_mailbox_build_continue(ctx)) {
+			*tryagain_r = TRUE;
+			return FALSE;
+		}
+		if (fctx->indexing_timed_out) {
+			*tryagain_r = FALSE;
+			return FALSE;
+		}
+	}
+	if (fctx != NULL && !fctx->fts_lookup_success &&
+	    fctx->enforced != FTS_ENFORCED_NO)
+		return FALSE;
+
+	return fbox->module_ctx.super.
+		search_next_nonblock(ctx, mail_r, tryagain_r);
+}
+
+static void
+fts_search_apply_results_level(struct mail_search_context *ctx,
+			       struct mail_search_arg *args, unsigned int *idx)
+{
+	struct fts_search_context *fctx = FTS_CONTEXT_REQUIRE(ctx);
+	const struct fts_search_level *level;
+
+	level = array_idx(&fctx->levels, *idx);
+
+	if (array_is_created(&level->definite_seqs) &&
+	    seq_range_exists(&level->definite_seqs, ctx->seq))
+		fts_search_deserialize_add_matches(args, level->args_matches);
+	else if (!array_is_created(&level->maybe_seqs) ||
+		 !seq_range_exists(&level->maybe_seqs, ctx->seq))
+		fts_search_deserialize_add_nonmatches(args, level->args_matches);
+
+	for (; args != NULL; args = args->next) {
+		if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
+			continue;
+
+		*idx += 1;
+		fts_search_apply_results_level(ctx, args->value.subargs, idx);
+	}
+}
+
+static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box);
+	struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+	unsigned int idx;
+
+	if (fctx == NULL || !fctx->fts_lookup_success) {
+		/* fts lookup not done for this search */
+		if (fctx != NULL && fctx->indexing_timed_out)
+			return FALSE;
+		return fbox->module_ctx.super.search_next_update_seq(ctx);
+	}
+
+	/* restore original [non]matches */
+	fts_search_deserialize(ctx->args->args, fctx->orig_matches);
+
+	if (!fbox->module_ctx.super.search_next_update_seq(ctx))
+		return FALSE;
+
+	if (ctx->seq >= fctx->first_unindexed_seq) {
+		/* we've not indexed this far */
+		return TRUE;
+	}
+
+	/* apply [non]matches based on the FTS lookup results */
+	idx = 0;
+	fts_search_apply_results_level(ctx, ctx->args->args, &idx);
+	return TRUE;
+}
+
+static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box);
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(ctx->transaction);
+	struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+	int ret = 0;
+
+	if (fctx != NULL) {
+		if (fctx->virtual_mailbox)
+			hash_table_destroy(&fctx->last_indexed_virtual_uids);
+		if (fctx->indexer_ctx != NULL) {
+			if (fts_indexer_deinit(&fctx->indexer_ctx) < 0)
+				ft->failure_reason = "FTS indexing failed";
+		}
+		if (fctx->indexing_timed_out)
+			ret = -1;
+		else if (!fctx->fts_lookup_success &&
+			 fctx->enforced != FTS_ENFORCED_NO) {
+			/* FTS lookup failed and we didn't want to fallback to
+			   opening all the mails and searching manually */
+			mail_storage_set_internal_error(ctx->transaction->box->storage);
+			ret = -1;
+		}
+
+		buffer_free(&fctx->orig_matches);
+		array_free(&fctx->levels);
+		pool_unref(&fctx->result_pool);
+		fts_scores_unref(&fctx->scores);
+		i_free(fctx);
+	}
+	if (fbox->module_ctx.super.search_deinit(ctx) < 0)
+		ret = -1;
+	return ret;
+}
+
+static int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score)
+{
+	return *uid < score->uid ? -1 :
+		(*uid > score->uid ? 1 : 0);
+}
+
+static int fts_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
+				const char **value_r)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(_mail->transaction);
+	const struct fts_score_map *scores;
+
+	if (field != MAIL_FETCH_SEARCH_RELEVANCY || ft->scores == NULL)
+		scores = NULL;
+	else {
+		scores = array_bsearch(&ft->scores->score_map, &_mail->uid,
+				       fts_score_cmp);
+	}
+	if (scores != NULL) {
+		i_assert(scores->uid == _mail->uid);
+		(void)i_snprintf(fmail->score, sizeof(fmail->score),
+				 "%f", scores->score);
+
+		*value_r = fmail->score;
+		return 0;
+	}
+
+	return fmail->module_ctx.super.get_special(_mail, field, value_r);
+}
+
+static int
+fts_mail_precache_range(struct mailbox_transaction_context *trans,
+			struct fts_backend_update_context *update_ctx,
+			uint32_t seq1, uint32_t seq2, unsigned int *extra_count)
+{
+	struct mail_search_args *search_args;
+	struct mail_search_context *ctx;
+	struct mail *mail;
+	int ret = 0;
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_seqset(search_args, seq1, seq2);
+	ctx = mailbox_search_init(trans, search_args, NULL,
+				  MAIL_FETCH_STREAM_HEADER |
+				  MAIL_FETCH_STREAM_BODY, NULL);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(ctx, &mail)) {
+		if (fts_build_mail(update_ctx, mail) < 0) {
+			ret = -1;
+			break;
+		}
+		if (mail_precache(mail) < 0) {
+			ret = -1;
+			break;
+		}
+		*extra_count += 1;
+	}
+	if (mailbox_search_deinit(&ctx) < 0)
+		ret = -1;
+	return ret;
+}
+
+static int fts_mail_precache_init(struct mail *_mail)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(_mail->transaction);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(_mail->box->list);
+	uint32_t last_seq;
+
+	if (fts_mailbox_get_last_cached_seq(_mail->box, &last_seq) < 0) {
+		ft->failure_reason = "Failed to lookup last indexed FTS mail";
+		return -1;
+	}
+
+	ft->precached = TRUE;
+	ft->next_index_seq = last_seq + 1;
+	if (flist->update_ctx == NULL)
+		flist->update_ctx = fts_backend_update_init(flist->backend);
+	flist->update_ctx_refcount++;
+	return 0;
+}
+
+static int fts_mail_index(struct mail *_mail)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(_mail->transaction);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(_mail->box->list);
+	struct mail_private *pmail = (struct mail_private *)_mail;
+
+	if (ft->failure_reason != NULL)
+		return -1;
+
+	if (!ft->precached) {
+		if (fts_mail_precache_init(_mail) < 0)
+			return -1;
+	}
+	if (pmail->vmail != NULL) {
+		/* Indexing via virtual mailbox: Index all the mails in this
+		   same real mailbox. */
+		uint32_t msgs_count =
+			mail_index_view_get_messages_count(_mail->box->view);
+
+		fts_backend_update_set_mailbox(flist->update_ctx, _mail->box);
+		if (ft->next_index_seq > msgs_count) {
+			/* everything indexed already */
+			return 0;
+		} else if (fts_mail_precache_range(_mail->transaction,
+						   flist->update_ctx,
+						   ft->next_index_seq,
+						   msgs_count,
+						   &ft->precache_extra_count) < 0) {
+			return -1;
+		} else {
+			ft->next_index_seq = msgs_count+1;
+			return 0;
+		}
+	}
+
+	if (ft->next_index_seq < _mail->seq) {
+		/* we'll first need to index all the missing mails up to the
+		   current one. */
+		fts_backend_update_set_mailbox(flist->update_ctx, _mail->box);
+		if (fts_mail_precache_range(_mail->transaction,
+					    flist->update_ctx,
+					    ft->next_index_seq,
+					    _mail->seq-1,
+					    &ft->precache_extra_count) < 0)
+			return -1;
+		ft->next_index_seq = _mail->seq;
+	}
+
+	if (ft->next_index_seq == _mail->seq) {
+		fts_backend_update_set_mailbox(flist->update_ctx, _mail->box);
+		if (fts_build_mail(flist->update_ctx, _mail) < 0)
+			return -1;
+		ft->next_index_seq = _mail->seq + 1;
+	}
+	return 0;
+}
+
+static int fts_mail_precache(struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(_mail->transaction);
+	int ret = 0;
+
+	fmail->module_ctx.super.precache(_mail);
+	if (fmail->virtual_mail) {
+		if (ft->highest_virtual_uid < _mail->uid)
+			ft->highest_virtual_uid = _mail->uid;
+	} else if (!ft->indexing) T_BEGIN {
+		/* avoid recursing here from fts_mail_precache_range() */
+		struct event_reason *reason =
+			event_reason_begin("fts:index");
+		ft->indexing = TRUE;
+		ret = fts_mail_index(_mail);
+		i_assert(ft->indexing);
+		ft->indexing = FALSE;
+		event_reason_end(&reason);
+	} T_END;
+	return ret;
+}
+
+void fts_mail_allocated(struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	struct mail_vfuncs *v = mail->vlast;
+	struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
+	struct fts_mail *fmail;
+
+	if (fbox == NULL)
+		return;
+
+	fmail = p_new(mail->pool, struct fts_mail, 1);
+	fmail->module_ctx.super = *v;
+	mail->vlast = &fmail->module_ctx.super;
+	fmail->virtual_mail = _mail->box->virtual_vfuncs != NULL;
+
+	v->get_special = fts_mail_get_special;
+	v->precache = fts_mail_precache;
+	MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
+}
+
+static struct mailbox_transaction_context *
+fts_transaction_begin(struct mailbox *box,
+		      enum mailbox_transaction_flags flags,
+		      const char *reason)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(box);
+	struct mailbox_transaction_context *t;
+	struct fts_transaction_context *ft;
+
+	ft = i_new(struct fts_transaction_context, 1);
+
+	t = fbox->module_ctx.super.transaction_begin(box, flags, reason);
+	MODULE_CONTEXT_SET(t, fts_storage_module, ft);
+	return t;
+}
+
+static int fts_transaction_end(struct mailbox_transaction_context *t, const char **error_r)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(t);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(t->box->list);
+	int ret = 0;
+
+	if (ft->failure_reason != NULL) {
+		*error_r = t_strdup(ft->failure_reason);
+		ret = -1;
+	}
+
+	struct event_reason *reason = event_reason_begin("fts:index");
+	if (ft->precached) {
+		i_assert(flist->update_ctx_refcount > 0);
+		if (--flist->update_ctx_refcount == 0) {
+			if (fts_backend_update_deinit(&flist->update_ctx) < 0) {
+				ret = -1;
+				*error_r = "backend deinit";
+			}
+		}
+	} else if (ft->highest_virtual_uid > 0) {
+		if (fts_index_set_last_uid(t->box, ft->highest_virtual_uid) < 0) {
+			ret = -1;
+			*error_r = "index last uid setting";
+		}
+	}
+	if (ft->scores != NULL)
+		fts_scores_unref(&ft->scores);
+	if (ft->precache_extra_count > 0) {
+		if (ret < 0) {
+			i_error("fts: Failed after indexing %u extra mails internally in %s: %s",
+			       ft->precache_extra_count, t->box->vname, *error_r);
+		} else {
+			i_info("fts: Indexed %u extra mails internally in %s",
+			       ft->precache_extra_count, t->box->vname);
+		}
+	}
+	event_reason_end(&reason);
+	i_free(ft);
+	return ret;
+}
+
+static void fts_transaction_rollback(struct mailbox_transaction_context *t)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(t->box);
+	const char *error;
+
+	(void)fts_transaction_end(t, &error);
+	fbox->module_ctx.super.transaction_rollback(t);
+}
+
+static void fts_queue_index(struct mailbox *box)
+{
+	struct mail_user *user = box->storage->user;
+	string_t *str = t_str_new(256);
+	const char *path, *value;
+	unsigned int max_recent_msgs;
+	int fd;
+
+	path = t_strconcat(user->set->base_dir, "/"INDEXER_SOCKET_NAME, NULL);
+	fd = net_connect_unix(path);
+	if (fd == -1) {
+		i_error("net_connect_unix(%s) failed: %m", path);
+		return;
+	}
+
+	value = mail_user_plugin_getenv(user, "fts_autoindex_max_recent_msgs");
+	if (value == NULL || str_to_uint(value, &max_recent_msgs) < 0)
+		max_recent_msgs = 0;
+
+	str_append(str, INDEXER_HANDSHAKE);
+	str_append(str, "APPEND\t0\t");
+	str_append_tabescaped(str, user->username);
+	str_append_c(str, '\t');
+	str_append_tabescaped(str, box->vname);
+	str_printfa(str, "\t%u", max_recent_msgs);
+	str_append_c(str, '\t');
+	str_append_tabescaped(str, box->storage->user->session_id);
+	str_append_c(str, '\n');
+	if (write_full(fd, str_data(str), str_len(str)) < 0)
+		i_error("write(%s) failed: %m", path);
+	i_close_fd(&fd);
+}
+
+static int
+fts_transaction_commit(struct mailbox_transaction_context *t,
+		       struct mail_transaction_commit_changes *changes_r)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(t);
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(t->box);
+	struct mailbox *box = t->box;
+	bool autoindex;
+	int ret = 0;
+	const char *error;
+
+	autoindex = ft->mails_saved && !fbox->fts_mailbox_excluded &&
+		mail_user_plugin_getenv_bool(box->storage->user,
+					"fts_autoindex");
+
+	if (fts_transaction_end(t, &error) < 0) {
+		mail_storage_set_error(t->box->storage, MAIL_ERROR_TEMP,
+				       t_strdup_printf("FTS transaction commit failed: %s",
+						       error));
+		ret = -1;
+	}
+	if (fbox->module_ctx.super.transaction_commit(t, changes_r) < 0)
+		ret = -1;
+	if (ret < 0)
+		return -1;
+
+	if (autoindex)
+		fts_queue_index(box);
+	return 0;
+}
+
+static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
+				    enum mailbox_sync_type sync_type)
+{
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(box->list);
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(box);
+
+	if (fbox->module_ctx.super.sync_notify != NULL)
+		fbox->module_ctx.super.sync_notify(box, uid, sync_type);
+
+	if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE) {
+		if (uid == 0 && fbox->sync_update_ctx != NULL) {
+			/* this sync is finished */
+			(void)fts_backend_update_deinit(&fbox->sync_update_ctx);
+		}
+		return;
+	}
+
+	if (fbox->sync_update_ctx == NULL) {
+		if (fts_backend_is_updating(flist->backend)) {
+			/* FIXME: maildir workaround - we could get here
+			   because we're building an index, which doesn't find
+			   some mail and starts syncing the mailbox.. */
+			return;
+		}
+		fbox->sync_update_ctx = fts_backend_update_init(flist->backend);
+		fts_backend_update_set_mailbox(fbox->sync_update_ctx, box);
+	}
+	fts_backend_update_expunge(fbox->sync_update_ctx, uid);
+}
+
+static int fts_sync_deinit(struct mailbox_sync_context *ctx,
+			   struct mailbox_sync_status *status_r)
+{
+	struct mailbox *box = ctx->box;
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(box);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
+	bool optimize;
+	int ret = 0;
+
+	optimize = (ctx->flags & (MAILBOX_SYNC_FLAG_FORCE_RESYNC |
+				  MAILBOX_SYNC_FLAG_OPTIMIZE)) != 0;
+	if (fbox->module_ctx.super.sync_deinit(ctx, status_r) < 0)
+		return -1;
+	ctx = NULL;
+
+	if (optimize) {
+		i_assert(flist != NULL);
+		if (fts_backend_optimize(flist->backend) < 0) {
+			mailbox_set_critical(box, "FTS optimize failed");
+			ret = -1;
+		}
+	}
+	return ret;
+}
+
+static int fts_save_finish(struct mail_save_context *ctx)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(ctx->transaction);
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box);
+
+	if (fbox->module_ctx.super.save_finish(ctx) < 0)
+		return -1;
+	ft->mails_saved = TRUE;
+	return 0;
+}
+
+static int fts_copy(struct mail_save_context *ctx, struct mail *mail)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(ctx->transaction);
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box);
+
+	if (fbox->module_ctx.super.copy(ctx, mail) < 0)
+		return -1;
+	ft->mails_saved = TRUE;
+	return 0;
+}
+
+static void fts_mailbox_virtual_match_mail(struct mail_search_context *ctx,
+					   struct mail *mail)
+{
+	struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+	unsigned int idx, be_last_uid;
+
+	if (fctx == NULL || !fctx->fts_lookup_success || !fctx->virtual_mailbox ||
+	    ctx->seq < fctx->first_unindexed_seq)
+		return;
+	/* Table of last indexed UID per backend mailbox */
+	HASH_TABLE_TYPE(virtual_last_indexed) hash_tbl =
+		fctx->last_indexed_virtual_uids;
+
+	struct mail *backend_mail;
+	if (mail->box->mail_vfuncs->get_backend_mail(mail, &backend_mail) < 0)
+		return;
+	const char *box_name = backend_mail->box->vname;
+	/* Get the last indexed UID in the backend mailbox */
+	void *uid_value =
+		hash_table_lookup(fctx->last_indexed_virtual_uids, box_name);
+	if (uid_value == NULL) {
+		/* This backend's last indexed uid is not yet inserted to the table */
+		struct fts_mailbox_list *flist =
+			FTS_LIST_CONTEXT(backend_mail->box->list);
+		if (flist == NULL || flist->failed ||
+		    mailbox_open(backend_mail->box) < 0 ||
+		    fts_backend_get_last_uid(flist->backend, backend_mail->box,
+					     &be_last_uid) < 0) {
+			be_last_uid = 0;
+		} else {
+			const char *vname_copy =
+				p_strdup(fctx->result_pool, backend_mail->box->vname);
+			hash_table_insert(hash_tbl, vname_copy,
+					  POINTER_CAST(be_last_uid + 1));
+		}
+	} else {
+		be_last_uid = POINTER_CAST_TO(uid_value, uint32_t) - 1;
+	}
+	if (backend_mail->uid <= be_last_uid) {
+		/* Mail was already indexed in the backend mailbox.
+		   Apply [non]matches based on the FTS lookup results */
+		struct fts_transaction_context *ft = FTS_CONTEXT_REQUIRE(ctx->transaction);
+
+		if (fctx->next_unindexed_seq == mail->seq) {
+			fctx->next_unindexed_seq++;
+			ft->highest_virtual_uid = mail->uid;
+		}
+		idx = 0;
+		fts_search_apply_results_level(ctx, ctx->args->args, &idx);
+	} else {
+		fctx->virtual_seen_unindexed_gaps = TRUE;
+	}
+}
+
+static int fts_mailbox_search_next_match_mail(struct mail_search_context *ctx,
+					      struct mail *mail)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT_REQUIRE(ctx->transaction->box);
+
+	fts_mailbox_virtual_match_mail(ctx, mail);
+	return fbox->module_ctx.super.search_next_match_mail(ctx, mail);
+}
+
+void fts_mailbox_allocated(struct mailbox *box)
+{
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
+	struct mailbox_vfuncs *v = box->vlast;
+	struct fts_mailbox *fbox;
+
+	if (flist == NULL || flist->failed)
+		return;
+
+	fbox = p_new(box->pool, struct fts_mailbox, 1);
+	fbox->module_ctx.super = *v;
+	box->vlast = &fbox->module_ctx.super;
+	fbox->fts_mailbox_excluded = fts_user_autoindex_exclude(box);
+
+	v->get_status = fts_mailbox_get_status;
+	v->search_init = fts_mailbox_search_init;
+	v->search_next_nonblock = fts_mailbox_search_next_nonblock;
+	v->search_next_update_seq = fts_mailbox_search_next_update_seq;
+	v->search_deinit = fts_mailbox_search_deinit;
+	v->transaction_begin = fts_transaction_begin;
+	v->transaction_rollback = fts_transaction_rollback;
+	v->transaction_commit = fts_transaction_commit;
+	v->sync_notify = fts_mailbox_sync_notify;
+	v->sync_deinit = fts_sync_deinit;
+	v->save_finish = fts_save_finish;
+	v->copy = fts_copy;
+	v->search_next_match_mail = fts_mailbox_search_next_match_mail;
+
+	MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
+}
+
+static void fts_mailbox_list_deinit(struct mailbox_list *list)
+{
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(list);
+
+	if (flist->backend != NULL)
+		fts_backend_deinit(&flist->backend);
+	flist->module_ctx.super.deinit(list);
+}
+
+static int
+fts_init_namespace(struct fts_mailbox_list *flist, struct mail_namespace *ns,
+		   const char **error_r)
+{
+	struct fts_backend *backend;
+	if (fts_backend_init(flist->backend_name, ns, error_r, &backend) < 0) {
+		flist->failed = TRUE;
+		return -1;
+	}
+	flist->backend = backend;
+	if ((flist->backend->flags & FTS_BACKEND_FLAG_FUZZY_SEARCH) != 0)
+		ns->user->fuzzy_search = TRUE;
+	return 0;
+}
+
+void fts_mail_namespaces_added(struct mail_namespace *ns)
+{
+	while(ns != NULL) {
+		struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(ns->list);
+		const char *error;
+
+		if (flist != NULL && !flist->failed && flist->backend == NULL &&
+		    fts_init_namespace(flist, ns, &error) < 0) {
+			i_error("fts: Failed to initialize backend '%s': %s",
+				flist->backend_name, error);
+		}
+		ns = ns->next;
+	}
+}
+
+void
+fts_mailbox_list_created(struct mailbox_list *list)
+{
+	const char *name = mail_user_plugin_getenv(list->ns->user, "fts");
+	const char *path;
+
+	if (name == NULL || name[0] == '\0') {
+		e_debug(list->ns->user->event,
+			"fts: No fts setting - plugin disabled");
+		return;
+	}
+
+	if (!mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_INDEX, &path)) {
+		e_debug(list->ns->user->event,
+			"fts: Indexes disabled for namespace '%s'",
+			list->ns->prefix);
+		return;
+	}
+
+	struct fts_mailbox_list *flist;
+	struct mailbox_list_vfuncs *v = list->vlast;
+
+	flist = p_new(list->pool, struct fts_mailbox_list, 1);
+	flist->module_ctx.super = *v;
+	flist->backend_name = name;
+	list->vlast = &flist->module_ctx.super;
+	v->deinit = fts_mailbox_list_deinit;
+	MODULE_CONTEXT_SET(list, fts_mailbox_list_module, flist);
+}
+
+struct fts_backend *fts_mailbox_backend(struct mailbox *box)
+{
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT_REQUIRE(box->list);
+
+	return flist->backend;
+}
+
+struct fts_backend *fts_list_backend(struct mailbox_list *list)
+{
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);
+
+	return flist == NULL ? NULL : flist->backend;
+}
diff --git a/src/plugins/fts/fts-storage.h b/src/plugins/fts/fts-storage.h
new file mode 100644
index 0000000..ea28ed2
--- /dev/null
+++ b/src/plugins/fts/fts-storage.h
@@ -0,0 +1,70 @@
+#ifndef FTS_STORAGE_H
+#define FTS_STORAGE_H
+
+#include "mail-storage-private.h"
+#include "fts-api.h"
+
+enum fts_enforced {
+	FTS_ENFORCED_NO,
+	FTS_ENFORCED_YES,
+	FTS_ENFORCED_BODY,
+};
+
+struct fts_scores {
+	int refcount;
+	ARRAY_TYPE(fts_score_map) score_map;
+};
+
+struct fts_search_level {
+	ARRAY_TYPE(seq_range) definite_seqs, maybe_seqs;
+	buffer_t *args_matches;
+	ARRAY_TYPE(fts_score_map) score_map;
+};
+
+HASH_TABLE_DEFINE_TYPE(virtual_last_indexed, const char *, void *);
+
+struct fts_search_context {
+	union mail_search_module_context module_ctx;
+
+	struct fts_backend *backend;
+	struct mailbox *box;
+	struct mailbox_transaction_context *t;
+	struct mail_search_args *args;
+	enum fts_lookup_flags flags;
+	enum fts_enforced enforced;
+
+	pool_t result_pool;
+	ARRAY(struct fts_search_level) levels;
+	buffer_t *orig_matches;
+
+	uint32_t first_unindexed_seq;
+	uint32_t next_unindexed_seq;
+	HASH_TABLE_TYPE(virtual_last_indexed) last_indexed_virtual_uids;
+
+	/* final scores, combined from all levels */
+	struct fts_scores *scores;
+
+	struct fts_indexer_context *indexer_ctx;
+	struct fts_search_state *search_state;
+
+	bool virtual_mailbox:1;
+	bool fts_lookup_success:1;
+	bool indexing_timed_out:1;
+	bool virtual_seen_unindexed_gaps:1;
+};
+
+/* Figure out if we want to use full text search indexes and update
+   backends in fctx accordingly. */
+void fts_search_analyze(struct fts_search_context *fctx);
+/* Perform the actual index lookup and update definite_uids and maybe_uids. */
+void fts_search_lookup(struct fts_search_context *fctx);
+/* Returns FTS backend for the given mailbox (assumes it has one). */
+struct fts_backend *fts_mailbox_backend(struct mailbox *box);
+/* Returns FTS backend for the given mailbox list, or NULL if it has none. */
+struct fts_backend *fts_list_backend(struct mailbox_list *list);
+
+void fts_mail_allocated(struct mail *mail);
+void fts_mail_namespaces_added(struct mail_namespace *ns);
+void fts_mailbox_allocated(struct mailbox *box);
+void fts_mailbox_list_created(struct mailbox_list *list);
+#endif
diff --git a/src/plugins/fts/fts-user.c b/src/plugins/fts/fts-user.c
new file mode 100644
index 0000000..3c813cd
--- /dev/null
+++ b/src/plugins/fts/fts-user.c
@@ -0,0 +1,423 @@
+/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "module-context.h"
+#include "mail-user.h"
+#include "mail-storage-private.h"
+#include "mailbox-match-plugin.h"
+#include "fts-language.h"
+#include "fts-filter.h"
+#include "fts-tokenizer.h"
+#include "fts-user.h"
+
+#define FTS_USER_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, fts_user_module)
+
+struct fts_user {
+	union mail_user_module_context module_ctx;
+	int refcount;
+
+	struct fts_language_list *lang_list;
+	struct fts_user_language *data_lang;
+	ARRAY_TYPE(fts_user_language) languages, data_languages;
+
+	struct mailbox_match_plugin *autoindex_exclude;
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(fts_user_module,
+				  &mail_user_module_register);
+
+static const char *const *str_keyvalues_to_array(const char *str)
+{
+	const char *key, *value, *const *keyvalues;
+	ARRAY_TYPE(const_string) arr;
+	unsigned int i;
+
+	if (str == NULL)
+		return NULL;
+
+	t_array_init(&arr, 8);
+	keyvalues = t_strsplit_spaces(str, " ");
+	for (i = 0; keyvalues[i] != NULL; i++) {
+		value = strchr(keyvalues[i], '=');
+		if (value != NULL)
+			key = t_strdup_until(keyvalues[i], value++);
+		else {
+			key = keyvalues[i];
+			value = "";
+		}
+		array_push_back(&arr, &key);
+		array_push_back(&arr, &value);
+	}
+	array_append_zero(&arr);
+	return array_front(&arr);
+}
+
+static int
+fts_user_init_languages(struct mail_user *user, struct fts_user *fuser,
+			const char **error_r)
+{
+	const char *languages, *unknown;
+	const char *lang_config[3] = {NULL, NULL, NULL};
+
+	languages = mail_user_plugin_getenv(user, "fts_languages");
+	if (languages == NULL) {
+		*error_r = "fts_languages setting is missing";
+		return -1;
+	}
+
+	lang_config[1] = mail_user_plugin_getenv(user, "fts_language_config");
+	if (lang_config[1] != NULL)
+		lang_config[0] = "fts_language_config";
+	if (fts_language_list_init(lang_config, &fuser->lang_list, error_r) < 0)
+		return -1;
+
+	if (!fts_language_list_add_names(fuser->lang_list, languages, &unknown)) {
+		*error_r = t_strdup_printf(
+			"fts_languages: Unknown language '%s'", unknown);
+		return -1;
+	}
+	if (array_count(fts_language_list_get_all(fuser->lang_list)) == 0) {
+		*error_r = "fts_languages setting is empty";
+		return -1;
+	}
+	return 0;
+}
+
+static int
+fts_user_create_filters(struct mail_user *user, const struct fts_language *lang,
+			struct fts_filter **filter_r, const char **error_r)
+{
+	const struct fts_filter *filter_class;
+	struct fts_filter *filter = NULL, *parent = NULL;
+	const char *filters_key, *const *filters, *filter_set_name;
+	const char *str, *error, *set_key;
+	unsigned int i;
+	int ret = 0;
+
+	/* try to get the language-specific filters first */
+	filters_key = t_strconcat("fts_filters_", lang->name, NULL);
+	str = mail_user_plugin_getenv(user, filters_key);
+	if (str == NULL) {
+		/* fallback to global filters */
+		filters_key = "fts_filters";
+		str = mail_user_plugin_getenv(user, filters_key);
+		if (str == NULL) {
+			/* No filters */
+			*filter_r = NULL;
+			return 0;
+		}
+	}
+
+	filters = t_strsplit_spaces(str, " ");
+	for (i = 0; filters[i] != NULL; i++) {
+		filter_class = fts_filter_find(filters[i]);
+		if (filter_class == NULL) {
+			*error_r = t_strdup_printf("%s: Unknown filter '%s'",
+						   filters_key, filters[i]);
+			ret = -1;
+			break;
+		}
+
+		/* try the language-specific setting first */
+		filter_set_name = t_str_replace(filters[i], '-', '_');
+		set_key = t_strdup_printf("fts_filter_%s_%s",
+					  lang->name, filter_set_name);
+		str = mail_user_plugin_getenv(user, set_key);
+		if (str == NULL) {
+			set_key = t_strdup_printf("fts_filter_%s", filter_set_name);
+			str = mail_user_plugin_getenv(user, set_key);
+		}
+
+		if (fts_filter_create(filter_class, parent, lang,
+				      str_keyvalues_to_array(str),
+				      &filter, &error) < 0) {
+			*error_r = t_strdup_printf("%s: %s", set_key, error);
+			ret = -1;
+			break;
+		}
+		if (parent != NULL)
+			fts_filter_unref(&parent);
+		parent = filter;
+	}
+	if (ret < 0) {
+		if (parent != NULL)
+			fts_filter_unref(&parent);
+		return -1;
+	}
+	*filter_r = filter;
+	return 0;
+}
+
+static int
+fts_user_create_tokenizer(struct mail_user *user,
+			  const struct fts_language *lang,
+			  struct fts_tokenizer **tokenizer_r, bool search,
+			  const char **error_r)
+{
+	const struct fts_tokenizer *tokenizer_class;
+	struct fts_tokenizer *tokenizer = NULL, *parent = NULL;
+	const char *tokenizers_key, *const *tokenizers, *tokenizer_set_name;
+	const char *str, *error, *set_key;
+	unsigned int i;
+	int ret = 0;
+
+	tokenizers_key = t_strconcat("fts_tokenizers_", lang->name, NULL);
+	str = mail_user_plugin_getenv(user, tokenizers_key);
+	if (str == NULL) {
+		str = mail_user_plugin_getenv(user, "fts_tokenizers");
+		if (str == NULL) {
+			*error_r = t_strdup_printf("%s or fts_tokenizers setting must exist", tokenizers_key);
+			return -1;
+		}
+		tokenizers_key = "fts_tokenizers";
+	}
+
+	tokenizers = t_strsplit_spaces(str, " ");
+
+	for (i = 0; tokenizers[i] != NULL; i++) {
+		tokenizer_class = fts_tokenizer_find(tokenizers[i]);
+		if (tokenizer_class == NULL) {
+			*error_r = t_strdup_printf("%s: Unknown tokenizer '%s'",
+						   tokenizers_key, tokenizers[i]);
+			ret = -1;
+			break;
+		}
+
+		tokenizer_set_name = t_str_replace(tokenizers[i], '-', '_');
+		set_key = t_strdup_printf("fts_tokenizer_%s_%s", tokenizer_set_name, lang->name);
+		str = mail_user_plugin_getenv(user, set_key);
+		if (str == NULL) {
+			set_key = t_strdup_printf("fts_tokenizer_%s", tokenizer_set_name);
+			str = mail_user_plugin_getenv(user, set_key);
+		}
+
+		/* tell the tokenizers that we're tokenizing a search string
+		   (instead of tokenizing indexed data) */
+		if (search)
+			str = t_strconcat("search=yes ", str, NULL);
+
+		if (fts_tokenizer_create(tokenizer_class, parent,
+					 str_keyvalues_to_array(str),
+					 &tokenizer, &error) < 0) {
+			*error_r = t_strdup_printf("%s: %s", set_key, error);
+			ret = -1;
+			break;
+		}
+		if (parent != NULL)
+			fts_tokenizer_unref(&parent);
+		parent = tokenizer;
+	}
+	if (ret < 0) {
+		if (parent != NULL)
+			fts_tokenizer_unref(&parent);
+		return -1;
+	}
+	*tokenizer_r = tokenizer;
+	return 0;
+}
+
+static int
+fts_user_language_init_tokenizers(struct mail_user *user,
+				  struct fts_user_language *user_lang,
+				  const char **error_r)
+{
+	if (fts_user_create_tokenizer(user, user_lang->lang,
+				      &user_lang->index_tokenizer, FALSE,
+	                              error_r) < 0)
+		return -1;
+
+	if (fts_user_create_tokenizer(user, user_lang->lang,
+				      &user_lang->search_tokenizer, TRUE,
+	                              error_r) < 0)
+		return -1;
+	return 0;
+}
+
+struct fts_user_language *
+fts_user_language_find(struct mail_user *user,
+		       const struct fts_language *lang)
+{
+	struct fts_user_language *user_lang;
+	struct fts_user *fuser = FTS_USER_CONTEXT(user);
+		
+	i_assert(fuser != NULL);
+	array_foreach_elem(&fuser->languages, user_lang) {
+		if (strcmp(user_lang->lang->name, lang->name) == 0)
+			return user_lang;
+	}
+	return NULL;
+}
+
+static int fts_user_language_create(struct mail_user *user,
+                                    struct fts_user *fuser,
+				    const struct fts_language *lang,
+				    const char **error_r)
+{
+	struct fts_user_language *user_lang;
+
+	user_lang = p_new(user->pool, struct fts_user_language, 1);
+	user_lang->lang = lang;
+	array_push_back(&fuser->languages, &user_lang);
+
+	if (fts_user_language_init_tokenizers(user, user_lang, error_r) < 0)
+		return -1;
+	if (fts_user_create_filters(user, lang, &user_lang->filter, error_r) < 0)
+		return -1;
+	return 0;
+}
+
+static int fts_user_languages_fill_all(struct mail_user *user,
+                                       struct fts_user *fuser,
+                                       const char **error_r)
+{
+	const struct fts_language *lang;
+
+	array_foreach_elem(fts_language_list_get_all(fuser->lang_list), lang) {
+		if (fts_user_language_create(user, fuser, lang, error_r) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int
+fts_user_init_data_language(struct mail_user *user, struct fts_user *fuser,
+			    const char **error_r)
+{
+	struct fts_user_language *user_lang;
+	const char *error;
+
+	user_lang = p_new(user->pool, struct fts_user_language, 1);
+	user_lang->lang = &fts_language_data;
+
+	if (fts_user_language_init_tokenizers(user, user_lang, error_r) < 0)
+		return -1;
+
+	if (fts_filter_create(fts_filter_lowercase, NULL, user_lang->lang, NULL,
+			      &user_lang->filter, &error) < 0)
+		i_unreached();
+	i_assert(user_lang->filter != NULL);
+
+	p_array_init(&fuser->data_languages, user->pool, 1);
+	array_push_back(&fuser->data_languages, &user_lang);
+	array_push_back(&fuser->languages, &user_lang);
+
+	fuser->data_lang = user_lang;
+	return 0;
+}
+
+struct fts_language_list *fts_user_get_language_list(struct mail_user *user)
+{
+	struct fts_user *fuser = FTS_USER_CONTEXT(user);
+
+	i_assert(fuser != NULL);
+	return fuser->lang_list;
+}
+
+const ARRAY_TYPE(fts_user_language) *
+fts_user_get_all_languages(struct mail_user *user)
+{
+	struct fts_user *fuser = FTS_USER_CONTEXT(user);
+
+	i_assert(fuser != NULL);
+	return &fuser->languages;
+}
+
+const ARRAY_TYPE(fts_user_language) *
+fts_user_get_data_languages(struct mail_user *user)
+{
+	struct fts_user *fuser = FTS_USER_CONTEXT(user);
+
+	i_assert(fuser != NULL);
+	return &fuser->data_languages;
+}
+
+struct fts_user_language *fts_user_get_data_lang(struct mail_user *user)
+{
+	struct fts_user *fuser = FTS_USER_CONTEXT(user);
+
+	i_assert(fuser != NULL);
+	return fuser->data_lang;
+}
+
+bool fts_user_autoindex_exclude(struct mailbox *box)
+{
+	struct fts_user *fuser = FTS_USER_CONTEXT(box->storage->user);
+
+	return mailbox_match_plugin_exclude(fuser->autoindex_exclude, box);
+}
+
+static void fts_user_language_free(struct fts_user_language *user_lang)
+{
+	if (user_lang->filter != NULL)
+		fts_filter_unref(&user_lang->filter);
+	if (user_lang->index_tokenizer != NULL)
+		fts_tokenizer_unref(&user_lang->index_tokenizer);
+	if (user_lang->search_tokenizer != NULL)
+		fts_tokenizer_unref(&user_lang->search_tokenizer);
+}
+
+static void fts_user_free(struct fts_user *fuser)
+{
+	struct fts_user_language *user_lang;
+
+	if (fuser->lang_list != NULL)
+		fts_language_list_deinit(&fuser->lang_list);
+
+	if (array_is_created(&fuser->languages)) {
+		array_foreach_elem(&fuser->languages, user_lang)
+			fts_user_language_free(user_lang);
+	}
+	mailbox_match_plugin_deinit(&fuser->autoindex_exclude);
+}
+
+static int
+fts_mail_user_init_libfts(struct mail_user *user, struct fts_user *fuser,
+			  const char **error_r)
+{
+	p_array_init(&fuser->languages, user->pool, 4);
+
+	if (fts_user_init_languages(user, fuser, error_r) < 0 ||
+	    fts_user_init_data_language(user, fuser, error_r) < 0)
+		return -1;
+	if (fts_user_languages_fill_all(user, fuser, error_r) < 0)
+		return -1;
+	return 0;
+}
+
+int fts_mail_user_init(struct mail_user *user, bool initialize_libfts,
+		       const char **error_r)
+{
+	struct fts_user *fuser = FTS_USER_CONTEXT(user);
+
+	if (fuser != NULL) {
+		/* multiple fts plugins are loaded */
+		fuser->refcount++;
+		return 0;
+	}
+
+	fuser = p_new(user->pool, struct fts_user, 1);
+	fuser->refcount = 1;
+	if (initialize_libfts) {
+		if (fts_mail_user_init_libfts(user, fuser, error_r) < 0) {
+			fts_user_free(fuser);
+			return -1;
+		}
+	}
+	fuser->autoindex_exclude =
+		mailbox_match_plugin_init(user, "fts_autoindex_exclude");
+
+	MODULE_CONTEXT_SET(user, fts_user_module, fuser);
+	return 0;
+}
+
+void fts_mail_user_deinit(struct mail_user *user)
+{
+	struct fts_user *fuser = FTS_USER_CONTEXT(user);
+
+	if (fuser != NULL) {
+		i_assert(fuser->refcount > 0);
+		if (--fuser->refcount == 0)
+			fts_user_free(fuser);
+	}
+}
diff --git a/src/plugins/fts/fts-user.h b/src/plugins/fts/fts-user.h
new file mode 100644
index 0000000..043f4e1
--- /dev/null
+++ b/src/plugins/fts/fts-user.h
@@ -0,0 +1,27 @@
+#ifndef FTS_USER_H
+#define FTS_USER_H
+
+struct fts_user_language {
+	const struct fts_language *lang;
+	struct fts_filter *filter;
+	struct fts_tokenizer *index_tokenizer, *search_tokenizer;
+};
+ARRAY_DEFINE_TYPE(fts_user_language, struct fts_user_language *);
+
+struct fts_user_language *
+fts_user_language_find(struct mail_user *user,
+                       const struct fts_language *lang);
+struct fts_language_list *fts_user_get_language_list(struct mail_user *user);
+const ARRAY_TYPE(fts_user_language) *
+fts_user_get_all_languages(struct mail_user *user);
+struct fts_user_language *fts_user_get_data_lang(struct mail_user *user);
+const ARRAY_TYPE(fts_user_language) *
+fts_user_get_data_languages(struct mail_user *user);
+
+bool fts_user_autoindex_exclude(struct mailbox *box);
+
+int fts_mail_user_init(struct mail_user *user, bool initialize_libfts,
+		       const char **error_r);
+void fts_mail_user_deinit(struct mail_user *user);
+
+#endif
diff --git a/src/plugins/fts/xml2text.c b/src/plugins/fts/xml2text.c
new file mode 100644
index 0000000..f3c573c
--- /dev/null
+++ b/src/plugins/fts/xml2text.c
@@ -0,0 +1,44 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "message-parser.h"
+#include "fts-parser.h"
+
+#include <unistd.h>
+
+int main(void)
+{
+	struct fts_parser *parser;
+	unsigned char buf[IO_BLOCK_SIZE];
+	struct message_block block;
+	ssize_t ret;
+	struct fts_parser_context parser_context = {.content_type = "text/html"};
+
+	lib_init();
+
+	parser = fts_parser_html.try_init(&parser_context);
+	i_assert(parser != NULL);
+
+	i_zero(&block);
+	while ((ret = read(STDIN_FILENO, buf, sizeof(buf))) > 0) {
+		block.data = buf;
+		block.size = ret;
+		parser->v.more(parser, &block);
+		if (write(STDOUT_FILENO, block.data, block.size) < 0)
+			i_fatal("write(stdout) failed: %m");
+	}
+	if (ret < 0)
+		i_fatal("read(stdin) failed: %m");
+
+	for (;;) {
+		block.size = 0;
+		parser->v.more(parser, &block);
+		if (block.size == 0)
+			break;
+		if (write(STDOUT_FILENO, block.data, block.size) < 0)
+			i_fatal("write(stdout) failed: %m");
+	}
+
+	lib_deinit();
+	return 0;
+}