summaryrefslogtreecommitdiffstats
path: root/src/lib-fs
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/lib-fs/Makefile.am70
-rw-r--r--src/lib-fs/Makefile.in955
-rw-r--r--src/lib-fs/fs-api-private.h213
-rw-r--r--src/lib-fs/fs-api.c1407
-rw-r--r--src/lib-fs/fs-api.h397
-rw-r--r--src/lib-fs/fs-dict.c372
-rw-r--r--src/lib-fs/fs-metawrap.c526
-rw-r--r--src/lib-fs/fs-posix.c1028
-rw-r--r--src/lib-fs/fs-randomfail.c555
-rw-r--r--src/lib-fs/fs-sis-common.c59
-rw-r--r--src/lib-fs/fs-sis-common.h14
-rw-r--r--src/lib-fs/fs-sis-queue.c210
-rw-r--r--src/lib-fs/fs-sis.c369
-rw-r--r--src/lib-fs/fs-test-async.c103
-rw-r--r--src/lib-fs/fs-test.c443
-rw-r--r--src/lib-fs/fs-test.h45
-rw-r--r--src/lib-fs/fs-wrapper.c172
-rw-r--r--src/lib-fs/fs-wrapper.h40
-rw-r--r--src/lib-fs/istream-fs-file.c61
-rw-r--r--src/lib-fs/istream-fs-file.h13
-rw-r--r--src/lib-fs/istream-fs-stats.c47
-rw-r--r--src/lib-fs/istream-fs-stats.h9
-rw-r--r--src/lib-fs/istream-metawrap.c152
-rw-r--r--src/lib-fs/istream-metawrap.h14
-rw-r--r--src/lib-fs/ostream-cmp.c95
-rw-r--r--src/lib-fs/ostream-cmp.h15
-rw-r--r--src/lib-fs/ostream-metawrap.c71
-rw-r--r--src/lib-fs/ostream-metawrap.h8
-rw-r--r--src/lib-fs/test-fs-metawrap.c103
-rw-r--r--src/lib-fs/test-fs-posix.c144
30 files changed, 7710 insertions, 0 deletions
diff --git a/src/lib-fs/Makefile.am b/src/lib-fs/Makefile.am
new file mode 100644
index 0000000..e5ae056
--- /dev/null
+++ b/src/lib-fs/Makefile.am
@@ -0,0 +1,70 @@
+noinst_LTLIBRARIES = libfs.la
+
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-dict \
+ -I$(top_srcdir)/src/lib-ssl-iostream \
+ -DMODULE_DIR=\""$(moduledir)"\"
+
+libfs_la_SOURCES = \
+ fs-api.c \
+ fs-dict.c \
+ fs-metawrap.c \
+ fs-randomfail.c \
+ fs-posix.c \
+ fs-test.c \
+ fs-test-async.c \
+ fs-sis.c \
+ fs-sis-common.c \
+ fs-sis-queue.c \
+ fs-wrapper.c \
+ istream-fs-file.c \
+ istream-fs-stats.c \
+ istream-metawrap.c \
+ ostream-metawrap.c \
+ ostream-cmp.c
+
+headers = \
+ fs-api.h \
+ fs-api-private.h \
+ fs-sis-common.h \
+ fs-wrapper.h \
+ fs-test.h \
+ istream-fs-file.h \
+ istream-fs-stats.h \
+ istream-metawrap.h \
+ ostream-metawrap.h \
+ ostream-cmp.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+
+noinst_PROGRAMS = $(test_programs)
+
+test_programs = \
+ test-fs-metawrap \
+ test-fs-posix
+
+test_deps = \
+ $(noinst_LTLIBRARIES) \
+ ../lib-dict/libdict.la \
+ ../lib-test/libtest.la \
+ ../lib/liblib.la
+
+test_libs = \
+ $(test_deps) \
+ $(MODULE_LIBS)
+
+test_fs_metawrap_SOURCES = test-fs-metawrap.c
+test_fs_metawrap_LDADD = $(test_libs)
+test_fs_metawrap_DEPENDENCIES = $(test_deps)
+
+test_fs_posix_SOURCES = test-fs-posix.c
+test_fs_posix_LDADD = $(test_libs)
+test_fs_posix_DEPENDENCIES = $(test_deps)
+
+check-local:
+ for bin in $(test_programs); do \
+ if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+ done
diff --git a/src/lib-fs/Makefile.in b/src/lib-fs/Makefile.in
new file mode 100644
index 0000000..c31cc22
--- /dev/null
+++ b/src/lib-fs/Makefile.in
@@ -0,0 +1,955 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+ if test -z '$(MAKELEVEL)'; then \
+ false; \
+ elif test -n '$(MAKE_HOST)'; then \
+ true; \
+ elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+ true; \
+ else \
+ false; \
+ fi; \
+}
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+noinst_PROGRAMS = $(am__EXEEXT_1)
+subdir = src/lib-fs
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+ $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+ $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+ $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+ $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+ $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+ $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+ $(top_srcdir)/m4/flexible_array_member.m4 \
+ $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+ $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+ $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+ $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+ $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+ $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+ $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+ $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+ $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+ $(top_srcdir)/m4/pr_set_dumpable.m4 \
+ $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+ $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+ $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+ $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+ $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+ $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+ $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+ $(top_srcdir)/m4/typeof_dev_t.m4 \
+ $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+ $(top_srcdir)/m4/want_apparmor.m4 \
+ $(top_srcdir)/m4/want_bsdauth.m4 \
+ $(top_srcdir)/m4/want_bzlib.m4 \
+ $(top_srcdir)/m4/want_cassandra.m4 \
+ $(top_srcdir)/m4/want_cdb.m4 \
+ $(top_srcdir)/m4/want_checkpassword.m4 \
+ $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+ $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+ $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+ $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+ $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+ $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+ $(top_srcdir)/m4/want_prefetch.m4 \
+ $(top_srcdir)/m4/want_shadow.m4 \
+ $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+ $(top_srcdir)/m4/want_sqlite.m4 \
+ $(top_srcdir)/m4/want_stemmer.m4 \
+ $(top_srcdir)/m4/want_systemd.m4 \
+ $(top_srcdir)/m4/want_textcat.m4 \
+ $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+ $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \
+ $(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__EXEEXT_1 = test-fs-metawrap$(EXEEXT) test-fs-posix$(EXEEXT)
+PROGRAMS = $(noinst_PROGRAMS)
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+libfs_la_LIBADD =
+am_libfs_la_OBJECTS = fs-api.lo fs-dict.lo fs-metawrap.lo \
+ fs-randomfail.lo fs-posix.lo fs-test.lo fs-test-async.lo \
+ fs-sis.lo fs-sis-common.lo fs-sis-queue.lo fs-wrapper.lo \
+ istream-fs-file.lo istream-fs-stats.lo istream-metawrap.lo \
+ ostream-metawrap.lo ostream-cmp.lo
+libfs_la_OBJECTS = $(am_libfs_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 =
+am_test_fs_metawrap_OBJECTS = test-fs-metawrap.$(OBJEXT)
+test_fs_metawrap_OBJECTS = $(am_test_fs_metawrap_OBJECTS)
+am__DEPENDENCIES_1 =
+am__DEPENDENCIES_2 = $(test_deps) $(am__DEPENDENCIES_1)
+am_test_fs_posix_OBJECTS = test-fs-posix.$(OBJEXT)
+test_fs_posix_OBJECTS = $(am_test_fs_posix_OBJECTS)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/fs-api.Plo ./$(DEPDIR)/fs-dict.Plo \
+ ./$(DEPDIR)/fs-metawrap.Plo ./$(DEPDIR)/fs-posix.Plo \
+ ./$(DEPDIR)/fs-randomfail.Plo ./$(DEPDIR)/fs-sis-common.Plo \
+ ./$(DEPDIR)/fs-sis-queue.Plo ./$(DEPDIR)/fs-sis.Plo \
+ ./$(DEPDIR)/fs-test-async.Plo ./$(DEPDIR)/fs-test.Plo \
+ ./$(DEPDIR)/fs-wrapper.Plo ./$(DEPDIR)/istream-fs-file.Plo \
+ ./$(DEPDIR)/istream-fs-stats.Plo \
+ ./$(DEPDIR)/istream-metawrap.Plo ./$(DEPDIR)/ostream-cmp.Plo \
+ ./$(DEPDIR)/ostream-metawrap.Plo \
+ ./$(DEPDIR)/test-fs-metawrap.Po ./$(DEPDIR)/test-fs-posix.Po
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+ $(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo " CC " $@;
+am__v_CC_1 =
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo " CCLD " $@;
+am__v_CCLD_1 =
+SOURCES = $(libfs_la_SOURCES) $(test_fs_metawrap_SOURCES) \
+ $(test_fs_posix_SOURCES)
+DIST_SOURCES = $(libfs_la_SOURCES) $(test_fs_metawrap_SOURCES) \
+ $(test_fs_posix_SOURCES)
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+ test -z "$$files" \
+ || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+ $(am__cd) "$$dir" && rm -f $$files; }; \
+ }
+am__installdirs = "$(DESTDIR)$(pkginc_libdir)"
+HEADERS = $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates. Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+ BEGIN { nonempty = 0; } \
+ { items[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique. This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+ list='$(am__tagged_files)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+noinst_LTLIBRARIES = libfs.la
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-dict \
+ -I$(top_srcdir)/src/lib-ssl-iostream \
+ -DMODULE_DIR=\""$(moduledir)"\"
+
+libfs_la_SOURCES = \
+ fs-api.c \
+ fs-dict.c \
+ fs-metawrap.c \
+ fs-randomfail.c \
+ fs-posix.c \
+ fs-test.c \
+ fs-test-async.c \
+ fs-sis.c \
+ fs-sis-common.c \
+ fs-sis-queue.c \
+ fs-wrapper.c \
+ istream-fs-file.c \
+ istream-fs-stats.c \
+ istream-metawrap.c \
+ ostream-metawrap.c \
+ ostream-cmp.c
+
+headers = \
+ fs-api.h \
+ fs-api-private.h \
+ fs-sis-common.h \
+ fs-wrapper.h \
+ fs-test.h \
+ istream-fs-file.h \
+ istream-fs-stats.h \
+ istream-metawrap.h \
+ ostream-metawrap.h \
+ ostream-cmp.h
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+test_programs = \
+ test-fs-metawrap \
+ test-fs-posix
+
+test_deps = \
+ $(noinst_LTLIBRARIES) \
+ ../lib-dict/libdict.la \
+ ../lib-test/libtest.la \
+ ../lib/liblib.la
+
+test_libs = \
+ $(test_deps) \
+ $(MODULE_LIBS)
+
+test_fs_metawrap_SOURCES = test-fs-metawrap.c
+test_fs_metawrap_LDADD = $(test_libs)
+test_fs_metawrap_DEPENDENCIES = $(test_deps)
+test_fs_posix_SOURCES = test-fs-posix.c
+test_fs_posix_LDADD = $(test_libs)
+test_fs_posix_DEPENDENCIES = $(test_deps)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-fs/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --foreign src/lib-fs/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstPROGRAMS:
+ @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \
+ echo " rm -f" $$list; \
+ rm -f $$list || exit $$?; \
+ test -n "$(EXEEXT)" || exit 0; \
+ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+ echo " rm -f" $$list; \
+ rm -f $$list
+
+clean-noinstLTLIBRARIES:
+ -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+ @list='$(noinst_LTLIBRARIES)'; \
+ locs=`for p in $$list; do echo $$p; done | \
+ sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+ sort -u`; \
+ test -z "$$locs" || { \
+ echo rm -f $${locs}; \
+ rm -f $${locs}; \
+ }
+
+libfs.la: $(libfs_la_OBJECTS) $(libfs_la_DEPENDENCIES) $(EXTRA_libfs_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(LINK) $(libfs_la_OBJECTS) $(libfs_la_LIBADD) $(LIBS)
+
+test-fs-metawrap$(EXEEXT): $(test_fs_metawrap_OBJECTS) $(test_fs_metawrap_DEPENDENCIES) $(EXTRA_test_fs_metawrap_DEPENDENCIES)
+ @rm -f test-fs-metawrap$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(test_fs_metawrap_OBJECTS) $(test_fs_metawrap_LDADD) $(LIBS)
+
+test-fs-posix$(EXEEXT): $(test_fs_posix_OBJECTS) $(test_fs_posix_DEPENDENCIES) $(EXTRA_test_fs_posix_DEPENDENCIES)
+ @rm -f test-fs-posix$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(test_fs_posix_OBJECTS) $(test_fs_posix_LDADD) $(LIBS)
+
+mostlyclean-compile:
+ -rm -f *.$(OBJEXT)
+
+distclean-compile:
+ -rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-api.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-dict.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-metawrap.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-posix.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-randomfail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-sis-common.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-sis-queue.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-sis.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-test-async.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-test.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-wrapper.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-fs-file.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-fs-stats.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-metawrap.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ostream-cmp.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ostream-metawrap.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-fs-metawrap.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-fs-posix.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+ @$(MKDIR_P) $(@D)
+ @echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+ -rm -f *.lo
+
+clean-libtool:
+ -rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+ @$(NORMAL_INSTALL)
+ @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+ fi; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+ $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+ done
+
+uninstall-pkginc_libHEADERS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+ $(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ set x; \
+ here=`pwd`; \
+ $(am__define_uniq_tagged_files); \
+ shift; \
+ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+ test -n "$$unique" || unique=$$empty_fix; \
+ if test $$# -gt 0; then \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ "$$@" $$unique; \
+ else \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ $$unique; \
+ fi; \
+ fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ $(am__define_uniq_tagged_files); \
+ test -z "$(CTAGS_ARGS)$$unique" \
+ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+ $$unique
+
+GTAGS:
+ here=`$(am__cd) $(top_builddir) && pwd` \
+ && $(am__cd) $(top_srcdir) \
+ && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+ list='$(am__tagged_files)'; \
+ case "$(srcdir)" in \
+ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+ *) sdir=$(subdir)/$(srcdir) ;; \
+ esac; \
+ for i in $$list; do \
+ if test -f "$$i"; then \
+ echo "$(subdir)/$$i"; \
+ else \
+ echo "$$sdir/$$i"; \
+ fi; \
+ done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+ -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+ $(MAKE) $(AM_MAKEFLAGS) check-local
+check: check-am
+all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS)
+installdirs:
+ for dir in "$(DESTDIR)$(pkginc_libdir)"; do \
+ test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+ done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+ if test -z '$(STRIP)'; then \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ install; \
+ else \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+ fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+ clean-noinstPROGRAMS mostlyclean-am
+
+distclean: distclean-am
+ -rm -f ./$(DEPDIR)/fs-api.Plo
+ -rm -f ./$(DEPDIR)/fs-dict.Plo
+ -rm -f ./$(DEPDIR)/fs-metawrap.Plo
+ -rm -f ./$(DEPDIR)/fs-posix.Plo
+ -rm -f ./$(DEPDIR)/fs-randomfail.Plo
+ -rm -f ./$(DEPDIR)/fs-sis-common.Plo
+ -rm -f ./$(DEPDIR)/fs-sis-queue.Plo
+ -rm -f ./$(DEPDIR)/fs-sis.Plo
+ -rm -f ./$(DEPDIR)/fs-test-async.Plo
+ -rm -f ./$(DEPDIR)/fs-test.Plo
+ -rm -f ./$(DEPDIR)/fs-wrapper.Plo
+ -rm -f ./$(DEPDIR)/istream-fs-file.Plo
+ -rm -f ./$(DEPDIR)/istream-fs-stats.Plo
+ -rm -f ./$(DEPDIR)/istream-metawrap.Plo
+ -rm -f ./$(DEPDIR)/ostream-cmp.Plo
+ -rm -f ./$(DEPDIR)/ostream-metawrap.Plo
+ -rm -f ./$(DEPDIR)/test-fs-metawrap.Po
+ -rm -f ./$(DEPDIR)/test-fs-posix.Po
+ -rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+ distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+ -rm -f ./$(DEPDIR)/fs-api.Plo
+ -rm -f ./$(DEPDIR)/fs-dict.Plo
+ -rm -f ./$(DEPDIR)/fs-metawrap.Plo
+ -rm -f ./$(DEPDIR)/fs-posix.Plo
+ -rm -f ./$(DEPDIR)/fs-randomfail.Plo
+ -rm -f ./$(DEPDIR)/fs-sis-common.Plo
+ -rm -f ./$(DEPDIR)/fs-sis-queue.Plo
+ -rm -f ./$(DEPDIR)/fs-sis.Plo
+ -rm -f ./$(DEPDIR)/fs-test-async.Plo
+ -rm -f ./$(DEPDIR)/fs-test.Plo
+ -rm -f ./$(DEPDIR)/fs-wrapper.Plo
+ -rm -f ./$(DEPDIR)/istream-fs-file.Plo
+ -rm -f ./$(DEPDIR)/istream-fs-stats.Plo
+ -rm -f ./$(DEPDIR)/istream-metawrap.Plo
+ -rm -f ./$(DEPDIR)/ostream-cmp.Plo
+ -rm -f ./$(DEPDIR)/ostream-metawrap.Plo
+ -rm -f ./$(DEPDIR)/test-fs-metawrap.Po
+ -rm -f ./$(DEPDIR)/test-fs-posix.Po
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+ mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS
+
+.MAKE: check-am install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \
+ check-local clean clean-generic clean-libtool \
+ clean-noinstLTLIBRARIES clean-noinstPROGRAMS cscopelist-am \
+ ctags ctags-am distclean distclean-compile distclean-generic \
+ distclean-libtool distclean-tags distdir dvi dvi-am html \
+ html-am info info-am install install-am install-data \
+ install-data-am install-dvi install-dvi-am install-exec \
+ install-exec-am install-html install-html-am install-info \
+ install-info-am install-man install-pdf install-pdf-am \
+ install-pkginc_libHEADERS install-ps install-ps-am \
+ install-strip installcheck installcheck-am installdirs \
+ maintainer-clean maintainer-clean-generic mostlyclean \
+ mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
+ pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \
+ uninstall-pkginc_libHEADERS
+
+.PRECIOUS: Makefile
+
+
+check-local:
+ for bin in $(test_programs); do \
+ if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+ done
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h
new file mode 100644
index 0000000..76eb2c5
--- /dev/null
+++ b/src/lib-fs/fs-api-private.h
@@ -0,0 +1,213 @@
+#ifndef FS_API_PRIVATE_H
+#define FS_API_PRIVATE_H
+
+#include "fs-api.h"
+#include "fs-wrapper.h"
+#include "module-context.h"
+
+#include <sys/time.h>
+
+#define FS_EVENT_FIELD_FS "lib-fs#fs"
+#define FS_EVENT_FIELD_FILE "lib-fs#file"
+#define FS_EVENT_FIELD_ITER "lib-fs#iter"
+
+enum fs_get_metadata_flags {
+ FS_GET_METADATA_FLAG_LOADED_ONLY = BIT(0),
+};
+
+struct fs_api_module_register {
+ unsigned int id;
+};
+
+union fs_api_module_context {
+ struct fs_api_module_register *reg;
+};
+
+extern struct fs_api_module_register fs_api_module_register;
+
+struct fs_vfuncs {
+ struct fs *(*alloc)(void);
+ int (*init)(struct fs *fs, const char *args,
+ const struct fs_settings *set, const char **error_r);
+ void (*deinit)(struct fs *fs);
+ void (*free)(struct fs *fs);
+
+ enum fs_properties (*get_properties)(struct fs *fs);
+
+ struct fs_file *(*file_alloc)(void);
+ void (*file_init)(struct fs_file *file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags);
+ void (*file_deinit)(struct fs_file *file);
+ void (*file_close)(struct fs_file *file);
+ const char *(*get_path)(struct fs_file *file);
+
+ void (*set_async_callback)(struct fs_file *file,
+ fs_file_async_callback_t *callback,
+ void *context);
+ void (*wait_async)(struct fs *fs);
+
+ void (*set_metadata)(struct fs_file *file, const char *key,
+ const char *value);
+ int (*get_metadata)(struct fs_file *file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r);
+
+ bool (*prefetch)(struct fs_file *file, uoff_t length);
+ ssize_t (*read)(struct fs_file *file, void *buf, size_t size);
+ struct istream *(*read_stream)(struct fs_file *file,
+ size_t max_buffer_size);
+
+ int (*write)(struct fs_file *file, const void *data, size_t size);
+ void (*write_stream)(struct fs_file *file);
+ /* After write_stream_finish() is called once, all the following
+ (async) calls will have success==TRUE. */
+ int (*write_stream_finish)(struct fs_file *file, bool success);
+
+ int (*lock)(struct fs_file *file, unsigned int secs,
+ struct fs_lock **lock_r);
+ void (*unlock)(struct fs_lock *lock);
+
+ int (*exists)(struct fs_file *file);
+ int (*stat)(struct fs_file *file, struct stat *st_r);
+ int (*copy)(struct fs_file *src, struct fs_file *dest);
+ int (*rename)(struct fs_file *src, struct fs_file *dest);
+ int (*delete_file)(struct fs_file *file);
+
+ struct fs_iter *(*iter_alloc)(void);
+ void (*iter_init)(struct fs_iter *iter, const char *path,
+ enum fs_iter_flags flags);
+ const char *(*iter_next)(struct fs_iter *iter);
+ int (*iter_deinit)(struct fs_iter *iter);
+
+ bool (*switch_ioloop)(struct fs *fs);
+ int (*get_nlinks)(struct fs_file *file, nlink_t *nlinks_r);
+};
+
+struct fs {
+ struct fs *parent; /* for wrapper filesystems */
+ const char *name;
+ struct fs_vfuncs v;
+ char *temp_path_prefix;
+ int refcount;
+
+ char *username, *session_id;
+
+ struct fs_settings set;
+
+ /* may be used by fs_wait_async() to do the waiting */
+ struct ioloop *wait_ioloop, *prev_ioloop;
+
+ unsigned int files_open_count;
+ struct fs_file *files;
+ struct fs_iter *iters;
+ struct event *event;
+
+ struct fs_stats stats;
+
+ ARRAY(union fs_api_module_context *) module_contexts;
+};
+
+struct fs_file {
+ /* linked list of all files */
+ struct fs_file *prev, *next;
+
+ struct fs_file *parent; /* for wrapper filesystems */
+ struct fs *fs;
+ struct ostream *output;
+ struct event *event;
+ char *path;
+ char *last_error;
+ enum fs_open_flags flags;
+
+ struct istream *seekable_input;
+ struct istream *pending_read_input;
+
+ const struct hash_method *write_digest_method;
+ void *write_digest;
+
+ pool_t metadata_pool;
+ ARRAY_TYPE(fs_metadata) metadata;
+
+ struct fs_file *copy_src;
+ struct istream *copy_input;
+ struct ostream *copy_output;
+
+ struct timeval timing_start[FS_OP_COUNT];
+
+ bool write_pending:1;
+ bool writing_stream:1;
+ bool metadata_changed:1;
+
+ bool read_or_prefetch_counted:1;
+ bool lookup_metadata_counted:1;
+ bool stat_counted:1;
+ bool copy_counted:1;
+ bool istream_open:1;
+ bool last_error_changed:1;
+};
+
+struct fs_lock {
+ struct fs_file *file;
+};
+
+struct fs_iter {
+ /* linked list of all iters */
+ struct fs_iter *prev, *next;
+
+ struct fs *fs;
+ struct event *event;
+ char *path;
+ enum fs_iter_flags flags;
+ struct timeval start_time;
+ char *last_error;
+
+ bool async_have_more;
+ fs_file_async_callback_t *async_callback;
+ void *async_context;
+};
+
+extern const struct fs fs_class_dict;
+extern const struct fs fs_class_posix;
+extern const struct fs fs_class_randomfail;
+extern const struct fs fs_class_metawrap;
+extern const struct fs fs_class_sis;
+extern const struct fs fs_class_sis_queue;
+extern const struct fs fs_class_test;
+
+void fs_class_register(const struct fs *fs_class);
+
+/* Event must be fs_file or fs_iter events. Set errno from err. */
+void fs_set_error(struct event *event, int err,
+ const char *fmt, ...) ATTR_FORMAT(3, 4);
+/* Like fs_set_error(), but use the existing errno. */
+void fs_set_error_errno(struct event *event, const char *fmt, ...) ATTR_FORMAT(2, 3);
+void fs_file_set_error_async(struct fs_file *file);
+
+ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size);
+int fs_write_via_stream(struct fs_file *file, const void *data, size_t size);
+void fs_metadata_init(struct fs_file *file);
+void fs_metadata_init_or_clear(struct fs_file *file);
+void fs_default_set_metadata(struct fs_file *file,
+ const char *key, const char *value);
+int fs_get_metadata_full(struct fs_file *file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r);
+const char *fs_metadata_find(const ARRAY_TYPE(fs_metadata) *metadata,
+ const char *key);
+int fs_default_copy(struct fs_file *src, struct fs_file *dest);
+
+void fs_file_timing_end(struct fs_file *file, enum fs_op op);
+
+struct fs_file *
+fs_file_init_parent(struct fs_file *parent, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags);
+struct fs_iter *
+fs_iter_init_parent(struct fs_iter *parent,
+ const char *path, enum fs_iter_flags flags);
+void fs_file_free(struct fs_file *file);
+
+/* Same as fs_write_stream_abort_error(), except it closes the *parent* file
+ and error is left untouched */
+void fs_write_stream_abort_parent(struct fs_file *file, struct ostream **output);
+
+#endif
diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
new file mode 100644
index 0000000..1b47ded
--- /dev/null
+++ b/src/lib-fs/fs-api.c
@@ -0,0 +1,1407 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "module-dir.h"
+#include "llist.h"
+#include "str.h"
+#include "hash-method.h"
+#include "istream.h"
+#include "istream-seekable.h"
+#include "ostream.h"
+#include "stats-dist.h"
+#include "time-util.h"
+#include "istream-fs-stats.h"
+#include "fs-api-private.h"
+
+static struct event_category event_category_fs = {
+ .name = "fs"
+};
+
+struct fs_api_module_register fs_api_module_register = { 0 };
+
+static struct module *fs_modules = NULL;
+static ARRAY(const struct fs *) fs_classes;
+
+static void fs_classes_init(void);
+
+static struct event *fs_create_event(struct fs *fs, struct event *parent)
+{
+ struct event *event;
+
+ event = event_create(parent);
+ event_add_category(event, &event_category_fs);
+ event_set_append_log_prefix(event,
+ t_strdup_printf("fs-%s: ", fs->name));
+ return event;
+}
+
+static int
+fs_alloc(const struct fs *fs_class, const char *args,
+ const struct fs_settings *set, struct fs **fs_r, const char **error_r)
+{
+ struct fs *fs;
+ const char *error;
+ int ret;
+
+ fs = fs_class->v.alloc();
+ fs->refcount = 1;
+ fs->set.debug = set->debug;
+ fs->set.enable_timing = set->enable_timing;
+ i_array_init(&fs->module_contexts, 5);
+ fs->event = fs_create_event(fs, set->event_parent);
+ event_set_forced_debug(fs->event, fs->set.debug);
+
+ T_BEGIN {
+ ret = fs_class->v.init(fs, args, set, &error);
+ } T_END_PASS_STR_IF(ret < 0, &error);
+ if (ret < 0) {
+ /* a bit kludgy way to allow data stack frame usage in normal
+ conditions but still be able to return error message from
+ data stack. */
+ *error_r = t_strdup_printf("%s: %s", fs_class->name, error);
+ fs_unref(&fs);
+ return -1;
+ }
+ fs->username = i_strdup(set->username);
+ fs->session_id = i_strdup(set->session_id);
+ *fs_r = fs;
+ return 0;
+}
+
+void fs_class_register(const struct fs *fs_class)
+{
+ if (!array_is_created(&fs_classes))
+ fs_classes_init();
+ array_push_back(&fs_classes, &fs_class);
+}
+
+static void fs_classes_deinit(void)
+{
+ array_free(&fs_classes);
+}
+
+static void fs_classes_init(void)
+{
+ i_array_init(&fs_classes, 8);
+ fs_class_register(&fs_class_dict);
+ fs_class_register(&fs_class_posix);
+ fs_class_register(&fs_class_randomfail);
+ fs_class_register(&fs_class_metawrap);
+ fs_class_register(&fs_class_sis);
+ fs_class_register(&fs_class_sis_queue);
+ fs_class_register(&fs_class_test);
+ lib_atexit(fs_classes_deinit);
+}
+
+static const struct fs *fs_class_find(const char *driver)
+{
+ const struct fs *class;
+
+ if (!array_is_created(&fs_classes))
+ fs_classes_init();
+
+ array_foreach_elem(&fs_classes, class) {
+ if (strcmp(class->name, driver) == 0)
+ return class;
+ }
+ return NULL;
+}
+
+static void fs_class_deinit_modules(void)
+{
+ module_dir_unload(&fs_modules);
+}
+
+static const char *fs_driver_module_name(const char *driver)
+{
+ return t_str_replace(driver, '-', '_');
+}
+
+static void fs_class_try_load_plugin(const char *driver)
+{
+ const char *module_name =
+ t_strdup_printf("fs_%s", fs_driver_module_name(driver));
+ struct module *module;
+ struct module_dir_load_settings mod_set;
+ const struct fs *fs_class;
+
+ i_zero(&mod_set);
+ mod_set.abi_version = DOVECOT_ABI_VERSION;
+ mod_set.ignore_missing = TRUE;
+
+ fs_modules = module_dir_load_missing(fs_modules, MODULE_DIR,
+ module_name, &mod_set);
+ module_dir_init(fs_modules);
+
+ module = module_dir_find(fs_modules, module_name);
+ fs_class = module == NULL ? NULL :
+ module_get_symbol(module, t_strdup_printf(
+ "fs_class_%s", fs_driver_module_name(driver)));
+ if (fs_class != NULL)
+ fs_class_register(fs_class);
+
+ lib_atexit(fs_class_deinit_modules);
+}
+
+int fs_init(const char *driver, const char *args,
+ const struct fs_settings *set,
+ struct fs **fs_r, const char **error_r)
+{
+ const struct fs *fs_class;
+ const char *temp_file_prefix;
+
+ fs_class = fs_class_find(driver);
+ if (fs_class == NULL) {
+ T_BEGIN {
+ fs_class_try_load_plugin(driver);
+ } T_END;
+ fs_class = fs_class_find(driver);
+ }
+ if (fs_class == NULL) {
+ *error_r = t_strdup_printf("Unknown fs driver: %s", driver);
+ return -1;
+ }
+ if (fs_alloc(fs_class, args, set, fs_r, error_r) < 0)
+ return -1;
+ event_set_ptr((*fs_r)->event, FS_EVENT_FIELD_FS, *fs_r);
+
+ temp_file_prefix = set->temp_file_prefix != NULL ?
+ set->temp_file_prefix : ".temp.dovecot";
+ if(set->temp_dir == NULL)
+ (*fs_r)->temp_path_prefix = i_strconcat("/tmp/",
+ temp_file_prefix, NULL);
+ else
+ (*fs_r)->temp_path_prefix = i_strconcat(set->temp_dir, "/",
+ temp_file_prefix, NULL);
+ return 0;
+}
+
+int fs_init_from_string(const char *str, const struct fs_settings *set,
+ struct fs **fs_r, const char **error_r)
+{
+ const char *args = strpbrk(str, " :");
+ if (args == NULL)
+ args = "";
+ else
+ str = t_strdup_until(str, args++);
+ return fs_init(str, args, set, fs_r, error_r);
+}
+
+void fs_deinit(struct fs **fs)
+{
+ fs_unref(fs);
+}
+
+void fs_ref(struct fs *fs)
+{
+ i_assert(fs->refcount > 0);
+
+ fs->refcount++;
+}
+
+void fs_unref(struct fs **_fs)
+{
+ struct fs *fs = *_fs;
+ struct array module_contexts_arr;
+ unsigned int i;
+
+ if (fs == NULL)
+ return;
+
+ module_contexts_arr = fs->module_contexts.arr;
+
+ i_assert(fs->refcount > 0);
+
+ *_fs = NULL;
+
+ if (--fs->refcount > 0)
+ return;
+
+ if (fs->files_open_count > 0) {
+ i_panic("fs-%s: %u files still open (first = %s)",
+ fs->name, fs->files_open_count, fs_file_path(fs->files));
+ }
+ i_assert(fs->files == NULL);
+
+ if (fs->v.deinit != NULL)
+ fs->v.deinit(fs);
+
+ fs_deinit(&fs->parent);
+ event_unref(&fs->event);
+ i_free(fs->username);
+ i_free(fs->session_id);
+ i_free(fs->temp_path_prefix);
+ for (i = 0; i < FS_OP_COUNT; i++) {
+ if (fs->stats.timings[i] != NULL)
+ stats_dist_deinit(&fs->stats.timings[i]);
+ }
+ T_BEGIN {
+ fs->v.free(fs);
+ } T_END;
+ array_free_i(&module_contexts_arr);
+}
+
+struct fs *fs_get_parent(struct fs *fs)
+{
+ return fs->parent;
+}
+
+const char *fs_get_driver(struct fs *fs)
+{
+ return fs->name;
+}
+
+const char *fs_get_root_driver(struct fs *fs)
+{
+ while (fs->parent != NULL)
+ fs = fs->parent;
+ return fs->name;
+}
+
+struct fs_file *fs_file_init(struct fs *fs, const char *path, int mode_flags)
+{
+ return fs_file_init_with_event(fs, fs->event, path, mode_flags);
+}
+
+struct fs_file *fs_file_init_with_event(struct fs *fs, struct event *event,
+ const char *path, int mode_flags)
+{
+ struct fs_file *file;
+
+ i_assert(path != NULL);
+ i_assert((mode_flags & FS_OPEN_FLAG_ASYNC_NOQUEUE) == 0 ||
+ (mode_flags & FS_OPEN_FLAG_ASYNC) != 0);
+
+ T_BEGIN {
+ file = fs->v.file_alloc();
+ file->fs = fs;
+ file->flags = mode_flags & ENUM_NEGATE(FS_OPEN_MODE_MASK);
+ file->event = fs_create_event(fs, event);
+ event_set_ptr(file->event, FS_EVENT_FIELD_FS, fs);
+ event_set_ptr(file->event, FS_EVENT_FIELD_FILE, file);
+ fs->v.file_init(file, path, mode_flags & FS_OPEN_MODE_MASK,
+ mode_flags & ENUM_NEGATE(FS_OPEN_MODE_MASK));
+ } T_END;
+
+ fs->files_open_count++;
+ DLLIST_PREPEND(&fs->files, file);
+
+ fs_set_metadata(file, FS_METADATA_ORIG_PATH, path);
+ return file;
+}
+
+void fs_file_deinit(struct fs_file **_file)
+{
+ struct fs_file *file = *_file;
+
+ if (file == NULL)
+ return;
+
+ i_assert(file->fs->files_open_count > 0);
+
+ *_file = NULL;
+
+ fs_file_close(file);
+
+ DLLIST_REMOVE(&file->fs->files, file);
+ file->fs->files_open_count--;
+ T_BEGIN {
+ file->fs->v.file_deinit(file);
+ } T_END;
+}
+
+void fs_file_free(struct fs_file *file)
+{
+ if (file->last_error_changed) {
+ /* fs_set_error() used without ever accessing it via
+ fs_file_last_error(). Log it to make sure it's not lost.
+ Note that the errors are always set only to the file at
+ the root of the parent hierarchy. */
+ e_error(file->event, "%s (in file %s deinit)",
+ file->last_error, fs_file_path(file));
+ }
+
+ fs_file_deinit(&file->parent);
+ event_unref(&file->event);
+ pool_unref(&file->metadata_pool);
+ i_free(file->last_error);
+}
+
+void fs_file_set_flags(struct fs_file *file,
+ enum fs_open_flags add_flags,
+ enum fs_open_flags remove_flags)
+{
+ file->flags |= add_flags;
+ file->flags &= ENUM_NEGATE(remove_flags);
+
+ if (file->parent != NULL)
+ fs_file_set_flags(file->parent, add_flags, remove_flags);
+}
+
+void fs_file_close(struct fs_file *file)
+{
+ if (file == NULL)
+ return;
+
+ i_assert(!file->writing_stream);
+ i_assert(file->output == NULL);
+
+ if (file->pending_read_input != NULL)
+ i_stream_unref(&file->pending_read_input);
+ if (file->seekable_input != NULL)
+ i_stream_unref(&file->seekable_input);
+
+ if (file->copy_input != NULL) {
+ i_stream_unref(&file->copy_input);
+ fs_write_stream_abort_error(file, &file->copy_output, "fs_file_close(%s)",
+ o_stream_get_name(file->copy_output));
+ }
+ i_free_and_null(file->write_digest);
+ if (file->fs->v.file_close != NULL) T_BEGIN {
+ file->fs->v.file_close(file);
+ } T_END;
+
+ /* check this only after closing, because some of the fs backends keep
+ the istream internally open and don't call the destroy-callback
+ until after file_close() */
+ i_assert(!file->istream_open);
+}
+
+enum fs_properties fs_get_properties(struct fs *fs)
+{
+ return fs->v.get_properties(fs);
+}
+
+void fs_metadata_init(struct fs_file *file)
+{
+ if (file->metadata_pool == NULL) {
+ i_assert(!array_is_created(&file->metadata));
+ file->metadata_pool = pool_alloconly_create("fs metadata", 1024);
+ p_array_init(&file->metadata, file->metadata_pool, 8);
+ }
+}
+
+void fs_metadata_init_or_clear(struct fs_file *file)
+{
+ if (file->metadata_pool == NULL)
+ fs_metadata_init(file);
+ else T_BEGIN {
+ const struct fs_metadata *md;
+ ARRAY_TYPE(fs_metadata) internal_metadata;
+
+ t_array_init(&internal_metadata, 4);
+ array_foreach(&file->metadata, md) {
+ if (strncmp(md->key, FS_METADATA_INTERNAL_PREFIX,
+ strlen(FS_METADATA_INTERNAL_PREFIX)) == 0)
+ array_push_back(&internal_metadata, md);
+ }
+ array_clear(&file->metadata);
+ array_append_array(&file->metadata, &internal_metadata);
+ } T_END;
+}
+
+static struct fs_metadata *
+fs_metadata_find_md(const ARRAY_TYPE(fs_metadata) *metadata,
+ const char *key)
+{
+ struct fs_metadata *md;
+
+ array_foreach_modifiable(metadata, md) {
+ if (strcmp(md->key, key) == 0)
+ return md;
+ }
+ return NULL;
+}
+
+void fs_default_set_metadata(struct fs_file *file,
+ const char *key, const char *value)
+{
+ struct fs_metadata *metadata;
+
+ fs_metadata_init(file);
+ metadata = fs_metadata_find_md(&file->metadata, key);
+ if (metadata == NULL) {
+ metadata = array_append_space(&file->metadata);
+ metadata->key = p_strdup(file->metadata_pool, key);
+ }
+ metadata->value = p_strdup(file->metadata_pool, value);
+}
+
+const char *fs_metadata_find(const ARRAY_TYPE(fs_metadata) *metadata,
+ const char *key)
+{
+ const struct fs_metadata *md;
+
+ if (!array_is_created(metadata))
+ return NULL;
+
+ md = fs_metadata_find_md(metadata, key);
+ return md == NULL ? NULL : md->value;
+}
+
+void fs_set_metadata(struct fs_file *file, const char *key, const char *value)
+{
+ i_assert(key != NULL);
+ i_assert(value != NULL);
+
+ if (file->fs->v.set_metadata != NULL) T_BEGIN {
+ file->fs->v.set_metadata(file, key, value);
+ if (strncmp(key, FS_METADATA_INTERNAL_PREFIX,
+ strlen(FS_METADATA_INTERNAL_PREFIX)) == 0) {
+ /* internal metadata change, which isn't stored. */
+ } else {
+ file->metadata_changed = TRUE;
+ }
+ } T_END;
+}
+
+static void fs_file_timing_start(struct fs_file *file, enum fs_op op)
+{
+ if (!file->fs->set.enable_timing)
+ return;
+ if (file->timing_start[op].tv_sec == 0)
+ i_gettimeofday(&file->timing_start[op]);
+}
+
+static void
+fs_timing_end(struct stats_dist **timing, const struct timeval *start_tv)
+{
+ struct timeval now;
+ long long diff;
+
+ i_gettimeofday(&now);
+
+ diff = timeval_diff_usecs(&now, start_tv);
+ if (diff > 0) {
+ if (*timing == NULL)
+ *timing = stats_dist_init();
+ stats_dist_add(*timing, diff);
+ }
+}
+
+void fs_file_timing_end(struct fs_file *file, enum fs_op op)
+{
+ if (!file->fs->set.enable_timing || file->timing_start[op].tv_sec == 0)
+ return;
+
+ fs_timing_end(&file->fs->stats.timings[op], &file->timing_start[op]);
+ /* don't count this again */
+ file->timing_start[op].tv_sec = 0;
+}
+
+int fs_get_metadata_full(struct fs_file *file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r)
+{
+ int ret;
+
+ if (file->fs->v.get_metadata == NULL) {
+ if (array_is_created(&file->metadata)) {
+ /* Return internal metadata. */
+ *metadata_r = &file->metadata;
+ return 0;
+ }
+ fs_set_error(file->event, ENOTSUP, "Metadata not supported by backend");
+ return -1;
+ }
+ if (!file->read_or_prefetch_counted &&
+ !file->lookup_metadata_counted) {
+ if ((flags & FS_GET_METADATA_FLAG_LOADED_ONLY) == 0) {
+ file->lookup_metadata_counted = TRUE;
+ file->fs->stats.lookup_metadata_count++;
+ }
+ fs_file_timing_start(file, FS_OP_METADATA);
+ }
+ T_BEGIN {
+ ret = file->fs->v.get_metadata(file, flags, metadata_r);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN))
+ fs_file_timing_end(file, FS_OP_METADATA);
+ return ret;
+}
+
+int fs_get_metadata(struct fs_file *file,
+ const ARRAY_TYPE(fs_metadata) **metadata_r)
+{
+ return fs_get_metadata_full(file, 0, metadata_r);
+}
+
+int fs_lookup_metadata(struct fs_file *file, const char *key,
+ const char **value_r)
+{
+ const ARRAY_TYPE(fs_metadata) *metadata;
+
+ if (fs_get_metadata(file, &metadata) < 0)
+ return -1;
+ *value_r = fs_metadata_find(metadata, key);
+ return *value_r != NULL ? 1 : 0;
+}
+
+const char *fs_lookup_loaded_metadata(struct fs_file *file, const char *key)
+{
+ const ARRAY_TYPE(fs_metadata) *metadata;
+
+ if (fs_get_metadata_full(file, FS_GET_METADATA_FLAG_LOADED_ONLY, &metadata) < 0)
+ i_panic("FS_GET_METADATA_FLAG_LOADED_ONLY lookup can't fail");
+ return fs_metadata_find(metadata, key);
+}
+
+const char *fs_file_path(struct fs_file *file)
+{
+ return file->fs->v.get_path == NULL ? file->path :
+ file->fs->v.get_path(file);
+}
+
+struct fs *fs_file_fs(struct fs_file *file)
+{
+ return file->fs;
+}
+
+struct event *fs_file_event(struct fs_file *file)
+{
+ return file->event;
+}
+
+static struct fs_file *fs_file_get_error_file(struct fs_file *file)
+{
+ /* the error is always kept in the parentmost file */
+ while (file->parent != NULL)
+ file = file->parent;
+ return file;
+}
+
+static void ATTR_FORMAT(2, 0)
+fs_set_verror(struct event *event, const char *fmt, va_list args)
+{
+ struct event *fs_event = event;
+ struct fs_file *file;
+ struct fs_iter *iter;
+
+ /* NOTE: the event might be a passthrough event. We must log it exactly
+ once so it gets freed. */
+
+ /* figure out if the error is for a file or iter */
+ while ((file = event_get_ptr(fs_event, FS_EVENT_FIELD_FILE)) == NULL &&
+ (iter = event_get_ptr(fs_event, FS_EVENT_FIELD_ITER)) == NULL) {
+ fs_event = event_get_parent(fs_event);
+ i_assert(fs_event != NULL);
+ }
+
+ char *new_error = i_strdup_vprintf(fmt, args);
+ /* Don't flood the debug log with "Asynchronous operation in progress"
+ messages. They tell nothing useful. */
+ if (errno != EAGAIN)
+ e_debug(event, "%s", new_error);
+ else
+ event_send_abort(event);
+
+ /* free old error after strdup in case args point to the old error */
+ if (file != NULL) {
+ file = fs_file_get_error_file(file);
+ char *old_error = file->last_error;
+
+ if (old_error == NULL) {
+ i_assert(!file->last_error_changed);
+ } else if (strcmp(old_error, new_error) == 0) {
+ /* identical error - ignore */
+ } else if (file->last_error_changed) {
+ /* multiple fs_set_error() calls used without
+ fs_file_last_error() in the middle. */
+ e_error(file->event, "%s (overwriting error for file %s)",
+ old_error, fs_file_path(file));
+ }
+ if (errno == EAGAIN || errno == ENOENT || errno == EEXIST ||
+ errno == ENOTEMPTY) {
+ /* These are (or can be) expected errors - don't log
+ them if they have a missing fs_file_last_error()
+ call */
+ file->last_error_changed = FALSE;
+ } else {
+ file->last_error_changed = TRUE;
+ }
+
+ i_free(file->last_error);
+ file->last_error = new_error;
+ } else {
+ i_assert(iter != NULL);
+ if (iter->last_error != NULL &&
+ strcmp(iter->last_error, new_error) == 0) {
+ /* identical error - ignore */
+ } else if (iter->last_error != NULL) {
+ /* multiple fs_set_error() calls before the iter
+ finishes */
+ e_error(iter->fs->event, "%s (overwriting error for file %s)",
+ iter->last_error, iter->path);
+ }
+ i_free(iter->last_error);
+ iter->last_error = new_error;
+ }
+}
+
+const char *fs_file_last_error(struct fs_file *file)
+{
+ struct fs_file *error_file = fs_file_get_error_file(file);
+
+ error_file->last_error_changed = FALSE;
+ if (error_file->last_error == NULL)
+ return "BUG: Unknown file error";
+ return error_file->last_error;
+}
+
+bool fs_prefetch(struct fs_file *file, uoff_t length)
+{
+ bool ret;
+
+ if (!file->read_or_prefetch_counted) {
+ file->read_or_prefetch_counted = TRUE;
+ file->fs->stats.prefetch_count++;
+ fs_file_timing_start(file, FS_OP_PREFETCH);
+ }
+ T_BEGIN {
+ ret = file->fs->v.prefetch(file, length);
+ } T_END;
+ fs_file_timing_end(file, FS_OP_PREFETCH);
+ return ret;
+}
+
+ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size)
+{
+ const unsigned char *data;
+ size_t data_size;
+ ssize_t ret;
+
+ i_assert(size > 0);
+
+ if (file->pending_read_input == NULL)
+ file->pending_read_input = fs_read_stream(file, size+1);
+ ret = i_stream_read_bytes(file->pending_read_input, &data,
+ &data_size, size);
+ if (ret == 0) {
+ fs_file_set_error_async(file);
+ return -1;
+ }
+ if (ret < 0 && file->pending_read_input->stream_errno != 0) {
+ fs_set_error(file->event,
+ file->pending_read_input->stream_errno,
+ "read(%s) failed: %s",
+ i_stream_get_name(file->pending_read_input),
+ i_stream_get_error(file->pending_read_input));
+ } else {
+ ret = I_MIN(size, data_size);
+ memcpy(buf, data, ret);
+ }
+ i_stream_unref(&file->pending_read_input);
+ return ret;
+}
+
+ssize_t fs_read(struct fs_file *file, void *buf, size_t size)
+{
+ int ret;
+
+ if (!file->read_or_prefetch_counted) {
+ file->read_or_prefetch_counted = TRUE;
+ file->fs->stats.read_count++;
+ fs_file_timing_start(file, FS_OP_READ);
+ }
+
+ if (file->fs->v.read != NULL) {
+ T_BEGIN {
+ ret = file->fs->v.read(file, buf, size);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN))
+ fs_file_timing_end(file, FS_OP_READ);
+ return ret;
+ }
+
+ /* backend didn't bother to implement read(), but we can do it with
+ streams. */
+ return fs_read_via_stream(file, buf, size);
+}
+
+static void fs_file_istream_destroyed(struct fs_file *file)
+{
+ i_assert(file->istream_open);
+
+ file->istream_open = FALSE;
+}
+
+struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size)
+{
+ struct istream *input, *inputs[2];
+ const unsigned char *data;
+ size_t size;
+ ssize_t ret;
+ bool want_seekable = FALSE;
+
+ if (!file->read_or_prefetch_counted) {
+ file->read_or_prefetch_counted = TRUE;
+ file->fs->stats.read_count++;
+ fs_file_timing_start(file, FS_OP_READ);
+ }
+
+ if (file->seekable_input != NULL) {
+ /* allow multiple open streams, each in a different position */
+ input = i_stream_create_limit(file->seekable_input, UOFF_T_MAX);
+ i_stream_seek(input, 0);
+ return input;
+ }
+ i_assert(!file->istream_open);
+ T_BEGIN {
+ input = file->fs->v.read_stream(file, max_buffer_size);
+ } T_END;
+ if (input->stream_errno != 0) {
+ /* read failed already */
+ fs_file_timing_end(file, FS_OP_READ);
+ return input;
+ }
+ if (file->fs->set.enable_timing) {
+ struct istream *input2 = i_stream_create_fs_stats(input, file);
+
+ i_stream_unref(&input);
+ input = input2;
+ }
+
+ if ((file->flags & FS_OPEN_FLAG_SEEKABLE) != 0)
+ want_seekable = TRUE;
+ else if ((file->flags & FS_OPEN_FLAG_ASYNC) == 0 && !input->blocking)
+ want_seekable = TRUE;
+
+ if (want_seekable && !input->seekable) {
+ /* need to make the stream seekable */
+ inputs[0] = input;
+ inputs[1] = NULL;
+ input = i_stream_create_seekable_path(inputs, max_buffer_size,
+ file->fs->temp_path_prefix);
+ i_stream_set_name(input, i_stream_get_name(inputs[0]));
+ i_stream_unref(&inputs[0]);
+ }
+ file->seekable_input = input;
+ i_stream_ref(file->seekable_input);
+
+ if ((file->flags & FS_OPEN_FLAG_ASYNC) == 0 && !input->blocking) {
+ /* read the whole input stream before returning */
+ while ((ret = i_stream_read_more(input, &data, &size)) >= 0) {
+ i_stream_skip(input, size);
+ if (ret == 0)
+ fs_wait_async(file->fs);
+ }
+ i_stream_seek(input, 0);
+ }
+ file->istream_open = TRUE;
+ i_stream_add_destroy_callback(input, fs_file_istream_destroyed, file);
+ return input;
+}
+
+int fs_write_via_stream(struct fs_file *file, const void *data, size_t size)
+{
+ struct ostream *output;
+ ssize_t ret;
+ int err;
+
+ if (!file->write_pending) {
+ output = fs_write_stream(file);
+ if ((ret = o_stream_send(output, data, size)) < 0) {
+ err = errno;
+ fs_write_stream_abort_error(file, &output, "fs_write(%s) failed: %s",
+ o_stream_get_name(output),
+ o_stream_get_error(output));
+ errno = err;
+ return -1;
+ }
+ i_assert((size_t)ret == size);
+ ret = fs_write_stream_finish(file, &output);
+ } else {
+ ret = fs_write_stream_finish_async(file);
+ }
+ if (ret == 0) {
+ fs_file_set_error_async(file);
+ file->write_pending = TRUE;
+ return -1;
+ }
+ file->write_pending = FALSE;
+ return ret < 0 ? -1 : 0;
+}
+
+int fs_write(struct fs_file *file, const void *data, size_t size)
+{
+ int ret;
+
+ if (file->fs->v.write != NULL) {
+ fs_file_timing_start(file, FS_OP_WRITE);
+ T_BEGIN {
+ ret = file->fs->v.write(file, data, size);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN)) {
+ file->fs->stats.write_count++;
+ file->fs->stats.write_bytes += size;
+ fs_file_timing_end(file, FS_OP_WRITE);
+ }
+ return ret;
+ }
+
+ /* backend didn't bother to implement write(), but we can do it with
+ streams. */
+ return fs_write_via_stream(file, data, size);
+}
+
+struct ostream *fs_write_stream(struct fs_file *file)
+{
+ i_assert(!file->writing_stream);
+ i_assert(file->output == NULL);
+
+ file->writing_stream = TRUE;
+ file->fs->stats.write_count++;
+ T_BEGIN {
+ file->fs->v.write_stream(file);
+ } T_END;
+ i_assert(file->output != NULL);
+ o_stream_cork(file->output);
+ return file->output;
+}
+
+static int fs_write_stream_finish_int(struct fs_file *file, bool success)
+{
+ int ret;
+
+ i_assert(file->writing_stream);
+
+ fs_file_timing_start(file, FS_OP_WRITE);
+ T_BEGIN {
+ ret = file->fs->v.write_stream_finish(file, success);
+ } T_END;
+ if (ret != 0) {
+ fs_file_timing_end(file, FS_OP_WRITE);
+ file->metadata_changed = FALSE;
+ } else {
+ /* write didn't finish yet. this shouldn't happen if we
+ indicated a failure. */
+ i_assert(success);
+ }
+ if (ret != 0) {
+ i_assert(file->output == NULL);
+ file->writing_stream = FALSE;
+ }
+ return ret;
+}
+
+int fs_write_stream_finish(struct fs_file *file, struct ostream **output)
+{
+ bool success = TRUE;
+ int ret;
+
+ i_assert(*output == file->output || *output == NULL);
+ i_assert(output != &file->output);
+
+ *output = NULL;
+ if (file->output != NULL) {
+ o_stream_uncork(file->output);
+ if ((ret = o_stream_finish(file->output)) <= 0) {
+ i_assert(ret < 0);
+ fs_set_error(file->event, file->output->stream_errno,
+ "write(%s) failed: %s",
+ o_stream_get_name(file->output),
+ o_stream_get_error(file->output));
+ success = FALSE;
+ }
+ file->fs->stats.write_bytes += file->output->offset;
+ }
+ return fs_write_stream_finish_int(file, success);
+}
+
+int fs_write_stream_finish_async(struct fs_file *file)
+{
+ return fs_write_stream_finish_int(file, TRUE);
+}
+
+static void fs_write_stream_abort(struct fs_file *file, struct ostream **output)
+{
+ int ret;
+
+ i_assert(*output == file->output);
+ i_assert(file->output != NULL);
+ i_assert(output != &file->output);
+
+ *output = NULL;
+ o_stream_abort(file->output);
+ /* make sure we don't have an old error lying around */
+ ret = fs_write_stream_finish_int(file, FALSE);
+ i_assert(ret != 0);
+}
+
+void fs_write_stream_abort_error(struct fs_file *file, struct ostream **output, const char *error_fmt, ...)
+{
+ va_list args;
+ va_start(args, error_fmt);
+ fs_set_verror(file->event, error_fmt, args);
+ /* the error shouldn't be automatically logged if
+ fs_file_last_error() is no longer used */
+ fs_file_get_error_file(file)->last_error_changed = FALSE;
+ fs_write_stream_abort(file, output);
+ va_end(args);
+}
+
+void fs_write_stream_abort_parent(struct fs_file *file, struct ostream **output)
+{
+ i_assert(file->parent != NULL);
+ i_assert(fs_file_last_error(file->parent) != NULL);
+ fs_write_stream_abort(file->parent, output);
+}
+
+void fs_write_set_hash(struct fs_file *file, const struct hash_method *method,
+ const void *digest)
+{
+ file->write_digest_method = method;
+
+ i_free(file->write_digest);
+ file->write_digest = i_malloc(method->digest_size);
+ memcpy(file->write_digest, digest, method->digest_size);
+}
+
+#undef fs_file_set_async_callback
+void fs_file_set_async_callback(struct fs_file *file,
+ fs_file_async_callback_t *callback,
+ void *context)
+{
+ if (file->fs->v.set_async_callback != NULL)
+ file->fs->v.set_async_callback(file, callback, context);
+ else
+ callback(context);
+}
+
+void fs_wait_async(struct fs *fs)
+{
+ /* recursion not allowed */
+ i_assert(fs->prev_ioloop == NULL);
+
+ if (fs->v.wait_async != NULL) T_BEGIN {
+ fs->prev_ioloop = current_ioloop;
+ fs->v.wait_async(fs);
+ i_assert(current_ioloop == fs->prev_ioloop);
+ fs->prev_ioloop = NULL;
+ } T_END;
+}
+
+bool fs_switch_ioloop(struct fs *fs)
+{
+ bool ret = FALSE;
+
+ if (fs->v.switch_ioloop != NULL) {
+ T_BEGIN {
+ ret = fs->v.switch_ioloop(fs);
+ } T_END;
+ } else if (fs->parent != NULL) {
+ ret = fs_switch_ioloop(fs->parent);
+ }
+ return ret;
+}
+
+int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r)
+{
+ int ret;
+
+ T_BEGIN {
+ ret = file->fs->v.lock(file, secs, lock_r);
+ } T_END;
+ return ret;
+}
+
+void fs_unlock(struct fs_lock **_lock)
+{
+ struct fs_lock *lock = *_lock;
+
+ if (lock == NULL)
+ return;
+
+ *_lock = NULL;
+ T_BEGIN {
+ lock->file->fs->v.unlock(lock);
+ } T_END;
+}
+
+int fs_exists(struct fs_file *file)
+{
+ struct stat st;
+ int ret;
+
+ if (file->fs->v.exists == NULL) {
+ /* fallback to stat() */
+ if (fs_stat(file, &st) == 0)
+ return 1;
+ else
+ return errno == ENOENT ? 0 : -1;
+ }
+ fs_file_timing_start(file, FS_OP_EXISTS);
+ T_BEGIN {
+ ret = file->fs->v.exists(file);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN)) {
+ file->fs->stats.exists_count++;
+ fs_file_timing_end(file, FS_OP_EXISTS);
+ }
+ return ret;
+}
+
+int fs_stat(struct fs_file *file, struct stat *st_r)
+{
+ int ret;
+
+ if (file->fs->v.stat == NULL) {
+ fs_set_error(file->event, ENOTSUP, "fs_stat() not supported");
+ return -1;
+ }
+
+ if (!file->read_or_prefetch_counted &&
+ !file->lookup_metadata_counted && !file->stat_counted) {
+ file->stat_counted = TRUE;
+ file->fs->stats.stat_count++;
+ fs_file_timing_start(file, FS_OP_STAT);
+ }
+ T_BEGIN {
+ ret = file->fs->v.stat(file, st_r);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN))
+ fs_file_timing_end(file, FS_OP_STAT);
+ return ret;
+}
+
+int fs_get_nlinks(struct fs_file *file, nlink_t *nlinks_r)
+{
+ int ret;
+
+ if (file->fs->v.get_nlinks == NULL) {
+ struct stat st;
+
+ if (fs_stat(file, &st) < 0)
+ return -1;
+ *nlinks_r = st.st_nlink;
+ return 0;
+ }
+
+ if (!file->read_or_prefetch_counted &&
+ !file->lookup_metadata_counted && !file->stat_counted) {
+ file->stat_counted = TRUE;
+ file->fs->stats.stat_count++;
+ fs_file_timing_start(file, FS_OP_STAT);
+ }
+ T_BEGIN {
+ ret = file->fs->v.get_nlinks(file, nlinks_r);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN))
+ fs_file_timing_end(file, FS_OP_STAT);
+ return ret;
+}
+
+int fs_default_copy(struct fs_file *src, struct fs_file *dest)
+{
+ int tmp_errno;
+ /* we're going to be counting this as read+write, so don't update
+ copy_count */
+ dest->copy_counted = TRUE;
+
+ if (dest->copy_src != NULL) {
+ i_assert(src == NULL || src == dest->copy_src);
+ if (dest->copy_output == NULL) {
+ i_assert(dest->copy_input == NULL);
+ if (fs_write_stream_finish_async(dest) <= 0)
+ return -1;
+ dest->copy_src = NULL;
+ return 0;
+ }
+ } else {
+ dest->copy_src = src;
+ dest->copy_input = fs_read_stream(src, IO_BLOCK_SIZE);
+ dest->copy_output = fs_write_stream(dest);
+ }
+ switch (o_stream_send_istream(dest->copy_output, dest->copy_input)) {
+ case OSTREAM_SEND_ISTREAM_RESULT_FINISHED:
+ break;
+ case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT:
+ case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT:
+ fs_file_set_error_async(dest);
+ return -1;
+ case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT:
+ fs_write_stream_abort_error(dest, &dest->copy_output,
+ "read(%s) failed: %s",
+ i_stream_get_name(dest->copy_input),
+ i_stream_get_error(dest->copy_input));
+ errno = dest->copy_input->stream_errno;
+ i_stream_unref(&dest->copy_input);
+ return -1;
+ case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT:
+ /* errno might not survive abort error */
+ tmp_errno = dest->copy_output->stream_errno;
+ fs_write_stream_abort_error(dest, &dest->copy_output,
+ "write(%s) failed: %s",
+ o_stream_get_name(dest->copy_output),
+ o_stream_get_error(dest->copy_output));
+ errno = tmp_errno;
+ i_stream_unref(&dest->copy_input);
+ return -1;
+ }
+ i_stream_unref(&dest->copy_input);
+ if (fs_write_stream_finish(dest, &dest->copy_output) <= 0)
+ return -1;
+ dest->copy_src = NULL;
+ return 0;
+}
+
+int fs_copy(struct fs_file *src, struct fs_file *dest)
+{
+ int ret;
+
+ i_assert(src->fs == dest->fs);
+
+ if (src->fs->v.copy == NULL) {
+ fs_set_error(src->event, ENOTSUP, "fs_copy() not supported");
+ return -1;
+ }
+
+ fs_file_timing_start(dest, FS_OP_COPY);
+ T_BEGIN {
+ ret = src->fs->v.copy(src, dest);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN)) {
+ fs_file_timing_end(dest, FS_OP_COPY);
+ if (dest->copy_counted)
+ dest->copy_counted = FALSE;
+ else
+ dest->fs->stats.copy_count++;
+ dest->metadata_changed = FALSE;
+ }
+ return ret;
+}
+
+int fs_copy_finish_async(struct fs_file *dest)
+{
+ int ret;
+
+ T_BEGIN {
+ ret = dest->fs->v.copy(NULL, dest);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN)) {
+ fs_file_timing_end(dest, FS_OP_COPY);
+ if (dest->copy_counted)
+ dest->copy_counted = FALSE;
+ else
+ dest->fs->stats.copy_count++;
+ dest->metadata_changed = FALSE;
+ }
+ return ret;
+}
+
+int fs_rename(struct fs_file *src, struct fs_file *dest)
+{
+ int ret;
+
+ i_assert(src->fs == dest->fs);
+
+ fs_file_timing_start(dest, FS_OP_RENAME);
+ T_BEGIN {
+ ret = src->fs->v.rename(src, dest);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN)) {
+ dest->fs->stats.rename_count++;
+ fs_file_timing_end(dest, FS_OP_RENAME);
+ }
+ return ret;
+}
+
+int fs_delete(struct fs_file *file)
+{
+ int ret;
+
+ i_assert(!file->writing_stream);
+
+ fs_file_timing_start(file, FS_OP_DELETE);
+ T_BEGIN {
+ ret = file->fs->v.delete_file(file);
+ } T_END;
+ if (!(ret < 0 && errno == EAGAIN)) {
+ file->fs->stats.delete_count++;
+ fs_file_timing_end(file, FS_OP_DELETE);
+ }
+ return ret;
+}
+
+struct fs_iter *
+fs_iter_init(struct fs *fs, const char *path, enum fs_iter_flags flags)
+{
+ return fs_iter_init_with_event(fs, fs->event, path, flags);
+}
+
+struct fs_iter *
+fs_iter_init_with_event(struct fs *fs, struct event *event,
+ const char *path, enum fs_iter_flags flags)
+{
+ struct fs_iter *iter;
+ struct timeval now = ioloop_timeval;
+
+ i_assert((flags & FS_ITER_FLAG_OBJECTIDS) == 0 ||
+ (fs_get_properties(fs) & FS_PROPERTY_OBJECTIDS) != 0);
+
+ fs->stats.iter_count++;
+ if (fs->set.enable_timing)
+ i_gettimeofday(&now);
+ if (fs->v.iter_init == NULL) {
+ iter = i_new(struct fs_iter, 1);
+ iter->fs = fs;
+ } else T_BEGIN {
+ iter = fs->v.iter_alloc();
+ iter->fs = fs;
+ iter->flags = flags;
+ iter->path = i_strdup(path);
+ iter->event = fs_create_event(fs, event);
+ event_set_ptr(iter->event, FS_EVENT_FIELD_FS, fs);
+ event_set_ptr(iter->event, FS_EVENT_FIELD_ITER, iter);
+ fs->v.iter_init(iter, path, flags);
+ } T_END;
+ iter->start_time = now;
+ DLLIST_PREPEND(&fs->iters, iter);
+ return iter;
+}
+
+int fs_iter_deinit(struct fs_iter **_iter, const char **error_r)
+{
+ struct fs_iter *iter = *_iter;
+ struct fs *fs;
+ struct event *event;
+ int ret;
+
+ if (iter == NULL)
+ return 0;
+
+ fs = iter->fs;
+ event = iter->event;
+
+ *_iter = NULL;
+ DLLIST_REMOVE(&fs->iters, iter);
+
+ if (fs->v.iter_deinit == NULL) {
+ fs_set_error(event, ENOTSUP, "FS iteration not supported");
+ ret = -1;
+ } else T_BEGIN {
+ ret = iter->fs->v.iter_deinit(iter);
+ } T_END;
+ if (ret < 0)
+ *error_r = t_strdup(iter->last_error);
+ i_free(iter->last_error);
+ i_free(iter->path);
+ i_free(iter);
+ event_unref(&event);
+ return ret;
+}
+
+const char *fs_iter_next(struct fs_iter *iter)
+{
+ const char *ret;
+
+ if (iter->fs->v.iter_next == NULL)
+ return NULL;
+ T_BEGIN {
+ ret = iter->fs->v.iter_next(iter);
+ } T_END;
+ if (iter->start_time.tv_sec != 0 &&
+ (ret != NULL || !fs_iter_have_more(iter))) {
+ /* first result returned - count this as the finish time, since
+ we don't want to count the time caller spends on this
+ iteration. */
+ fs_timing_end(&iter->fs->stats.timings[FS_OP_ITER], &iter->start_time);
+ /* don't count this again */
+ iter->start_time.tv_sec = 0;
+ }
+ return ret;
+}
+
+#undef fs_iter_set_async_callback
+void fs_iter_set_async_callback(struct fs_iter *iter,
+ fs_file_async_callback_t *callback,
+ void *context)
+{
+ iter->async_callback = callback;
+ iter->async_context = context;
+}
+
+bool fs_iter_have_more(struct fs_iter *iter)
+{
+ return iter->async_have_more;
+}
+
+const struct fs_stats *fs_get_stats(struct fs *fs)
+{
+ return &fs->stats;
+}
+
+void fs_set_error(struct event *event, int err, const char *fmt, ...)
+{
+ va_list args;
+
+ i_assert(err != 0);
+
+ errno = err;
+ va_start(args, fmt);
+ fs_set_verror(event, fmt, args);
+ va_end(args);
+}
+
+void fs_set_error_errno(struct event *event, const char *fmt, ...)
+{
+ va_list args;
+
+ i_assert(errno != 0);
+
+ va_start(args, fmt);
+ fs_set_verror(event, fmt, args);
+ va_end(args);
+}
+
+void fs_file_set_error_async(struct fs_file *file)
+{
+ fs_set_error(file->event, EAGAIN, "Asynchronous operation in progress");
+}
+
+static uint64_t
+fs_stats_count_ops(const struct fs_stats *stats, const enum fs_op ops[],
+ unsigned int ops_count)
+{
+ uint64_t ret = 0;
+
+ for (unsigned int i = 0; i < ops_count; i++) {
+ if (stats->timings[ops[i]] != NULL)
+ ret += stats_dist_get_sum(stats->timings[ops[i]]);
+ }
+ return ret;
+}
+
+uint64_t fs_stats_get_read_usecs(const struct fs_stats *stats)
+{
+ const enum fs_op read_ops[] = {
+ FS_OP_METADATA, FS_OP_PREFETCH, FS_OP_READ, FS_OP_EXISTS,
+ FS_OP_STAT, FS_OP_ITER
+ };
+ return fs_stats_count_ops(stats, read_ops, N_ELEMENTS(read_ops));
+}
+
+uint64_t fs_stats_get_write_usecs(const struct fs_stats *stats)
+{
+ const enum fs_op write_ops[] = {
+ FS_OP_WRITE, FS_OP_COPY, FS_OP_DELETE
+ };
+ return fs_stats_count_ops(stats, write_ops, N_ELEMENTS(write_ops));
+}
+
+struct fs_file *
+fs_file_init_parent(struct fs_file *parent, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags)
+{
+ return fs_file_init_with_event(parent->fs->parent, parent->event,
+ path, (int)mode | (int)flags);
+}
+
+struct fs_iter *
+fs_iter_init_parent(struct fs_iter *parent,
+ const char *path, enum fs_iter_flags flags)
+{
+ return fs_iter_init_with_event(parent->fs->parent, parent->event,
+ path, flags);
+}
diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h
new file mode 100644
index 0000000..c4918f5
--- /dev/null
+++ b/src/lib-fs/fs-api.h
@@ -0,0 +1,397 @@
+#ifndef FS_API_H
+#define FS_API_H
+
+struct stat;
+struct fs;
+struct fs_file;
+struct fs_lock;
+struct hash_method;
+
+/* Metadata with this prefix shouldn't actually be sent to storage. */
+#define FS_METADATA_INTERNAL_PREFIX ":/X-Dovecot-fs-api-"
+/* fs_write*() may return a hex-encoded object ID after write is finished.
+ This can be later on used to optimize reads by setting it before reading
+ the file. */
+#define FS_METADATA_OBJECTID FS_METADATA_INTERNAL_PREFIX"ObjectID"
+/* Calling this before fs_write_stream_finish() allows renaming the filename.
+ This can be useful if you don't know the final filename before writing it
+ (e.g. filename contains the file size). The given filename must include the
+ full path also. */
+#define FS_METADATA_WRITE_FNAME FS_METADATA_INTERNAL_PREFIX"WriteFilename"
+/* Original path of the file. The path that's eventually visible to a fs
+ backend may be something different, e.g. object ID. This allows the backend
+ to still access the original path. */
+#define FS_METADATA_ORIG_PATH FS_METADATA_INTERNAL_PREFIX"OrigPath"
+
+enum fs_properties {
+ FS_PROPERTY_METADATA = 0x01,
+ FS_PROPERTY_LOCKS = 0x02,
+ FS_PROPERTY_FASTCOPY = 0x04,
+ FS_PROPERTY_RENAME = 0x08,
+ FS_PROPERTY_STAT = 0x10,
+ /* Iteration is possible */
+ FS_PROPERTY_ITER = 0x20,
+ /* Iteration always returns all of the files (instead of possibly
+ slightly out of date view) */
+ FS_PROPERTY_RELIABLEITER= 0x40,
+ /* Backend uses directories, which aren't automatically deleted
+ when its children are deleted. */
+ FS_PROPERTY_DIRECTORIES = 0x80,
+ FS_PROPERTY_WRITE_HASH_MD5 = 0x100,
+ FS_PROPERTY_WRITE_HASH_SHA256 = 0x200,
+ /* fs_copy() will copy the metadata if fs_set_metadata() hasn't
+ been explicitly called. */
+ FS_PROPERTY_COPY_METADATA = 0x400,
+ /* Backend support asynchronous file operations. */
+ FS_PROPERTY_ASYNC = 0x800,
+ /* Backend supports FS_ITER_FLAG_OBJECTIDS. */
+ FS_PROPERTY_OBJECTIDS = 0x1000,
+ /* fs_copy() is fast even when file's metadata is changed */
+ FS_PROPERTY_FASTCOPY_CHANGED_METADATA = 0x2000,
+};
+
+enum fs_open_mode {
+ /* Open only for reading, or fail with ENOENT if it doesn't exist */
+ FS_OPEN_MODE_READONLY,
+ /* Create a new file, fail with EEXIST if it already exists */
+ FS_OPEN_MODE_CREATE,
+ /* Create a new file with a new unique name. The generated name is a
+ 128bit hex-encoded string. The fs_open()'s path parameter specifies
+ only the directory where the file is created to. */
+ FS_OPEN_MODE_CREATE_UNIQUE_128,
+ /* Create or replace a file */
+ FS_OPEN_MODE_REPLACE,
+ /* Append to existing file, fail with ENOENT if it doesn't exist */
+ FS_OPEN_MODE_APPEND
+
+#define FS_OPEN_MODE_MASK 0x0f
+};
+
+enum fs_open_flags {
+ /* File is important and writing must call fsync() or have equivalent
+ behavior. */
+ FS_OPEN_FLAG_FSYNC = 0x10,
+ /* Asynchronous writes: fs_write() will fail with EAGAIN if it needs to
+ be called again (the retries can use size=0). For streams
+ fs_write_stream_finish() may request retrying with 0.
+
+ Asynchronous reads: fs_read() will fail with EAGAIN if it's not
+ finished and fs_read_stream() returns a nonblocking stream. */
+ FS_OPEN_FLAG_ASYNC = 0x20,
+ /* fs_read_stream() must return a seekable input stream */
+ FS_OPEN_FLAG_SEEKABLE = 0x40,
+ /* Backend should handle this file's operations immediately without
+ any additional command queueing. The caller is assumed to be the one
+ doing any rate limiting if needed. This flag can only be used with
+ ASYNC flag, synchronous requests are never queued. */
+ FS_OPEN_FLAG_ASYNC_NOQUEUE = 0x80
+};
+
+enum fs_iter_flags {
+ /* Iterate only directories, not files */
+ FS_ITER_FLAG_DIRS = 0x01,
+ /* Request asynchronous iteration. */
+ FS_ITER_FLAG_ASYNC = 0x02,
+ /* Instead of returning object names, return <objectid>/<object name>.
+ If this isn't supported, the <objectid> is returned empty. The
+ object IDs are always hex-encoded data. This flag can be used only
+ if FS_PROPERTY_OBJECTIDS is enabled. */
+ FS_ITER_FLAG_OBJECTIDS = 0x04,
+ /* Explicitly disable all caching for this iteration (if anything
+ happens to be enabled). This should be used only in situations where
+ the iteration is used to fix something that is broken, e.g. doveadm
+ force-resync. */
+ FS_ITER_FLAG_NOCACHE = 0x08
+};
+
+enum fs_op {
+ FS_OP_WAIT,
+ FS_OP_METADATA,
+ FS_OP_PREFETCH,
+ FS_OP_READ,
+ FS_OP_WRITE,
+ FS_OP_LOCK,
+ FS_OP_EXISTS,
+ FS_OP_STAT,
+ FS_OP_COPY,
+ FS_OP_RENAME,
+ FS_OP_DELETE,
+ FS_OP_ITER,
+
+ FS_OP_COUNT
+};
+
+struct fs_settings {
+ /* Username and session ID are mainly used for debugging/logging,
+ but may also be useful for other purposes if they exist (they
+ may be NULL). */
+ const char *username;
+ const char *session_id;
+
+ /* Dovecot instance's base_dir */
+ const char *base_dir;
+ /* Directory where temporary files can be created at any time
+ (e.g. /tmp or mail_temp_dir) */
+ const char *temp_dir;
+ /* SSL client settings. */
+ const struct ssl_iostream_settings *ssl_client_set;
+
+ /* Automatically try to rmdir() directories up to this path when
+ deleting files. */
+ const char *root_path;
+ /* When creating temporary files, use this prefix
+ (to avoid conflicts with existing files). */
+ const char *temp_file_prefix;
+ /* If the backend needs to do DNS lookups, use this dns_client for
+ them. */
+ struct dns_client *dns_client;
+
+ /* Parent event to use, unless overridden by
+ fs_file_init_with_event() */
+ struct event *event_parent;
+
+ /* Enable debugging */
+ bool debug;
+ /* Enable timing statistics */
+ bool enable_timing;
+};
+
+struct fs_stats {
+ /* Number of fs_prefetch() calls. Counted only if fs_read*() hasn't
+ already been called for the file (which would be pretty pointless
+ to do). */
+ unsigned int prefetch_count;
+ /* Number of fs_read*() calls. Counted only if fs_prefetch() hasn't
+ already been called for the file. */
+ unsigned int read_count;
+ /* Number of fs_lookup_metadata() calls. Counted only if neither
+ fs_read*() nor fs_prefetch() has been called for the file. */
+ unsigned int lookup_metadata_count;
+ /* Number of fs_stat() calls. Counted only if none of the above
+ has been called (because the stat result should be cached). */
+ unsigned int stat_count;
+
+ /* Number of fs_write*() calls. */
+ unsigned int write_count;
+ /* Number of fs_exists() calls, which actually went to the backend
+ instead of being handled by fs_stat() call due to fs_exists() not
+ being implemented. */
+ unsigned int exists_count;
+ /* Number of fs_delete() calls. */
+ unsigned int delete_count;
+ /* Number of fs_copy() calls. If backend doesn't implement copying
+ operation but falls back to regular read+write instead, this count
+ isn't increased but the read+write counters are. */
+ unsigned int copy_count;
+ /* Number of fs_rename() calls. */
+ unsigned int rename_count;
+ /* Number of fs_iter_init() calls. */
+ unsigned int iter_count;
+
+ /* Number of bytes written by fs_write*() calls. */
+ uint64_t write_bytes;
+
+ /* Cumulative sum of usecs spent on calls - set only if
+ fs_settings.enable_timing=TRUE */
+ struct stats_dist *timings[FS_OP_COUNT];
+};
+
+struct fs_metadata {
+ const char *key;
+ const char *value;
+};
+ARRAY_DEFINE_TYPE(fs_metadata, struct fs_metadata);
+
+typedef void fs_file_async_callback_t(void *context);
+
+int fs_init(const char *driver, const char *args,
+ const struct fs_settings *set,
+ struct fs **fs_r, const char **error_r);
+/* helper for fs_init, accepts a filesystem string
+ that can come directly from config */
+int fs_init_from_string(const char *str, const struct fs_settings *set,
+ struct fs **fs_r, const char **error_r);
+/* same as fs_unref() */
+void fs_deinit(struct fs **fs);
+
+void fs_ref(struct fs *fs);
+void fs_unref(struct fs **fs);
+
+/* Returns the parent filesystem (if this is a wrapper fs) or NULL if
+ there's no parent. */
+struct fs *fs_get_parent(struct fs *fs);
+/* Returns the filesystem's driver name. */
+const char *fs_get_driver(struct fs *fs);
+/* Returns the root fs's driver name (bypassing all wrapper fses) */
+const char *fs_get_root_driver(struct fs *fs);
+
+struct fs_file *fs_file_init(struct fs *fs, const char *path, int mode_flags);
+struct fs_file *fs_file_init_with_event(struct fs *fs, struct event *event,
+ const char *path, int mode_flags);
+void fs_file_deinit(struct fs_file **file);
+
+/* Change flags for a file (and its parents). */
+void fs_file_set_flags(struct fs_file *file,
+ enum fs_open_flags add_flags,
+ enum fs_open_flags remove_flags);
+/* If the file has an input streams open, close them. */
+void fs_file_close(struct fs_file *file);
+
+/* Return properties supported by backend. */
+enum fs_properties fs_get_properties(struct fs *fs);
+
+/* Add/replace metadata when saving a file. This makes sense only when the
+ file is being created/replaced. */
+void fs_set_metadata(struct fs_file *file, const char *key, const char *value);
+/* Return file's all metadata. */
+int fs_get_metadata(struct fs_file *file,
+ const ARRAY_TYPE(fs_metadata) **metadata_r);
+/* Wrapper to fs_get_metadata() to lookup a specific key. Returns 1 if value_r
+ is set, 0 if key wasn't found, -1 if error. */
+int fs_lookup_metadata(struct fs_file *file, const char *key,
+ const char **value_r);
+/* Try to find key from the currently set metadata (without refreshing it).
+ This is typically used e.g. after writing or copying a file to find some
+ extra metadata they may have set. It can also be used after
+ fs_get_metadata() or fs_lookup_metadata() to search within the looked up
+ metadata. */
+const char *fs_lookup_loaded_metadata(struct fs_file *file, const char *key);
+
+/* Returns the path given to fs_open(). If file was opened with
+ FS_OPEN_MODE_CREATE_UNIQUE_128 and the write has already finished,
+ return the path including the generated filename. */
+const char *fs_file_path(struct fs_file *file);
+/* Returns the file's fs. */
+struct fs *fs_file_fs(struct fs_file *file);
+/* Returns the file's event. */
+struct event *fs_file_event(struct fs_file *file);
+
+/* Return the error message for the last failed file operation. Each file
+ keeps track of its own errors. For failed copy/rename operations the "dest"
+ file contains the error. */
+const char *fs_file_last_error(struct fs_file *file);
+
+/* Try to asynchronously prefetch file into memory. Returns TRUE if file is
+ already in memory (i.e. caller should handle this file before prefetching
+ more), FALSE if not. The length is a hint of how much the caller expects
+ to read, but it may be more or less (0=whole file). */
+bool fs_prefetch(struct fs_file *file, uoff_t length);
+/* Returns >0 if something was read, -1 if error (errno is set). */
+ssize_t fs_read(struct fs_file *file, void *buf, size_t size);
+/* Returns a stream for reading from file. Multiple streams can be opened,
+ and caller must destroy the streams before closing the file. */
+struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size);
+
+/* Returns 0 if ok, -1 if error (errno is set). Note: With CREATE/REPLACE mode
+ files you can call fs_write() only once, the file creation is finished by it.
+ CREATE can return EEXIST here, if the destination file was already created.
+ With APPEND mode each fs_write() atomically appends the given data to
+ file. */
+int fs_write(struct fs_file *file, const void *data, size_t size);
+
+/* Write to file via output stream. The stream will be destroyed by
+ fs_write_stream_finish/abort. The returned ostream is already corked and
+ it doesn't need to be uncorked. */
+struct ostream *fs_write_stream(struct fs_file *file);
+/* Finish writing via stream, calling also o_stream_flush() on the stream and
+ handling any pending errors. The file will be created/replaced/appended only
+ after this call, same as with fs_write(). Anything written to the stream
+ won't be visible earlier. Returns 1 if ok, 0 if async write isn't finished
+ yet (retry calling fs_write_stream_finish_async()), -1 if error */
+int fs_write_stream_finish(struct fs_file *file, struct ostream **output);
+int fs_write_stream_finish_async(struct fs_file *file);
+/* Abort writing via stream. Anything written to the stream is discarded.
+ o_stream_ignore_last_errors() is called on the output stream so the caller
+ doesn't need to do it. This must not be called after
+ fs_write_stream_finish(), i.e. it can't be used to abort a pending async
+ write. */
+void fs_write_stream_abort_error(struct fs_file *file, struct ostream **output, const char *error_fmt, ...) ATTR_FORMAT(3, 4);
+
+/* Set a hash to the following write. The storage can then verify that the
+ input data matches the specified hash, or fail if it doesn't. Typically
+ implemented by Content-MD5 header. */
+void fs_write_set_hash(struct fs_file *file, const struct hash_method *method,
+ const void *digest);
+
+/* Call the specified callback whenever the file can be read/written to.
+ May call the callback immediately. */
+void fs_file_set_async_callback(struct fs_file *file,
+ fs_file_async_callback_t *callback,
+ void *context);
+#define fs_file_set_async_callback(file, callback, context) \
+ fs_file_set_async_callback(file, (fs_file_async_callback_t *)(callback), \
+ 1 ? (context) : \
+ CALLBACK_TYPECHECK(callback, void (*)(typeof(context))))
+/* Wait until some file can be read/written to more before returning.
+ It's an error to call this when there are no pending async operations. */
+void fs_wait_async(struct fs *fs);
+/* Switch the fs to the current ioloop. This can be used to do fs_wait_async()
+ among other IO work. Returns TRUE if there is actually some work that can
+ be waited on. */
+bool fs_switch_ioloop(struct fs *fs) ATTR_NOWARN_UNUSED_RESULT;
+
+/* Returns 1 if file exists, 0 if not, -1 if error occurred. */
+int fs_exists(struct fs_file *file);
+/* Delete a file. Returns 0 if file was actually deleted by us, -1 if error. */
+int fs_delete(struct fs_file *file);
+
+/* Returns 0 if ok, -1 if error occurred (e.g. errno=ENOENT).
+ All fs backends may not support all stat fields. */
+int fs_stat(struct fs_file *file, struct stat *st_r);
+/* Get number of links to the file. This is the same as using fs_stat()'s
+ st_nlinks field, except not all backends support returning it via fs_stat().
+ Returns 0 if ok, -1 if error occurred. */
+int fs_get_nlinks(struct fs_file *file, nlink_t *nlinks_r);
+/* Copy an object with possibly updated metadata. Destination parent
+ directories are created automatically. Returns 0 if ok, -1 if error
+ occurred. The "dest" file contains the error. */
+int fs_copy(struct fs_file *src, struct fs_file *dest);
+/* Try to finish asynchronous fs_copy(). Returns the same as fs_copy(). */
+int fs_copy_finish_async(struct fs_file *dest);
+/* Atomically rename a file. Destination parent directories are created
+ automatically. Returns 0 if ok, -1 if error occurred. The "dest" file
+ contains the error. */
+int fs_rename(struct fs_file *src, struct fs_file *dest);
+
+/* Exclusively lock a file. If file is already locked, wait for it for given
+ number of seconds (0 = fail immediately). Returns 1 if locked, 0 if wait
+ timed out, -1 if error. */
+int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r);
+void fs_unlock(struct fs_lock **lock);
+
+/* Iterate through all files or directories in the given directory.
+ Doesn't recurse to child directories. It's not an error to iterate a
+ nonexistent directory. */
+struct fs_iter *
+fs_iter_init(struct fs *fs, const char *path, enum fs_iter_flags flags);
+struct fs_iter *
+fs_iter_init_with_event(struct fs *fs, struct event *event,
+ const char *path, enum fs_iter_flags flags);
+/* Returns 0 if ok, -1 if iteration failed. */
+int fs_iter_deinit(struct fs_iter **iter, const char **error_r);
+/* Returns the next filename. */
+const char *fs_iter_next(struct fs_iter *iter);
+
+/* For asynchronous iterations: Specify the callback that is called whenever
+ there's more data available for reading. */
+void fs_iter_set_async_callback(struct fs_iter *iter,
+ fs_file_async_callback_t *callback,
+ void *context);
+#define fs_iter_set_async_callback(iter, callback, context) \
+ fs_iter_set_async_callback(iter, (fs_file_async_callback_t *)(callback), \
+ 1 ? (context) : \
+ CALLBACK_TYPECHECK(callback, void (*)(typeof(context))))
+/* For asynchronous iterations: If fs_iter_next() returns NULL, use this
+ function to determine if you should wait for more data or finish up. */
+bool fs_iter_have_more(struct fs_iter *iter);
+
+/* Return the filesystem's fs_stats. Note that each wrapper filesystem keeps
+ track of its own fs_stats calls. You can use fs_get_parent() to get to the
+ filesystem whose stats you want to see. */
+const struct fs_stats *fs_get_stats(struct fs *fs);
+
+/* Helper functions to count number of usecs for read/write operations. */
+uint64_t fs_stats_get_read_usecs(const struct fs_stats *stats);
+uint64_t fs_stats_get_write_usecs(const struct fs_stats *stats);
+
+#endif
diff --git a/src/lib-fs/fs-dict.c b/src/lib-fs/fs-dict.c
new file mode 100644
index 0000000..936e9b6
--- /dev/null
+++ b/src/lib-fs/fs-dict.c
@@ -0,0 +1,372 @@
+/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "guid.h"
+#include "hex-binary.h"
+#include "base64.h"
+#include "istream.h"
+#include "ostream.h"
+#include "dict.h"
+#include "fs-api-private.h"
+
+enum fs_dict_value_encoding {
+ FS_DICT_VALUE_ENCODING_RAW,
+ FS_DICT_VALUE_ENCODING_HEX,
+ FS_DICT_VALUE_ENCODING_BASE64
+};
+
+struct dict_fs {
+ struct fs fs;
+ struct dict *dict;
+ char *path_prefix;
+ enum fs_dict_value_encoding encoding;
+};
+
+struct dict_fs_file {
+ struct fs_file file;
+ pool_t pool;
+ const char *key, *value;
+ buffer_t *write_buffer;
+};
+
+struct dict_fs_iter {
+ struct fs_iter iter;
+ struct dict_iterate_context *dict_iter;
+};
+
+static struct fs *fs_dict_alloc(void)
+{
+ struct dict_fs *fs;
+
+ fs = i_new(struct dict_fs, 1);
+ fs->fs = fs_class_dict;
+ return &fs->fs;
+}
+
+static int
+fs_dict_init(struct fs *_fs, const char *args, const struct fs_settings *set,
+ const char **error_r)
+{
+ struct dict_fs *fs = (struct dict_fs *)_fs;
+ struct dict_settings dict_set;
+ const char *p, *encoding_str, *error;
+
+ p = strchr(args, ':');
+ if (p == NULL) {
+ *error_r = "':' missing in args";
+ return -1;
+ }
+ encoding_str = t_strdup_until(args, p++);
+ if (strcmp(encoding_str, "raw") == 0)
+ fs->encoding = FS_DICT_VALUE_ENCODING_RAW;
+ else if (strcmp(encoding_str, "hex") == 0)
+ fs->encoding = FS_DICT_VALUE_ENCODING_HEX;
+ else if (strcmp(encoding_str, "base64") == 0)
+ fs->encoding = FS_DICT_VALUE_ENCODING_BASE64;
+ else {
+ *error_r = t_strdup_printf("Unknown value encoding '%s'",
+ encoding_str);
+ return -1;
+ }
+
+ i_zero(&dict_set);
+ dict_set.base_dir = set->base_dir;
+ dict_set.event_parent = set->event_parent;
+
+ if (dict_init(p, &dict_set, &fs->dict, &error) < 0) {
+ *error_r = t_strdup_printf("dict_init(%s) failed: %s",
+ args, error);
+ return -1;
+ }
+ return 0;
+}
+
+static void fs_dict_free(struct fs *_fs)
+{
+ struct dict_fs *fs = (struct dict_fs *)_fs;
+
+ if (fs->dict != NULL) dict_deinit(&fs->dict);
+ i_free(fs);
+}
+
+static enum fs_properties fs_dict_get_properties(struct fs *fs ATTR_UNUSED)
+{
+ return FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER;
+}
+
+static struct fs_file *fs_dict_file_alloc(void)
+{
+ struct dict_fs_file *file;
+ pool_t pool;
+
+ pool = pool_alloconly_create("fs dict file", 128);
+ file = p_new(pool, struct dict_fs_file, 1);
+ file->pool = pool;
+ return &file->file;
+}
+
+static void
+fs_dict_file_init(struct fs_file *_file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags ATTR_UNUSED)
+{
+ struct dict_fs_file *file = (struct dict_fs_file *)_file;
+ struct dict_fs *fs = (struct dict_fs *)_file->fs;
+ guid_128_t guid;
+
+ i_assert(mode != FS_OPEN_MODE_APPEND); /* not supported */
+ i_assert(mode != FS_OPEN_MODE_CREATE); /* not supported */
+
+ if (mode != FS_OPEN_MODE_CREATE_UNIQUE_128)
+ file->file.path = p_strdup(file->pool, path);
+ else {
+ guid_128_generate(guid);
+ file->file.path = p_strdup_printf(file->pool, "%s/%s", path,
+ guid_128_to_string(guid));
+ }
+ file->key = fs->path_prefix == NULL ?
+ p_strdup(file->pool, file->file.path) :
+ p_strconcat(file->pool, fs->path_prefix, file->file.path, NULL);
+}
+
+static void fs_dict_file_deinit(struct fs_file *_file)
+{
+ struct dict_fs_file *file = (struct dict_fs_file *)_file;
+
+ i_assert(_file->output == NULL);
+
+ fs_file_free(_file);
+ pool_unref(&file->pool);
+}
+
+static bool fs_dict_prefetch(struct fs_file *_file ATTR_UNUSED,
+ uoff_t length ATTR_UNUSED)
+{
+ /* once async dict_lookup() is implemented, we want to start it here */
+ return TRUE;
+}
+
+static int fs_dict_lookup(struct dict_fs_file *file)
+{
+ struct dict_fs *fs = (struct dict_fs *)file->file.fs;
+ const char *error;
+ int ret;
+
+ if (file->value != NULL)
+ return 0;
+
+ struct dict_op_settings set = {
+ .username = file->file.fs->username,
+ };
+ ret = dict_lookup(fs->dict, &set, file->pool, file->key, &file->value, &error);
+ if (ret > 0)
+ return 0;
+ else if (ret < 0) {
+ fs_set_error(file->file.event, EIO,
+ "dict_lookup(%s) failed: %s", file->key, error);
+ return -1;
+ } else {
+ fs_set_error(file->file.event, ENOENT,
+ "Dict key %s doesn't exist", file->key);
+ return -1;
+ }
+}
+
+static struct istream *
+fs_dict_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED)
+{
+ struct dict_fs_file *file = (struct dict_fs_file *)_file;
+ struct istream *input;
+
+ if (fs_dict_lookup(file) < 0)
+ input = i_stream_create_error_str(errno, "%s", fs_file_last_error(_file));
+ else
+ input = i_stream_create_from_data(file->value, strlen(file->value));
+ i_stream_set_name(input, file->key);
+ return input;
+}
+
+static void fs_dict_write_stream(struct fs_file *_file)
+{
+ struct dict_fs_file *file = (struct dict_fs_file *)_file;
+
+ i_assert(_file->output == NULL);
+
+ file->write_buffer = buffer_create_dynamic(file->pool, 128);
+ _file->output = o_stream_create_buffer(file->write_buffer);
+ o_stream_set_name(_file->output, file->key);
+}
+
+static void fs_dict_write_rename_if_needed(struct dict_fs_file *file)
+{
+ struct dict_fs *fs = (struct dict_fs *)file->file.fs;
+ const char *new_fname;
+
+ new_fname = fs_metadata_find(&file->file.metadata, FS_METADATA_WRITE_FNAME);
+ if (new_fname == NULL)
+ return;
+
+ file->file.path = p_strdup(file->pool, new_fname);
+ file->key = fs->path_prefix == NULL ? p_strdup(file->pool, new_fname) :
+ p_strconcat(file->pool, fs->path_prefix, new_fname, NULL);
+}
+
+static int fs_dict_write_stream_finish(struct fs_file *_file, bool success)
+{
+ struct dict_fs_file *file = (struct dict_fs_file *)_file;
+ struct dict_fs *fs = (struct dict_fs *)_file->fs;
+ struct dict_transaction_context *trans;
+ const char *error;
+
+ o_stream_destroy(&_file->output);
+ if (!success)
+ return -1;
+
+ struct dict_op_settings set = {
+ .username = _file->fs->username,
+ };
+ fs_dict_write_rename_if_needed(file);
+ trans = dict_transaction_begin(fs->dict, &set);
+ switch (fs->encoding) {
+ case FS_DICT_VALUE_ENCODING_RAW:
+ dict_set(trans, file->key, str_c(file->write_buffer));
+ break;
+ case FS_DICT_VALUE_ENCODING_HEX: {
+ string_t *hex = t_str_new(file->write_buffer->used * 2 + 1);
+ binary_to_hex_append(hex, file->write_buffer->data,
+ file->write_buffer->used);
+ dict_set(trans, file->key, str_c(hex));
+ break;
+ }
+ case FS_DICT_VALUE_ENCODING_BASE64: {
+ const size_t base64_size =
+ MAX_BASE64_ENCODED_SIZE(file->write_buffer->used);
+ string_t *base64 = t_str_new(base64_size);
+ base64_encode(file->write_buffer->data,
+ file->write_buffer->used, base64);
+ dict_set(trans, file->key, str_c(base64));
+ }
+ }
+ if (dict_transaction_commit(&trans, &error) < 0) {
+ fs_set_error(_file->event, EIO,
+ "Dict transaction commit failed: %s", error);
+ return -1;
+ }
+ return 1;
+}
+
+static int fs_dict_stat(struct fs_file *_file, struct stat *st_r)
+{
+ struct dict_fs_file *file = (struct dict_fs_file *)_file;
+
+ i_zero(st_r);
+
+ if (fs_dict_lookup(file) < 0)
+ return -1;
+ st_r->st_size = strlen(file->value);
+ return 0;
+}
+
+static int fs_dict_delete(struct fs_file *_file)
+{
+ struct dict_fs_file *file = (struct dict_fs_file *)_file;
+ struct dict_fs *fs = (struct dict_fs *)_file->fs;
+ struct dict_transaction_context *trans;
+ const char *error;
+
+ struct dict_op_settings set = {
+ .username = fs->fs.username,
+ };
+ trans = dict_transaction_begin(fs->dict, &set);
+ dict_unset(trans, file->key);
+ if (dict_transaction_commit(&trans, &error) < 0) {
+ fs_set_error(_file->event, EIO,
+ "Dict transaction commit failed: %s", error);
+ return -1;
+ }
+ return 0;
+}
+
+static struct fs_iter *fs_dict_iter_alloc(void)
+{
+ struct dict_fs_iter *iter = i_new(struct dict_fs_iter, 1);
+ return &iter->iter;
+}
+
+static void
+fs_dict_iter_init(struct fs_iter *_iter, const char *path,
+ enum fs_iter_flags flags ATTR_UNUSED)
+{
+ struct dict_fs_iter *iter = (struct dict_fs_iter *)_iter;
+ struct dict_fs *fs = (struct dict_fs *)_iter->fs;
+
+ if (fs->path_prefix != NULL)
+ path = t_strconcat(fs->path_prefix, path, NULL);
+
+ struct dict_op_settings set = {
+ .username = iter->iter.fs->username,
+ };
+ iter->dict_iter = dict_iterate_init(fs->dict, &set, path, 0);
+}
+
+static const char *fs_dict_iter_next(struct fs_iter *_iter)
+{
+ struct dict_fs_iter *iter = (struct dict_fs_iter *)_iter;
+ const char *key, *value;
+
+ if (!dict_iterate(iter->dict_iter, &key, &value))
+ return NULL;
+ return key;
+}
+
+static int fs_dict_iter_deinit(struct fs_iter *_iter)
+{
+ struct dict_fs_iter *iter = (struct dict_fs_iter *)_iter;
+ const char *error;
+ int ret;
+
+ ret = dict_iterate_deinit(&iter->dict_iter, &error);
+ if (ret < 0)
+ fs_set_error(_iter->event, EIO,
+ "Dict iteration failed: %s", error);
+ return ret;
+}
+
+const struct fs fs_class_dict = {
+ .name = "dict",
+ .v = {
+ fs_dict_alloc,
+ fs_dict_init,
+ NULL,
+ fs_dict_free,
+ fs_dict_get_properties,
+ fs_dict_file_alloc,
+ fs_dict_file_init,
+ fs_dict_file_deinit,
+ NULL,
+ NULL,
+ NULL, NULL,
+ fs_default_set_metadata,
+ NULL,
+ fs_dict_prefetch,
+ NULL,
+ fs_dict_read_stream,
+ NULL,
+ fs_dict_write_stream,
+ fs_dict_write_stream_finish,
+ NULL,
+ NULL,
+ NULL,
+ fs_dict_stat,
+ fs_default_copy,
+ NULL,
+ fs_dict_delete,
+ fs_dict_iter_alloc,
+ fs_dict_iter_init,
+ fs_dict_iter_next,
+ fs_dict_iter_deinit,
+ NULL,
+ NULL
+ }
+};
diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
new file mode 100644
index 0000000..22517e7
--- /dev/null
+++ b/src/lib-fs/fs-metawrap.c
@@ -0,0 +1,526 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "strescape.h"
+#include "istream.h"
+#include "istream-private.h"
+#include "istream-concat.h"
+#include "istream-metawrap.h"
+#include "ostream.h"
+#include "ostream-metawrap.h"
+#include "iostream-temp.h"
+#include "fs-api-private.h"
+
+struct metawrap_fs {
+ struct fs fs;
+ bool wrap_metadata;
+};
+
+struct metawrap_fs_file {
+ struct fs_file file;
+ struct metawrap_fs *fs;
+ struct fs_file *super_read;
+ enum fs_open_mode open_mode;
+ struct istream *input;
+ bool metadata_read;
+
+ struct ostream *super_output;
+ struct ostream *temp_output;
+ string_t *metadata_header;
+ uoff_t metadata_write_size;
+ bool metadata_changed_since_write;
+};
+
+#define METAWRAP_FS(ptr) container_of((ptr), struct metawrap_fs, fs)
+#define METAWRAP_FILE(ptr) container_of((ptr), struct metawrap_fs_file, file)
+
+static struct fs *fs_metawrap_alloc(void)
+{
+ struct metawrap_fs *fs;
+
+ fs = i_new(struct metawrap_fs, 1);
+ fs->fs = fs_class_metawrap;
+ return &fs->fs;
+}
+
+static int
+fs_metawrap_init(struct fs *_fs, const char *args,
+ const struct fs_settings *set, const char **error_r)
+{
+ struct metawrap_fs *fs = METAWRAP_FS(_fs);
+ const char *parent_name, *parent_args;
+
+ if (*args == '\0') {
+ *error_r = "Parent filesystem not given as parameter";
+ return -1;
+ }
+
+ parent_args = strchr(args, ':');
+ if (parent_args == NULL) {
+ parent_name = args;
+ parent_args = "";
+ } else {
+ parent_name = t_strdup_until(args, parent_args);
+ parent_args++;
+ }
+ if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0)
+ return -1;
+ if ((fs_get_properties(_fs->parent) & FS_PROPERTY_METADATA) == 0)
+ fs->wrap_metadata = TRUE;
+ return 0;
+}
+
+static void fs_metawrap_free(struct fs *_fs)
+{
+ struct metawrap_fs *fs = METAWRAP_FS(_fs);
+
+ i_free(fs);
+}
+
+static enum fs_properties fs_metawrap_get_properties(struct fs *_fs)
+{
+ const struct metawrap_fs *fs = METAWRAP_FS(_fs);
+ enum fs_properties props;
+
+ props = fs_get_properties(_fs->parent);
+ if (fs->wrap_metadata) {
+ /* we don't have a quick stat() to see the file's size,
+ because of the metadata header */
+ props &= ENUM_NEGATE(FS_PROPERTY_STAT);
+ /* Copying can copy the whole metadata. */
+ props |= FS_PROPERTY_COPY_METADATA;
+ }
+ return props;
+}
+
+static struct fs_file *fs_metawrap_file_alloc(void)
+{
+ struct metawrap_fs_file *file = i_new(struct metawrap_fs_file, 1);
+ return &file->file;
+}
+
+static void
+fs_metawrap_file_init(struct fs_file *_file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+ struct metawrap_fs *fs = METAWRAP_FS(_file->fs);
+
+ file->file.path = i_strdup(path);
+ file->fs = fs;
+ file->open_mode = mode;
+
+ /* avoid unnecessarily creating two seekable streams */
+ flags &= ENUM_NEGATE(FS_OPEN_FLAG_SEEKABLE);
+
+ file->file.parent = fs_file_init_parent(_file, path, mode, flags);
+ if (file->fs->wrap_metadata && mode == FS_OPEN_MODE_READONLY &&
+ (flags & FS_OPEN_FLAG_ASYNC) == 0) {
+ /* use async stream for parent, so fs_read_stream() won't create
+ another seekable stream needlessly */
+ file->super_read = fs_file_init_parent(_file, path,
+ mode, flags | FS_OPEN_FLAG_ASYNC |
+ FS_OPEN_FLAG_ASYNC_NOQUEUE);
+ } else {
+ file->super_read = file->file.parent;
+ }
+ fs_metadata_init(&file->file);
+}
+
+static void fs_metawrap_file_deinit(struct fs_file *_file)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+
+ if (file->super_read != _file->parent)
+ fs_file_deinit(&file->super_read);
+ str_free(&file->metadata_header);
+ fs_file_free(_file);
+ i_free(file->file.path);
+ i_free(file);
+}
+
+static void fs_metawrap_file_close(struct fs_file *_file)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+
+ i_stream_unref(&file->input);
+ fs_file_close(file->super_read);
+ fs_file_close(_file->parent);
+}
+
+static void
+fs_metawrap_set_metadata(struct fs_file *_file, const char *key,
+ const char *value)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+
+ if (!file->fs->wrap_metadata ||
+ strcmp(key, FS_METADATA_WRITE_FNAME) == 0)
+ fs_set_metadata(_file->parent, key, value);
+ else {
+ fs_default_set_metadata(_file, key, value);
+ file->metadata_changed_since_write = TRUE;
+ }
+}
+
+static int
+fs_metawrap_get_metadata(struct fs_file *_file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+ ssize_t ret;
+ char c;
+
+ if (!file->fs->wrap_metadata)
+ return fs_get_metadata_full(_file->parent, flags, metadata_r);
+
+ if (file->metadata_read) {
+ /* we have the metadata */
+ } else if ((flags & FS_GET_METADATA_FLAG_LOADED_ONLY) != 0) {
+ /* use the existing metadata only */
+ } else if (file->input == NULL) {
+ if (fs_read(_file, &c, 1) < 0)
+ return -1;
+ } else {
+ /* use the existing istream to read it */
+ while ((ret = i_stream_read(file->input)) == 0) {
+ if (file->metadata_read)
+ break;
+
+ i_assert(!file->input->blocking);
+ fs_wait_async(_file->fs);
+ }
+ if (ret == -1 && file->input->stream_errno != 0) {
+ fs_set_error(_file->event, file->input->stream_errno,
+ "read(%s) failed: %s",
+ i_stream_get_name(file->input),
+ i_stream_get_error(file->input));
+ return -1;
+ }
+ i_assert(file->metadata_read);
+ }
+ *metadata_r = &_file->metadata;
+ return 0;
+}
+
+static bool fs_metawrap_prefetch(struct fs_file *_file, uoff_t length)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+
+ if (!file->fs->wrap_metadata)
+ return fs_prefetch(_file->parent, length);
+ else
+ return fs_prefetch(file->super_read, length);
+}
+
+static ssize_t fs_metawrap_read(struct fs_file *_file, void *buf, size_t size)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+
+ if (!file->fs->wrap_metadata)
+ return fs_read(_file->parent, buf, size);
+ return fs_read_via_stream(_file, buf, size);
+}
+
+static void
+fs_metawrap_callback(const char *key, const char *value, void *context)
+{
+ struct metawrap_fs_file *file = context;
+
+ if (key == NULL) {
+ file->metadata_read = TRUE;
+ return;
+ }
+
+ T_BEGIN {
+ key = str_tabunescape(t_strdup_noconst(key));
+ value = str_tabunescape(t_strdup_noconst(value));
+ fs_default_set_metadata(&file->file, key, value);
+ } T_END;
+}
+
+static struct istream *
+fs_metawrap_read_stream(struct fs_file *_file, size_t max_buffer_size)
+{
+ struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file;
+ struct istream *input;
+
+ if (!file->fs->wrap_metadata)
+ return fs_read_stream(_file->parent, max_buffer_size);
+
+ if (file->input != NULL) {
+ i_stream_ref(file->input);
+ i_stream_seek(file->input, 0);
+ return file->input;
+ }
+
+ input = fs_read_stream(file->super_read, max_buffer_size);
+ file->input = i_stream_create_metawrap(input, fs_metawrap_callback, file);
+ i_stream_unref(&input);
+ i_stream_ref(file->input);
+ return file->input;
+}
+
+static int fs_metawrap_write(struct fs_file *_file, const void *data, size_t size)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+
+ if (!file->fs->wrap_metadata)
+ return fs_write(_file->parent, data, size);
+ return fs_write_via_stream(_file, data, size);
+}
+
+static void
+fs_metawrap_append_metadata(struct metawrap_fs_file *file, string_t *str)
+{
+ const struct fs_metadata *metadata;
+
+ array_foreach(&file->file.metadata, metadata) {
+ if (str_begins(metadata->key, FS_METADATA_INTERNAL_PREFIX))
+ continue;
+
+ str_append_tabescaped(str, metadata->key);
+ str_append_c(str, ':');
+ str_append_tabescaped(str, metadata->value);
+ str_append_c(str, '\n');
+ }
+ str_append_c(str, '\n');
+}
+
+static void
+fs_metawrap_write_metadata_to(struct metawrap_fs_file *file,
+ struct ostream *output)
+{
+ string_t *str = t_str_new(256);
+ ssize_t ret;
+
+ fs_metawrap_append_metadata(file, str);
+ file->metadata_write_size = str_len(str);
+
+ ret = o_stream_send(output, str_data(str), str_len(str));
+ if (ret < 0)
+ o_stream_close(output);
+ else
+ i_assert((size_t)ret == str_len(str));
+ file->metadata_changed_since_write = FALSE;
+}
+
+static void fs_metawrap_write_metadata(void *context)
+{
+ struct metawrap_fs_file *file = context;
+
+ fs_metawrap_write_metadata_to(file, file->file.output);
+}
+
+static void fs_metawrap_write_stream(struct fs_file *_file)
+{
+ struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file;
+
+ i_assert(_file->output == NULL);
+
+ if (!file->fs->wrap_metadata) {
+ file->super_output = fs_write_stream(_file->parent);
+ _file->output = file->super_output;
+ } else {
+ file->temp_output =
+ iostream_temp_create_named(_file->fs->temp_path_prefix,
+ IOSTREAM_TEMP_FLAG_TRY_FD_DUP,
+ fs_file_path(_file));
+ _file->output = o_stream_create_metawrap(file->temp_output,
+ fs_metawrap_write_metadata, file);
+ }
+}
+
+static struct istream *
+fs_metawrap_create_updated_istream(struct metawrap_fs_file *file,
+ struct istream *input)
+{
+ struct istream *input2, *inputs[3];
+
+ if (file->metadata_header != NULL)
+ str_truncate(file->metadata_header, 0);
+ else
+ file->metadata_header = str_new(default_pool, 1024);
+ fs_metawrap_append_metadata(file, file->metadata_header);
+ inputs[0] = i_stream_create_from_data(str_data(file->metadata_header),
+ str_len(file->metadata_header));
+
+ i_stream_seek(input, file->metadata_write_size);
+ inputs[1] = i_stream_create_limit(input, UOFF_T_MAX);
+ inputs[2] = NULL;
+ input2 = i_stream_create_concat(inputs);
+ i_stream_unref(&inputs[0]);
+ i_stream_unref(&inputs[1]);
+
+ file->metadata_write_size = str_len(file->metadata_header);
+ return input2;
+}
+
+static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+ struct istream *input;
+ int ret;
+
+ if (_file->output != NULL) {
+ if (_file->output == file->super_output)
+ _file->output = NULL;
+ else
+ o_stream_unref(&_file->output);
+ }
+ if (!success) {
+ if (file->super_output != NULL) {
+ /* no metawrap */
+ i_assert(file->temp_output == NULL);
+ fs_write_stream_abort_parent(_file, &file->super_output);
+ } else {
+ i_assert(file->temp_output != NULL);
+ o_stream_destroy(&file->temp_output);
+ }
+ return -1;
+ }
+
+ if (file->super_output != NULL) {
+ /* no metawrap */
+ i_assert(file->temp_output == NULL);
+ return fs_write_stream_finish(_file->parent, &file->super_output);
+ }
+ if (file->temp_output == NULL) {
+ /* finishing up */
+ i_assert(file->super_output == NULL);
+ return fs_write_stream_finish_async(_file->parent);
+ }
+ /* finish writing the temporary file */
+ if (file->temp_output->offset == 0) {
+ /* empty file - temp_output is already finished,
+ so we can't write to it. To make sure metadata is still
+ appended/written to file use metadata_changed_since_write */
+ file->metadata_changed_since_write = TRUE;
+ }
+ input = iostream_temp_finish(&file->temp_output, IO_BLOCK_SIZE);
+ if (file->metadata_changed_since_write) {
+ /* we'll need to recreate the metadata. do this by creating a
+ new istream combining the new metadata header and the
+ old body. */
+ struct istream *input2 = input;
+
+ input = fs_metawrap_create_updated_istream(file, input);
+ i_stream_unref(&input2);
+ }
+ file->super_output = fs_write_stream(_file->parent);
+ o_stream_nsend_istream(file->super_output, input);
+ /* because of the "end of metadata" LF, there's always at least
+ 1 byte */
+ i_assert(file->super_output->offset > 0 ||
+ file->super_output->stream_errno != 0);
+ ret = fs_write_stream_finish(_file->parent, &file->super_output);
+ i_stream_unref(&input);
+ return ret;
+}
+
+static int fs_metawrap_stat(struct fs_file *_file, struct stat *st_r)
+{
+ struct metawrap_fs_file *file = METAWRAP_FILE(_file);
+ struct istream *input;
+ uoff_t input_size;
+ ssize_t ret;
+
+ if (!file->fs->wrap_metadata)
+ return fs_stat(_file->parent, st_r);
+
+ if (file->metadata_write_size != 0) {
+ /* fs_stat() after a write. we can do this quickly. */
+ if (fs_stat(_file->parent, st_r) < 0)
+ return -1;
+ if ((uoff_t)st_r->st_size < file->metadata_write_size) {
+ fs_set_error(_file->event, EIO,
+ "Just-written %s shrank unexpectedly "
+ "(%"PRIuUOFF_T" < %"PRIuUOFF_T")",
+ fs_file_path(_file), st_r->st_size,
+ file->metadata_write_size);
+ return -1;
+ }
+ st_r->st_size -= file->metadata_write_size;
+ return 0;
+ }
+
+ if (file->input == NULL)
+ input = fs_read_stream(_file, IO_BLOCK_SIZE);
+ else {
+ input = file->input;
+ i_stream_ref(input);
+ }
+ if ((ret = i_stream_get_size(input, TRUE, &input_size)) < 0) {
+ fs_set_error(_file->event, input->stream_errno,
+ "i_stream_get_size(%s) failed: %s",
+ fs_file_path(_file), i_stream_get_error(input));
+ i_stream_unref(&input);
+ return -1;
+ }
+ i_stream_unref(&input);
+ if (ret == 0) {
+ /* we shouldn't get here */
+ fs_set_error(_file->event, EIO, "i_stream_get_size(%s) returned size as unknown",
+ fs_file_path(_file));
+ return -1;
+ }
+
+ if (fs_stat(_file->parent, st_r) < 0) {
+ i_assert(errno != EAGAIN); /* read should have caught this */
+ return -1;
+ }
+ st_r->st_size = input_size;
+ return 0;
+}
+
+static int fs_metawrap_copy(struct fs_file *_src, struct fs_file *_dest)
+{
+ struct metawrap_fs_file *dest = METAWRAP_FILE(_dest);
+
+ if (!dest->fs->wrap_metadata || !_dest->metadata_changed)
+ return fs_wrapper_copy(_src, _dest);
+ else
+ return fs_default_copy(_src, _dest);
+}
+
+const struct fs fs_class_metawrap = {
+ .name = "metawrap",
+ .v = {
+ fs_metawrap_alloc,
+ fs_metawrap_init,
+ NULL,
+ fs_metawrap_free,
+ fs_metawrap_get_properties,
+ fs_metawrap_file_alloc,
+ fs_metawrap_file_init,
+ fs_metawrap_file_deinit,
+ fs_metawrap_file_close,
+ fs_wrapper_file_get_path,
+ fs_wrapper_set_async_callback,
+ fs_wrapper_wait_async,
+ fs_metawrap_set_metadata,
+ fs_metawrap_get_metadata,
+ fs_metawrap_prefetch,
+ fs_metawrap_read,
+ fs_metawrap_read_stream,
+ fs_metawrap_write,
+ fs_metawrap_write_stream,
+ fs_metawrap_write_stream_finish,
+ fs_wrapper_lock,
+ fs_wrapper_unlock,
+ fs_wrapper_exists,
+ fs_metawrap_stat,
+ fs_metawrap_copy,
+ fs_wrapper_rename,
+ fs_wrapper_delete,
+ fs_wrapper_iter_alloc,
+ fs_wrapper_iter_init,
+ fs_wrapper_iter_next,
+ fs_wrapper_iter_deinit,
+ NULL,
+ fs_wrapper_get_nlinks,
+ }
+};
diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c
new file mode 100644
index 0000000..657938e
--- /dev/null
+++ b/src/lib-fs/fs-posix.c
@@ -0,0 +1,1028 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "guid.h"
+#include "istream.h"
+#include "ostream.h"
+#include "safe-mkstemp.h"
+#include "mkdir-parents.h"
+#include "write-full.h"
+#include "file-lock.h"
+#include "file-dotlock.h"
+#include "time-util.h"
+#include "fs-api-private.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+#define FS_POSIX_DOTLOCK_STALE_TIMEOUT_SECS (60*10)
+#define MAX_MKDIR_RETRY_COUNT 5
+#define FS_DEFAULT_MODE 0600
+
+enum fs_posix_lock_method {
+ FS_POSIX_LOCK_METHOD_FLOCK,
+ FS_POSIX_LOCK_METHOD_DOTLOCK
+};
+
+struct posix_fs {
+ struct fs fs;
+ char *temp_file_prefix, *root_path, *path_prefix;
+ size_t temp_file_prefix_len;
+ enum fs_posix_lock_method lock_method;
+ mode_t mode;
+ bool mode_auto;
+ bool have_dirs;
+ bool disable_fsync;
+ bool accurate_mtime;
+};
+
+struct posix_fs_file {
+ struct fs_file file;
+ char *temp_path, *full_path;
+ int fd;
+ enum fs_open_mode open_mode;
+ enum fs_open_flags open_flags;
+
+ buffer_t *write_buf;
+
+ bool seek_to_beginning;
+};
+
+struct posix_fs_lock {
+ struct fs_lock lock;
+ struct file_lock *file_lock;
+ struct dotlock *dotlock;
+};
+
+struct posix_fs_iter {
+ struct fs_iter iter;
+ char *path;
+ DIR *dir;
+ int err;
+};
+
+static struct fs *fs_posix_alloc(void)
+{
+ struct posix_fs *fs;
+
+ fs = i_new(struct posix_fs, 1);
+ fs->fs = fs_class_posix;
+ return &fs->fs;
+}
+
+static int
+fs_posix_init(struct fs *_fs, const char *args, const struct fs_settings *set,
+ const char **error_r)
+{
+ struct posix_fs *fs = container_of(_fs, struct posix_fs, fs);
+ const char *const *tmp;
+
+ fs->temp_file_prefix = set->temp_file_prefix != NULL ?
+ i_strdup(set->temp_file_prefix) : i_strdup("temp.dovecot.");
+ fs->temp_file_prefix_len = strlen(fs->temp_file_prefix);
+ fs->root_path = i_strdup(set->root_path);
+ fs->fs.set.temp_file_prefix = fs->temp_file_prefix;
+ fs->fs.set.root_path = fs->root_path;
+
+ fs->lock_method = FS_POSIX_LOCK_METHOD_FLOCK;
+ fs->mode = FS_DEFAULT_MODE;
+
+ tmp = t_strsplit_spaces(args, ":");
+ for (; *tmp != NULL; tmp++) {
+ const char *arg = *tmp;
+
+ if (strcmp(arg, "lock=flock") == 0)
+ fs->lock_method = FS_POSIX_LOCK_METHOD_FLOCK;
+ else if (strcmp(arg, "lock=dotlock") == 0)
+ fs->lock_method = FS_POSIX_LOCK_METHOD_DOTLOCK;
+ else if (str_begins(arg, "prefix=")) {
+ i_free(fs->path_prefix);
+ fs->path_prefix = i_strdup(arg + 7);
+ } else if (strcmp(arg, "mode=auto") == 0) {
+ fs->mode_auto = TRUE;
+ } else if (strcmp(arg, "dirs") == 0) {
+ fs->have_dirs = TRUE;
+ } else if (strcmp(arg, "no-fsync") == 0) {
+ fs->disable_fsync = TRUE;
+ } else if (strcmp(arg, "accurate-mtime") == 0) {
+ fs->accurate_mtime = TRUE;
+ } else if (str_begins(arg, "mode=")) {
+ unsigned int mode;
+ if (str_to_uint_oct(arg+5, &mode) < 0) {
+ *error_r = t_strdup_printf("Invalid mode value: %s", arg+5);
+ return -1;
+ }
+ fs->mode = mode & 0666;
+ if (fs->mode == 0) {
+ *error_r = t_strdup_printf("Invalid mode: %s", arg+5);
+ return -1;
+ }
+ } else {
+ *error_r = t_strdup_printf("Unknown arg '%s'", arg);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static void fs_posix_free(struct fs *_fs)
+{
+ struct posix_fs *fs = container_of(_fs, struct posix_fs, fs);
+
+ i_free(fs->temp_file_prefix);
+ i_free(fs->root_path);
+ i_free(fs->path_prefix);
+ i_free(fs);
+}
+
+static enum fs_properties fs_posix_get_properties(struct fs *_fs)
+{
+ struct posix_fs *fs = container_of(_fs, struct posix_fs, fs);
+ enum fs_properties props =
+ FS_PROPERTY_LOCKS | FS_PROPERTY_FASTCOPY | FS_PROPERTY_RENAME |
+ FS_PROPERTY_STAT | FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER;
+
+ /* FS_PROPERTY_DIRECTORIES is not returned normally because fs_delete()
+ automatically rmdir()s parents. For backwards compatibility
+ (especially with SIS code) we'll do it that way, but optionally with
+ "dirs" parameter enable them. This is especially important to be
+ able to use doveadm fs commands to delete empty directories. */
+ if (fs->have_dirs)
+ props |= FS_PROPERTY_DIRECTORIES;
+ return props;
+}
+
+static int
+fs_posix_get_mode(struct posix_fs_file *file, const char *path, mode_t *mode_r)
+{
+ struct posix_fs *fs = (struct posix_fs *)file->file.fs;
+ struct stat st;
+ const char *p;
+
+ *mode_r = fs->mode;
+
+ /* This function is used to get mode of the parent directory, so path
+ is never the same as file->path. The file is used just to set the
+ errors. */
+ while (stat(path, &st) < 0) {
+ if (errno != ENOENT) {
+ fs_set_error_errno(file->file.event,
+ "stat(%s) failed: %m", path);
+ return -1;
+ }
+ p = strrchr(path, '/');
+ if (p != NULL)
+ path = t_strdup_until(path, p);
+ else if (strcmp(path, ".") != 0)
+ path = ".";
+ else
+ return 0;
+ }
+ if ((st.st_mode & S_ISGID) != 0) {
+ /* setgid set - copy mode from parent */
+ *mode_r = st.st_mode & 0666;
+ }
+ return 0;
+}
+
+static int fs_posix_mkdir_parents(struct posix_fs_file *file, const char *path)
+{
+ const char *dir, *fname;
+ mode_t mode, dir_mode;
+
+ fname = strrchr(path, '/');
+ if (fname == NULL)
+ return 1;
+ dir = t_strdup_until(path, fname);
+
+ if (fs_posix_get_mode(file, dir, &mode) < 0)
+ return -1;
+ dir_mode = mode;
+ if ((dir_mode & 0600) != 0) dir_mode |= 0100;
+ if ((dir_mode & 0060) != 0) dir_mode |= 0010;
+ if ((dir_mode & 0006) != 0) dir_mode |= 0001;
+
+ if (mkdir_parents(dir, dir_mode) == 0)
+ return 0;
+ else if (errno == EEXIST)
+ return 1;
+ else {
+ fs_set_error_errno(file->file.event,
+ "mkdir_parents(%s) failed: %m", dir);
+ return -1;
+ }
+}
+
+static int fs_posix_rmdir_parents(struct posix_fs_file *file, const char *path)
+{
+ struct posix_fs *fs = (struct posix_fs *)file->file.fs;
+ const char *p;
+
+ if (fs->have_dirs)
+ return 0;
+ if (fs->root_path == NULL && fs->path_prefix == NULL)
+ return 0;
+
+ while ((p = strrchr(path, '/')) != NULL) {
+ path = t_strdup_until(path, p);
+ if ((fs->root_path != NULL && strcmp(path, fs->root_path) == 0) ||
+ (fs->path_prefix != NULL && str_begins(fs->path_prefix, path)))
+ break;
+ if (rmdir(path) == 0) {
+ /* success, continue to parent */
+ } else if (errno == ENOTEMPTY || errno == EEXIST) {
+ /* there are other entries in this directory */
+ break;
+ } else if (errno == EBUSY || errno == ENOENT) {
+ /* some other not-unexpected error */
+ break;
+ } else {
+ fs_set_error_errno(file->file.event,
+ "rmdir(%s) failed: %m", path);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int fs_posix_create(struct posix_fs_file *file)
+{
+ struct posix_fs *fs = container_of(file->file.fs, struct posix_fs, fs);
+ string_t *str = t_str_new(256);
+ const char *slash;
+ unsigned int try_count = 0;
+ mode_t mode;
+ int fd;
+
+ i_assert(file->temp_path == NULL);
+
+ if ((slash = strrchr(file->full_path, '/')) != NULL) {
+ str_append_data(str, file->full_path, slash - file->full_path);
+ if (fs_posix_get_mode(file, str_c(str), &mode) < 0)
+ return -1;
+ str_append_c(str, '/');
+ } else {
+ if (fs_posix_get_mode(file, ".", &mode) < 0)
+ return -1;
+ }
+ str_append(str, fs->temp_file_prefix);
+
+ fd = safe_mkstemp_hostpid(str, mode, (uid_t)-1, (gid_t)-1);
+ while (fd == -1 && errno == ENOENT &&
+ try_count <= MAX_MKDIR_RETRY_COUNT) {
+ if (fs_posix_mkdir_parents(file, str_c(str)) < 0)
+ return -1;
+ fd = safe_mkstemp_hostpid(str, mode, (uid_t)-1, (gid_t)-1);
+ try_count++;
+ }
+ if (fd == -1) {
+ fs_set_error_errno(file->file.event,
+ "safe_mkstemp(%s) failed: %m", str_c(str));
+ return -1;
+ }
+ file->temp_path = i_strdup(str_c(str));
+ return fd;
+}
+
+static int fs_posix_open(struct posix_fs_file *file)
+{
+ const char *path = file->full_path;
+
+ i_assert(file->fd == -1);
+
+ switch (file->open_mode) {
+ case FS_OPEN_MODE_READONLY:
+ file->fd = open(path, O_RDONLY);
+ if (file->fd == -1)
+ fs_set_error_errno(file->file.event,
+ "open(%s) failed: %m", path);
+ break;
+ case FS_OPEN_MODE_APPEND:
+ file->fd = open(path, O_RDWR | O_APPEND);
+ if (file->fd == -1)
+ fs_set_error_errno(file->file.event,
+ "open(%s) failed: %m", path);
+ break;
+ case FS_OPEN_MODE_CREATE_UNIQUE_128:
+ case FS_OPEN_MODE_CREATE:
+ case FS_OPEN_MODE_REPLACE:
+ T_BEGIN {
+ file->fd = fs_posix_create(file);
+ } T_END;
+ break;
+ }
+ if (file->fd == -1)
+ return -1;
+ return 0;
+}
+
+static struct fs_file *fs_posix_file_alloc(void)
+{
+ struct posix_fs_file *file = i_new(struct posix_fs_file, 1);
+ return &file->file;
+}
+
+static void
+fs_posix_file_init(struct fs_file *_file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+ struct posix_fs *fs = container_of(_file->fs, struct posix_fs, fs);
+ guid_128_t guid;
+ size_t path_len = strlen(path);
+
+ if (path_len > 0 && path[path_len-1] == '/') {
+ /* deleting "path/" (used e.g. by doveadm fs delete) - strip
+ out the trailing "/" since it doesn't work well with NFS. */
+ path = t_strndup(path, path_len-1);
+ }
+
+ if (mode != FS_OPEN_MODE_CREATE_UNIQUE_128)
+ file->file.path = i_strdup(path);
+ else {
+ guid_128_generate(guid);
+ file->file.path = i_strdup_printf("%s/%s", path,
+ guid_128_to_string(guid));
+ }
+ file->full_path = fs->path_prefix == NULL ? i_strdup(file->file.path) :
+ i_strconcat(fs->path_prefix, file->file.path, NULL);
+ file->open_mode = mode;
+ file->open_flags = flags;
+ file->fd = -1;
+}
+
+static void fs_posix_file_close(struct fs_file *_file)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+
+ if (file->fd != -1 && file->file.output == NULL) {
+ if (close(file->fd) < 0) {
+ e_error(_file->event, "close(%s) failed: %m",
+ file->full_path);
+ }
+ file->fd = -1;
+ }
+}
+
+static void fs_posix_file_deinit(struct fs_file *_file)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+
+ i_assert(_file->output == NULL);
+
+ switch (file->open_mode) {
+ case FS_OPEN_MODE_READONLY:
+ case FS_OPEN_MODE_APPEND:
+ break;
+ case FS_OPEN_MODE_CREATE_UNIQUE_128:
+ case FS_OPEN_MODE_CREATE:
+ case FS_OPEN_MODE_REPLACE:
+ if (file->temp_path == NULL)
+ break;
+ /* failed to create/replace this. delete the temp file */
+ if (unlink(file->temp_path) < 0) {
+ e_error(_file->event, "unlink(%s) failed: %m",
+ file->temp_path);
+ }
+ break;
+ }
+
+ fs_file_free(_file);
+ i_free(file->temp_path);
+ i_free(file->full_path);
+ i_free(file->file.path);
+ i_free(file);
+}
+
+static int fs_posix_open_for_read(struct posix_fs_file *file)
+{
+ i_assert(file->file.output == NULL);
+ i_assert(file->temp_path == NULL);
+
+ if (file->fd == -1) {
+ if (fs_posix_open(file) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static bool fs_posix_prefetch(struct fs_file *_file, uoff_t length ATTR_UNUSED)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+
+ if (fs_posix_open_for_read(file) < 0)
+ return TRUE;
+
+/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */
+#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
+ if (posix_fadvise(file->fd, 0, length, POSIX_FADV_WILLNEED) < 0) {
+ e_error(_file->event, "posix_fadvise(%s) failed: %m", file->full_path);
+ return TRUE;
+ }
+#endif
+ return FALSE;
+}
+
+static ssize_t fs_posix_read(struct fs_file *_file, void *buf, size_t size)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+ ssize_t ret;
+
+ if (fs_posix_open_for_read(file) < 0)
+ return -1;
+
+ if (file->seek_to_beginning) {
+ file->seek_to_beginning = FALSE;
+ if (lseek(file->fd, 0, SEEK_SET) < 0) {
+ fs_set_error_errno(_file->event,
+ "lseek(%s, 0) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ }
+
+ ret = read(file->fd, buf, size);
+ if (ret < 0)
+ fs_set_error_errno(_file->event, "read(%s) failed: %m",
+ file->full_path);
+ fs_posix_file_close(_file);
+ return ret;
+}
+
+static struct istream *
+fs_posix_read_stream(struct fs_file *_file, size_t max_buffer_size)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+ struct istream *input;
+ int fd_dup;
+
+ if (fs_posix_open_for_read(file) < 0)
+ input = i_stream_create_error_str(errno, "%s", fs_file_last_error(_file));
+ else if ((fd_dup = dup(file->fd)) == -1)
+ input = i_stream_create_error_str(errno, "dup() failed: %m");
+ else {
+ /* The stream could live even after the fs_file.
+ Don't use file->fd directly, because the fd may still be
+ used for other purposes. It's especially important for files
+ that were just created. */
+ input = i_stream_create_fd_autoclose(&fd_dup, max_buffer_size);
+ }
+ i_stream_set_name(input, file->full_path);
+ return input;
+}
+
+static void fs_posix_write_rename_if_needed(struct posix_fs_file *file)
+{
+ struct posix_fs *fs = container_of(file->file.fs, struct posix_fs, fs);
+ const char *new_fname;
+
+ new_fname = fs_metadata_find(&file->file.metadata, FS_METADATA_WRITE_FNAME);
+ if (new_fname == NULL)
+ return;
+
+ i_free(file->file.path);
+ file->file.path = i_strdup(new_fname);
+
+ i_free(file->full_path);
+ file->full_path = fs->path_prefix == NULL ? i_strdup(file->file.path) :
+ i_strconcat(fs->path_prefix, file->file.path, NULL);
+}
+
+static int fs_posix_write_finish_link(struct posix_fs_file *file)
+{
+ unsigned int try_count = 0;
+ int ret;
+
+ ret = link(file->temp_path, file->full_path);
+ while (ret < 0 && errno == ENOENT &&
+ try_count <= MAX_MKDIR_RETRY_COUNT) {
+ if (fs_posix_mkdir_parents(file, file->full_path) < 0)
+ return -1;
+ ret = link(file->temp_path, file->full_path);
+ try_count++;
+ }
+ if (ret < 0) {
+ fs_set_error_errno(file->file.event, "link(%s, %s) failed: %m",
+ file->temp_path, file->full_path);
+ }
+ return ret;
+}
+
+static int fs_posix_write_finish(struct posix_fs_file *file)
+{
+ struct posix_fs *fs = container_of(file->file.fs, struct posix_fs, fs);
+ unsigned int try_count = 0;
+ int ret, old_errno;
+
+ if ((file->open_flags & FS_OPEN_FLAG_FSYNC) != 0 &&
+ !fs->disable_fsync) {
+ if (fdatasync(file->fd) < 0) {
+ fs_set_error_errno(file->file.event,
+ "fdatasync(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ }
+ if (fs->accurate_mtime) {
+ /* Linux updates the mtime timestamp only on timer interrupts.
+ This isn't anywhere close to being microsecond precision.
+ If requested, use utimes() to explicitly set a more accurate
+ mtime. */
+ struct timeval tv[2];
+ i_gettimeofday(&tv[0]);
+ tv[1] = tv[0];
+ if ((utimes(file->temp_path, tv)) < 0) {
+ fs_set_error_errno(file->file.event,
+ "utimes(%s) failed: %m",
+ file->temp_path);
+ return -1;
+ }
+ }
+
+ fs_posix_write_rename_if_needed(file);
+ switch (file->open_mode) {
+ case FS_OPEN_MODE_CREATE_UNIQUE_128:
+ case FS_OPEN_MODE_CREATE:
+ ret = fs_posix_write_finish_link(file);
+ old_errno = errno;
+ if (unlink(file->temp_path) < 0) {
+ fs_set_error_errno(file->file.event,
+ "unlink(%s) failed: %m",
+ file->temp_path);
+ }
+ errno = old_errno;
+ if (ret < 0) {
+ fs_posix_file_close(&file->file);
+ i_free_and_null(file->temp_path);
+ return -1;
+ }
+ break;
+ case FS_OPEN_MODE_REPLACE:
+ ret = rename(file->temp_path, file->full_path);
+ while (ret < 0 && errno == ENOENT &&
+ try_count <= MAX_MKDIR_RETRY_COUNT) {
+ if (fs_posix_mkdir_parents(file, file->full_path) < 0)
+ return -1;
+ ret = rename(file->temp_path, file->full_path);
+ try_count++;
+ }
+ if (ret < 0) {
+ fs_set_error_errno(file->file.event,
+ "rename(%s, %s) failed: %m",
+ file->temp_path, file->full_path);
+ return -1;
+ }
+ break;
+ default:
+ i_unreached();
+ }
+ i_free_and_null(file->temp_path);
+ file->seek_to_beginning = TRUE;
+ /* allow opening the file after writing to it */
+ file->open_mode = FS_OPEN_MODE_READONLY;
+ return 0;
+}
+
+static int fs_posix_write(struct fs_file *_file, const void *data, size_t size)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+ ssize_t ret;
+
+ if (file->fd == -1) {
+ if (fs_posix_open(file) < 0)
+ return -1;
+ i_assert(file->fd != -1);
+ }
+
+ if (file->open_mode != FS_OPEN_MODE_APPEND) {
+ if (write_full(file->fd, data, size) < 0) {
+ fs_set_error_errno(_file->event, "write(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ return fs_posix_write_finish(file);
+ }
+
+ /* atomic append - it should either succeed or fail */
+ ret = write(file->fd, data, size);
+ if (ret < 0) {
+ fs_set_error_errno(_file->event, "write(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ if ((size_t)ret != size) {
+ fs_set_error(_file->event, ENOSPC,
+ "write(%s) returned %zu/%zu",
+ file->full_path, (size_t)ret, size);
+ return -1;
+ }
+ return 0;
+}
+
+static void fs_posix_write_stream(struct fs_file *_file)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+
+ i_assert(_file->output == NULL);
+
+ if (file->open_mode == FS_OPEN_MODE_APPEND) {
+ file->write_buf = buffer_create_dynamic(default_pool, 1024*32);
+ _file->output = o_stream_create_buffer(file->write_buf);
+ } else if (file->fd == -1 && fs_posix_open(file) < 0) {
+ _file->output = o_stream_create_error_str(errno, "%s",
+ fs_file_last_error(_file));
+ } else {
+ i_assert(file->fd != -1);
+ _file->output = o_stream_create_fd_file(file->fd,
+ UOFF_T_MAX, FALSE);
+ }
+ o_stream_set_name(_file->output, file->full_path);
+}
+
+static int fs_posix_write_stream_finish(struct fs_file *_file, bool success)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+ int ret = success ? 0 : -1;
+
+ o_stream_destroy(&_file->output);
+
+ switch (file->open_mode) {
+ case FS_OPEN_MODE_APPEND:
+ if (ret == 0) {
+ ret = fs_posix_write(_file, file->write_buf->data,
+ file->write_buf->used);
+ }
+ buffer_free(&file->write_buf);
+ break;
+ case FS_OPEN_MODE_CREATE:
+ case FS_OPEN_MODE_CREATE_UNIQUE_128:
+ case FS_OPEN_MODE_REPLACE:
+ if (ret == 0)
+ ret = fs_posix_write_finish(file);
+ break;
+ case FS_OPEN_MODE_READONLY:
+ i_unreached();
+ }
+ return ret < 0 ? -1 : 1;
+}
+
+static int
+fs_posix_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lock_r)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+ struct posix_fs *fs = container_of(_file->fs, struct posix_fs, fs);
+ struct dotlock_settings dotlock_set;
+ struct posix_fs_lock fs_lock, *ret_lock;
+ const char *error;
+ int ret = -1;
+
+ i_zero(&fs_lock);
+ fs_lock.lock.file = _file;
+
+ struct file_lock_settings lock_set = {
+ .lock_method = FILE_LOCK_METHOD_FLOCK,
+ };
+ switch (fs->lock_method) {
+ case FS_POSIX_LOCK_METHOD_FLOCK:
+#ifndef HAVE_FLOCK
+ fs_set_error(_file->event, ENOTSUP,
+ "flock() not supported by OS (for file %s)",
+ file->full_path);
+#else
+ if (secs == 0) {
+ ret = file_try_lock(file->fd, file->full_path, F_WRLCK,
+ &lock_set, &fs_lock.file_lock,
+ &error);
+ } else {
+ ret = file_wait_lock(file->fd, file->full_path, F_WRLCK,
+ &lock_set, secs,
+ &fs_lock.file_lock, &error);
+ }
+ if (ret < 0) {
+ fs_set_error_errno(_file->event, "flock(%s) failed: %s",
+ file->full_path, error);
+ }
+#endif
+ break;
+ case FS_POSIX_LOCK_METHOD_DOTLOCK:
+ i_zero(&dotlock_set);
+ dotlock_set.stale_timeout = FS_POSIX_DOTLOCK_STALE_TIMEOUT_SECS;
+ dotlock_set.use_excl_lock = TRUE;
+ dotlock_set.timeout = secs;
+
+ ret = file_dotlock_create(&dotlock_set, file->full_path,
+ secs == 0 ? 0 :
+ DOTLOCK_CREATE_FLAG_NONBLOCK,
+ &fs_lock.dotlock);
+ if (ret < 0) {
+ fs_set_error_errno(_file->event,
+ "file_dotlock_create(%s) failed: %m",
+ file->full_path);
+ }
+ break;
+ }
+ if (ret <= 0)
+ return ret;
+
+ ret_lock = i_new(struct posix_fs_lock, 1);
+ *ret_lock = fs_lock;
+ *lock_r = &ret_lock->lock;
+ return 1;
+}
+
+static void fs_posix_unlock(struct fs_lock *_lock)
+{
+ struct posix_fs_lock *lock =
+ container_of(_lock, struct posix_fs_lock, lock);
+
+ if (lock->file_lock != NULL)
+ file_unlock(&lock->file_lock);
+ if (lock->dotlock != NULL)
+ file_dotlock_delete(&lock->dotlock);
+ i_free(lock);
+}
+
+static int fs_posix_exists(struct fs_file *_file)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+ struct stat st;
+
+ if (stat(file->full_path, &st) < 0) {
+ if (errno != ENOENT) {
+ fs_set_error_errno(_file->event, "stat(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ return 0;
+ }
+ return 1;
+}
+
+static int fs_posix_stat(struct fs_file *_file, struct stat *st_r)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+
+ /* in case output != NULL it means that we're still writing to the file
+ and fs_stat() shouldn't stat the unfinished file. this is done by
+ fs-sis after fs_copy(). */
+ if (file->fd != -1 && _file->output == NULL) {
+ if (fstat(file->fd, st_r) < 0) {
+ fs_set_error_errno(_file->event, "fstat(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ } else {
+ if (stat(file->full_path, st_r) < 0) {
+ fs_set_error_errno(_file->event, "stat(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int fs_posix_copy(struct fs_file *_src, struct fs_file *_dest)
+{
+ struct posix_fs_file *src =
+ container_of(_src, struct posix_fs_file, file);
+ struct posix_fs_file *dest =
+ container_of(_dest, struct posix_fs_file, file);
+ unsigned int try_count = 0;
+ int ret;
+
+ fs_posix_write_rename_if_needed(dest);
+ ret = link(src->full_path, dest->full_path);
+ if (errno == EEXIST && dest->open_mode == FS_OPEN_MODE_REPLACE) {
+ /* destination file already exists - replace it */
+ i_unlink_if_exists(dest->full_path);
+ ret = link(src->full_path, dest->full_path);
+ }
+ while (ret < 0 && errno == ENOENT &&
+ try_count <= MAX_MKDIR_RETRY_COUNT) {
+ if (fs_posix_mkdir_parents(dest, dest->full_path) < 0)
+ return -1;
+ ret = link(src->full_path, dest->full_path);
+ try_count++;
+ }
+ if (ret < 0) {
+ fs_set_error_errno(_src->event, "link(%s, %s) failed: %m",
+ src->full_path, dest->full_path);
+ return -1;
+ }
+ return 0;
+}
+
+static int fs_posix_rename(struct fs_file *_src, struct fs_file *_dest)
+{
+ struct posix_fs_file *src =
+ container_of(_src, struct posix_fs_file, file);
+ struct posix_fs_file *dest =
+ container_of(_dest, struct posix_fs_file, file);
+ unsigned int try_count = 0;
+ int ret;
+
+ ret = rename(src->full_path, dest->full_path);
+ while (ret < 0 && errno == ENOENT &&
+ try_count <= MAX_MKDIR_RETRY_COUNT) {
+ if (fs_posix_mkdir_parents(dest, dest->full_path) < 0)
+ return -1;
+ ret = rename(src->full_path, dest->full_path);
+ try_count++;
+ }
+ if (ret < 0) {
+ fs_set_error_errno(_src->event, "rename(%s, %s) failed: %m",
+ src->full_path, dest->full_path);
+ return -1;
+ }
+ return 0;
+}
+
+static int fs_posix_delete(struct fs_file *_file)
+{
+ struct posix_fs_file *file =
+ container_of(_file, struct posix_fs_file, file);
+
+ if (unlink(file->full_path) < 0) {
+ if (!UNLINK_EISDIR(errno)) {
+ fs_set_error_errno(_file->event, "unlink(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ /* attempting to delete a directory. convert it to rmdir()
+ automatically. */
+ if (rmdir(file->full_path) < 0) {
+ fs_set_error_errno(_file->event, "rmdir(%s) failed: %m",
+ file->full_path);
+ return -1;
+ }
+ }
+ (void)fs_posix_rmdir_parents(file, file->full_path);
+ return 0;
+}
+
+static struct fs_iter *fs_posix_iter_alloc(void)
+{
+ struct posix_fs_iter *iter = i_new(struct posix_fs_iter, 1);
+ return &iter->iter;
+}
+
+static void
+fs_posix_iter_init(struct fs_iter *_iter, const char *path,
+ enum fs_iter_flags flags ATTR_UNUSED)
+{
+ struct posix_fs_iter *iter =
+ container_of(_iter, struct posix_fs_iter, iter);
+ struct posix_fs *fs = container_of(_iter->fs, struct posix_fs, fs);
+
+ iter->path = fs->path_prefix == NULL ? i_strdup(path) :
+ i_strconcat(fs->path_prefix, path, NULL);
+ if (iter->path[0] == '\0') {
+ i_free(iter->path);
+ iter->path = i_strdup(".");
+ }
+ iter->dir = opendir(iter->path);
+ if (iter->dir == NULL && errno != ENOENT) {
+ iter->err = errno;
+ fs_set_error_errno(_iter->event,
+ "opendir(%s) failed: %m", iter->path);
+ }
+}
+
+static bool fs_posix_iter_want(struct posix_fs_iter *iter, const char *fname)
+{
+ bool ret;
+
+ T_BEGIN {
+ const char *path = t_strdup_printf("%s/%s", iter->path, fname);
+ struct stat st;
+
+ if (stat(path, &st) < 0 &&
+ lstat(path, &st) < 0)
+ ret = FALSE;
+ else if (!S_ISDIR(st.st_mode))
+ ret = (iter->iter.flags & FS_ITER_FLAG_DIRS) == 0;
+ else
+ ret = (iter->iter.flags & FS_ITER_FLAG_DIRS) != 0;
+ } T_END;
+ return ret;
+}
+
+static const char *fs_posix_iter_next(struct fs_iter *_iter)
+{
+ struct posix_fs_iter *iter =
+ container_of(_iter, struct posix_fs_iter, iter);
+ struct posix_fs *fs = container_of(_iter->fs, struct posix_fs, fs);
+ struct dirent *d;
+
+ if (iter->dir == NULL)
+ return NULL;
+
+ errno = 0;
+ for (; (d = readdir(iter->dir)) != NULL; errno = 0) {
+ if (strcmp(d->d_name, ".") == 0 ||
+ strcmp(d->d_name, "..") == 0)
+ continue;
+ if (strncmp(d->d_name, fs->temp_file_prefix,
+ fs->temp_file_prefix_len) == 0)
+ continue;
+#ifdef HAVE_DIRENT_D_TYPE
+ switch (d->d_type) {
+ case DT_UNKNOWN:
+ if (fs_posix_iter_want(iter, d->d_name))
+ return d->d_name;
+ break;
+ case DT_DIR:
+ if ((iter->iter.flags & FS_ITER_FLAG_DIRS) != 0)
+ return d->d_name;
+ break;
+ default:
+ if ((iter->iter.flags & FS_ITER_FLAG_DIRS) == 0)
+ return d->d_name;
+ break;
+ }
+#else
+ if (fs_posix_iter_want(iter, d->d_name))
+ return d->d_name;
+#endif
+ }
+ if (errno != 0) {
+ iter->err = errno;
+ fs_set_error_errno(_iter->event,
+ "readdir(%s) failed: %m", iter->path);
+ }
+ return NULL;
+}
+
+static int fs_posix_iter_deinit(struct fs_iter *_iter)
+{
+ struct posix_fs_iter *iter =
+ container_of(_iter, struct posix_fs_iter, iter);
+ int ret = 0;
+
+ if (iter->dir != NULL && closedir(iter->dir) < 0 && iter->err == 0) {
+ iter->err = errno;
+ fs_set_error_errno(_iter->event,
+ "closedir(%s) failed: %m", iter->path);
+ }
+ if (iter->err != 0) {
+ errno = iter->err;
+ ret = -1;
+ }
+ i_free(iter->path);
+ return ret;
+}
+
+const struct fs fs_class_posix = {
+ .name = "posix",
+ .v = {
+ fs_posix_alloc,
+ fs_posix_init,
+ NULL,
+ fs_posix_free,
+ fs_posix_get_properties,
+ fs_posix_file_alloc,
+ fs_posix_file_init,
+ fs_posix_file_deinit,
+ fs_posix_file_close,
+ NULL,
+ NULL, NULL,
+ fs_default_set_metadata,
+ NULL,
+ fs_posix_prefetch,
+ fs_posix_read,
+ fs_posix_read_stream,
+ fs_posix_write,
+ fs_posix_write_stream,
+ fs_posix_write_stream_finish,
+ fs_posix_lock,
+ fs_posix_unlock,
+ fs_posix_exists,
+ fs_posix_stat,
+ fs_posix_copy,
+ fs_posix_rename,
+ fs_posix_delete,
+ fs_posix_iter_alloc,
+ fs_posix_iter_init,
+ fs_posix_iter_next,
+ fs_posix_iter_deinit,
+ NULL,
+ NULL,
+ }
+};
diff --git a/src/lib-fs/fs-randomfail.c b/src/lib-fs/fs-randomfail.c
new file mode 100644
index 0000000..48b1bf0
--- /dev/null
+++ b/src/lib-fs/fs-randomfail.c
@@ -0,0 +1,555 @@
+/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "istream-private.h"
+#include "istream-concat.h"
+#include "istream-failure-at.h"
+#include "ostream-failure-at.h"
+#include "ostream.h"
+#include "fs-api-private.h"
+
+
+#define RANDOMFAIL_ERROR "Random failure injection"
+
+static const char *fs_op_names[FS_OP_COUNT] = {
+ "wait", "metadata", "prefetch", "read", "write", "lock", "exists",
+ "stat", "copy", "rename", "delete", "iter"
+};
+
+struct randomfail_fs {
+ struct fs fs;
+ unsigned int op_probability[FS_OP_COUNT];
+ uoff_t range_start[FS_OP_COUNT], range_end[FS_OP_COUNT];
+};
+
+struct randomfail_fs_file {
+ struct fs_file file;
+ struct fs_file *super_read;
+ struct istream *input;
+ bool op_pending[FS_OP_COUNT];
+
+ struct ostream *super_output;
+};
+
+struct randomfail_fs_iter {
+ struct fs_iter iter;
+ struct fs_iter *super;
+ unsigned int fail_pos;
+};
+
+#define RANDOMFAIL_FS(ptr) container_of((ptr), struct randomfail_fs, fs)
+#define RANDOMFAIL_FILE(ptr) container_of((ptr), struct randomfail_fs_file, file)
+#define RANDOMFAIL_ITER(ptr) container_of((ptr), struct randomfail_fs_iter, iter)
+
+static struct fs *fs_randomfail_alloc(void)
+{
+ struct randomfail_fs *fs;
+
+ fs = i_new(struct randomfail_fs, 1);
+ fs->fs = fs_class_randomfail;
+ return &fs->fs;
+}
+
+static bool fs_op_find(const char *str, enum fs_op *op_r)
+{
+ enum fs_op op;
+
+ for (op = 0; op < FS_OP_COUNT; op++) {
+ if (strcmp(fs_op_names[op], str) == 0) {
+ *op_r = op;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static int
+fs_randomfail_add_probability(struct randomfail_fs *fs,
+ const char *key, const char *value,
+ const char **error_r)
+{
+ unsigned int num;
+ enum fs_op op;
+ bool invalid_value = FALSE;
+
+ if (str_to_uint(value, &num) < 0 || num > 100)
+ invalid_value = TRUE;
+ if (fs_op_find(key, &op)) {
+ if (invalid_value) {
+ *error_r = "Invalid probability value";
+ return -1;
+ }
+ fs->op_probability[op] = num;
+ return 1;
+ }
+ if (strcmp(key, "all") == 0) {
+ if (invalid_value) {
+ *error_r = "Invalid probability value";
+ return -1;
+ }
+ for (op = 0; op < FS_OP_COUNT; op++)
+ fs->op_probability[op] = num;
+ return 1;
+ }
+ return 0;
+}
+
+static int
+fs_randomfail_add_probability_range(struct randomfail_fs *fs,
+ const char *key, const char *value,
+ const char **error_r)
+{
+ enum fs_op op;
+ const char *p;
+ uoff_t num1, num2;
+
+ if (strcmp(key, "read-range") == 0)
+ op = FS_OP_READ;
+ else if (strcmp(key, "write-range") == 0)
+ op = FS_OP_WRITE;
+ else if (strcmp(key, "iter-range") == 0)
+ op = FS_OP_ITER;
+ else
+ return 0;
+
+ p = strchr(value, '-');
+ if (p == NULL) {
+ if (str_to_uoff(value, &num1) < 0) {
+ *error_r = "Invalid range value";
+ return -1;
+ }
+ num2 = num1;
+ } else if (str_to_uoff(t_strdup_until(value, p), &num1) < 0 ||
+ str_to_uoff(p+1, &num2) < 0 || num1 > num2) {
+ *error_r = "Invalid range values";
+ return -1;
+ }
+ fs->range_start[op] = num1;
+ fs->range_end[op] = num2;
+ return 1;
+}
+
+static int fs_randomfail_parse_params(struct randomfail_fs *fs,
+ const char *params, const char **error_r)
+{
+ const char *const *tmp;
+ int ret;
+
+ for (tmp = t_strsplit_spaces(params, ","); *tmp != NULL; tmp++) {
+ const char *key = *tmp;
+ const char *value = strchr(key, '=');
+
+ if (value == NULL) {
+ *error_r = "Missing '='";
+ return -1;
+ }
+ key = t_strdup_until(key, value++);
+ if ((ret = fs_randomfail_add_probability(fs, key, value, error_r)) != 0) {
+ if (ret < 0)
+ return -1;
+ continue;
+ }
+ if ((ret = fs_randomfail_add_probability_range(fs, key, value, error_r)) != 0) {
+ if (ret < 0)
+ return -1;
+ continue;
+ }
+ *error_r = t_strdup_printf("Unknown key '%s'", key);
+ return -1;
+ }
+ return 0;
+}
+
+static int
+fs_randomfail_init(struct fs *_fs, const char *args,
+ const struct fs_settings *set, const char **error_r)
+{
+ struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
+ const char *p, *parent_name, *parent_args, *error;
+
+ p = strchr(args, ':');
+ if (p == NULL) {
+ *error_r = "Randomfail parameters missing";
+ return -1;
+ }
+ if (fs_randomfail_parse_params(fs, t_strdup_until(args, p++), &error) < 0) {
+ *error_r = t_strdup_printf(
+ "Invalid randomfail parameters: %s", error);
+ return -1;
+ }
+ args = p;
+
+ if (*args == '\0') {
+ *error_r = "Parent filesystem not given as parameter";
+ return -1;
+ }
+
+ parent_args = strchr(args, ':');
+ if (parent_args == NULL) {
+ parent_name = args;
+ parent_args = "";
+ } else {
+ parent_name = t_strdup_until(args, parent_args);
+ parent_args++;
+ }
+ if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0)
+ return -1;
+ return 0;
+}
+
+static void fs_randomfail_free(struct fs *_fs)
+{
+ struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
+
+ i_free(fs);
+}
+
+static enum fs_properties fs_randomfail_get_properties(struct fs *_fs)
+{
+ return fs_get_properties(_fs->parent);
+}
+
+static struct fs_file *fs_randomfail_file_alloc(void)
+{
+ struct randomfail_fs_file *file = i_new(struct randomfail_fs_file, 1);
+ return &file->file;
+}
+
+static void
+fs_randomfail_file_init(struct fs_file *_file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+
+ file->file.path = i_strdup(path);
+ file->file.parent = fs_file_init_parent(_file, path, mode, flags);
+}
+
+static void fs_randomfail_file_deinit(struct fs_file *_file)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+
+ fs_file_free(_file);
+ i_free(file->file.path);
+ i_free(file);
+}
+
+static bool fs_random_fail(struct fs *_fs, struct event *event,
+ int divider, enum fs_op op)
+{
+ struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
+
+ if (fs->op_probability[op] == 0)
+ return FALSE;
+ if ((unsigned int)i_rand_limit(100 * divider) <= fs->op_probability[op]) {
+ fs_set_error(event, EIO, RANDOMFAIL_ERROR);
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static bool
+fs_file_random_fail_begin(struct randomfail_fs_file *file, enum fs_op op)
+{
+ if (!file->op_pending[op]) {
+ if (fs_random_fail(file->file.fs, file->file.event, 2, op))
+ return TRUE;
+ }
+ file->op_pending[op] = TRUE;
+ return FALSE;
+}
+
+static int
+fs_file_random_fail_end(struct randomfail_fs_file *file,
+ int ret, enum fs_op op)
+{
+ if (ret == 0 || errno != EAGAIN) {
+ if (fs_random_fail(file->file.fs, file->file.event, 2, op))
+ return -1;
+ file->op_pending[op] = FALSE;
+ }
+ return ret;
+}
+
+static bool
+fs_random_fail_range(struct fs *_fs, struct event *event,
+ enum fs_op op, uoff_t *offset_r)
+{
+ struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
+
+ if (!fs_random_fail(_fs, event, 1, op))
+ return FALSE;
+ *offset_r = i_rand_minmax(fs->range_start[op], fs->range_end[op]);
+ return TRUE;
+}
+
+static int
+fs_randomfail_get_metadata(struct fs_file *_file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ int ret;
+
+ if (fs_file_random_fail_begin(file, FS_OP_METADATA))
+ return -1;
+ ret = fs_get_metadata_full(_file->parent, flags, metadata_r);
+ return fs_file_random_fail_end(file, ret, FS_OP_METADATA);
+}
+
+static bool fs_randomfail_prefetch(struct fs_file *_file, uoff_t length)
+{
+ if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_PREFETCH))
+ return TRUE;
+ return fs_prefetch(_file->parent, length);
+}
+
+static ssize_t fs_randomfail_read(struct fs_file *_file, void *buf, size_t size)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ int ret;
+
+ if (fs_file_random_fail_begin(file, FS_OP_READ))
+ return -1;
+ ret = fs_read(_file->parent, buf, size);
+ if (fs_file_random_fail_end(file, ret < 0 ? -1 : 0, FS_OP_READ) < 0)
+ return -1;
+ return ret;
+}
+
+static struct istream *
+fs_randomfail_read_stream(struct fs_file *_file, size_t max_buffer_size)
+{
+ struct istream *input, *input2;
+ uoff_t offset;
+
+ input = fs_read_stream(_file->parent, max_buffer_size);
+ if (!fs_random_fail_range(_file->fs, _file->event, FS_OP_READ, &offset))
+ return input;
+ input2 = i_stream_create_failure_at(input, offset, EIO, RANDOMFAIL_ERROR);
+ i_stream_unref(&input);
+ return input2;
+}
+
+static int fs_randomfail_write(struct fs_file *_file, const void *data, size_t size)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ int ret;
+
+ if (fs_file_random_fail_begin(file, FS_OP_WRITE))
+ return -1;
+ ret = fs_write(_file->parent, data, size);
+ return fs_file_random_fail_end(file, ret, FS_OP_EXISTS);
+}
+
+static void fs_randomfail_write_stream(struct fs_file *_file)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ uoff_t offset;
+
+ i_assert(_file->output == NULL);
+
+ file->super_output = fs_write_stream(_file->parent);
+ if (!fs_random_fail_range(_file->fs, _file->event, FS_OP_WRITE, &offset))
+ _file->output = file->super_output;
+ else {
+ _file->output = o_stream_create_failure_at(file->super_output, offset,
+ RANDOMFAIL_ERROR);
+ }
+}
+
+static int fs_randomfail_write_stream_finish(struct fs_file *_file, bool success)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+
+ if (_file->output != NULL) {
+ if (_file->output == file->super_output)
+ _file->output = NULL;
+ else
+ o_stream_unref(&_file->output);
+ if (!success) {
+ fs_write_stream_abort_parent(_file, &file->super_output);
+ return -1;
+ }
+ if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_WRITE)) {
+ fs_write_stream_abort_error(_file->parent, &file->super_output, RANDOMFAIL_ERROR);
+ return -1;
+ }
+ }
+ return fs_write_stream_finish(_file->parent, &file->super_output);
+}
+
+static int
+fs_randomfail_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lock_r)
+{
+ if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_LOCK))
+ return -1;
+ return fs_lock(_file->parent, secs, lock_r);
+}
+
+static void fs_randomfail_unlock(struct fs_lock *_lock ATTR_UNUSED)
+{
+ i_unreached();
+}
+
+static int fs_randomfail_exists(struct fs_file *_file)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ int ret;
+
+ if (fs_file_random_fail_begin(file, FS_OP_EXISTS))
+ return -1;
+ ret = fs_exists(_file->parent);
+ return fs_file_random_fail_end(file, ret, FS_OP_EXISTS);
+}
+
+static int fs_randomfail_stat(struct fs_file *_file, struct stat *st_r)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ int ret;
+
+ if (fs_file_random_fail_begin(file, FS_OP_STAT))
+ return -1;
+ ret = fs_stat(_file->parent, st_r);
+ return fs_file_random_fail_end(file, ret, FS_OP_STAT);
+}
+
+static int fs_randomfail_get_nlinks(struct fs_file *_file, nlink_t *nlinks_r)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ int ret;
+
+ if (fs_file_random_fail_begin(file, FS_OP_STAT))
+ return -1;
+ ret = fs_get_nlinks(_file->parent, nlinks_r);
+ return fs_file_random_fail_end(file, ret, FS_OP_STAT);
+}
+
+static int fs_randomfail_copy(struct fs_file *_src, struct fs_file *_dest)
+{
+ struct randomfail_fs_file *dest = RANDOMFAIL_FILE(_dest);
+ int ret;
+
+ if (fs_file_random_fail_begin(dest, FS_OP_COPY))
+ return -1;
+
+ if (_src != NULL)
+ ret = fs_copy(_src->parent, _dest->parent);
+ else
+ ret = fs_copy_finish_async(_dest->parent);
+ return fs_file_random_fail_end(dest, ret, FS_OP_COPY);
+}
+
+static int fs_randomfail_rename(struct fs_file *_src, struct fs_file *_dest)
+{
+ struct randomfail_fs_file *dest = RANDOMFAIL_FILE(_dest);
+ int ret;
+
+ if (fs_file_random_fail_begin(dest, FS_OP_RENAME))
+ return -1;
+ ret = fs_rename(_src->parent, _dest->parent);
+ return fs_file_random_fail_end(dest, ret, FS_OP_RENAME);
+}
+
+static int fs_randomfail_delete(struct fs_file *_file)
+{
+ struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
+ int ret;
+
+ if (fs_file_random_fail_begin(file, FS_OP_DELETE))
+ return -1;
+ ret = fs_delete(_file->parent);
+ return fs_file_random_fail_end(file, ret, FS_OP_DELETE);
+}
+
+static struct fs_iter *fs_randomfail_iter_alloc(void)
+{
+ struct randomfail_fs_iter *iter = i_new(struct randomfail_fs_iter, 1);
+ return &iter->iter;
+}
+
+static void
+fs_randomfail_iter_init(struct fs_iter *_iter, const char *path,
+ enum fs_iter_flags flags)
+{
+ struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter);
+ uoff_t pos;
+
+ iter->super = fs_iter_init_parent(_iter, path, flags);
+ if (fs_random_fail_range(_iter->fs, _iter->event, FS_OP_ITER, &pos))
+ iter->fail_pos = pos + 1;
+}
+
+static const char *fs_randomfail_iter_next(struct fs_iter *_iter)
+{
+ struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter);
+ const char *fname;
+
+ if (iter->fail_pos > 0) {
+ if (iter->fail_pos == 1)
+ return NULL;
+ iter->fail_pos--;
+ }
+
+ iter->super->async_callback = _iter->async_callback;
+ iter->super->async_context = _iter->async_context;
+
+ fname = fs_iter_next(iter->super);
+ _iter->async_have_more = iter->super->async_have_more;
+ return fname;
+}
+
+static int fs_randomfail_iter_deinit(struct fs_iter *_iter)
+{
+ struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter);
+ const char *error;
+ int ret;
+
+ if ((ret = fs_iter_deinit(&iter->super, &error)) < 0)
+ fs_set_error_errno(_iter->event, "%s", error);
+ if (iter->fail_pos == 1) {
+ fs_set_error(_iter->event, EIO, RANDOMFAIL_ERROR);
+ ret = -1;
+ }
+ return ret;
+}
+
+const struct fs fs_class_randomfail = {
+ .name = "randomfail",
+ .v = {
+ fs_randomfail_alloc,
+ fs_randomfail_init,
+ NULL,
+ fs_randomfail_free,
+ fs_randomfail_get_properties,
+ fs_randomfail_file_alloc,
+ fs_randomfail_file_init,
+ fs_randomfail_file_deinit,
+ fs_wrapper_file_close,
+ fs_wrapper_file_get_path,
+ fs_wrapper_set_async_callback,
+ fs_wrapper_wait_async,
+ fs_wrapper_set_metadata,
+ fs_randomfail_get_metadata,
+ fs_randomfail_prefetch,
+ fs_randomfail_read,
+ fs_randomfail_read_stream,
+ fs_randomfail_write,
+ fs_randomfail_write_stream,
+ fs_randomfail_write_stream_finish,
+ fs_randomfail_lock,
+ fs_randomfail_unlock,
+ fs_randomfail_exists,
+ fs_randomfail_stat,
+ fs_randomfail_copy,
+ fs_randomfail_rename,
+ fs_randomfail_delete,
+ fs_randomfail_iter_alloc,
+ fs_randomfail_iter_init,
+ fs_randomfail_iter_next,
+ fs_randomfail_iter_deinit,
+ NULL,
+ fs_randomfail_get_nlinks,
+ }
+};
diff --git a/src/lib-fs/fs-sis-common.c b/src/lib-fs/fs-sis-common.c
new file mode 100644
index 0000000..654c44e
--- /dev/null
+++ b/src/lib-fs/fs-sis-common.c
@@ -0,0 +1,59 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "fs-sis-common.h"
+
+#include <sys/stat.h>
+
+int fs_sis_path_parse(struct fs_file *file, const char *path,
+ const char **dir_r, const char **hash_r)
+{
+ const char *fname, *p;
+
+ fname = strrchr(path, '/');
+ if (fname == NULL) {
+ *dir_r = ".";
+ fname = path;
+ } else {
+ *dir_r = t_strdup_until(path, fname);
+ fname++;
+ }
+
+ /* assume filename begins with "<hash>-" */
+ p = strchr(fname, '-');
+ if (p == NULL) {
+ fs_set_error(file->event, EINVAL, "open(%s) failed: "
+ "Filenames must begin with '<hash>-'", path);
+ return -1;
+ }
+ *hash_r = t_strdup_until(fname, p);
+ return 0;
+}
+
+void fs_sis_try_unlink_hash_file(struct fs_file *sis_file,
+ struct fs_file *super_file)
+{
+ struct fs_file *hash_file;
+ struct stat st1, st2;
+ const char *dir, *hash, *hash_path;
+
+ if (fs_sis_path_parse(sis_file, super_file->path, &dir, &hash) == 0 &&
+ fs_stat(super_file, &st1) == 0 && st1.st_nlink == 2) {
+ /* this may be the last link. if hashes/ file is the same,
+ delete it. */
+ hash_path = t_strdup_printf("%s/"HASH_DIR_NAME"/%s", dir, hash);
+ hash_file = fs_file_init_with_event(super_file->fs,
+ super_file->event, hash_path,
+ FS_OPEN_MODE_READONLY);
+ if (fs_stat(hash_file, &st2) == 0 &&
+ st1.st_ino == st2.st_ino &&
+ CMP_DEV_T(st1.st_dev, st2.st_dev)) {
+ if (fs_delete(hash_file) < 0) {
+ e_error(hash_file->event, "%s",
+ fs_file_last_error(hash_file));
+ }
+ }
+ fs_file_deinit(&hash_file);
+ }
+}
+
diff --git a/src/lib-fs/fs-sis-common.h b/src/lib-fs/fs-sis-common.h
new file mode 100644
index 0000000..5156ca6
--- /dev/null
+++ b/src/lib-fs/fs-sis-common.h
@@ -0,0 +1,14 @@
+#ifndef FS_SIS_COMMON_H
+#define FS_SIS_COMMON_H
+
+#include "fs-api-private.h"
+
+#define HASH_DIR_NAME "hashes"
+
+int fs_sis_path_parse(struct fs_file *file, const char *path,
+ const char **dir_r, const char **hash_r);
+void fs_sis_try_unlink_hash_file(struct fs_file *sis_file,
+ struct fs_file *super_file);
+
+#endif
+
diff --git a/src/lib-fs/fs-sis-queue.c b/src/lib-fs/fs-sis-queue.c
new file mode 100644
index 0000000..8ad7d98
--- /dev/null
+++ b/src/lib-fs/fs-sis-queue.c
@@ -0,0 +1,210 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "ostream.h"
+#include "fs-sis-common.h"
+
+#define QUEUE_DIR_NAME "queue"
+
+struct sis_queue_fs {
+ struct fs fs;
+ char *queue_dir;
+};
+
+struct sis_queue_fs_file {
+ struct fs_file file;
+ struct sis_queue_fs *fs;
+};
+
+#define SISQUEUE_FS(ptr) container_of((ptr), struct sis_queue_fs, fs)
+#define SISQUEUE_FILE(ptr) container_of((ptr), struct sis_queue_fs_file, file)
+
+static struct fs *fs_sis_queue_alloc(void)
+{
+ struct sis_queue_fs *fs;
+
+ fs = i_new(struct sis_queue_fs, 1);
+ fs->fs = fs_class_sis_queue;
+ return &fs->fs;
+}
+
+static int
+fs_sis_queue_init(struct fs *_fs, const char *args,
+ const struct fs_settings *set, const char **error_r)
+{
+ struct sis_queue_fs *fs = SISQUEUE_FS(_fs);
+ const char *p, *parent_name, *parent_args;
+
+ /* <queue_dir>:<parent fs>[:<args>] */
+
+ p = strchr(args, ':');
+ if (p == NULL || p[1] == '\0') {
+ *error_r = "Parent filesystem not given as parameter";
+ return -1;
+ }
+
+ fs->queue_dir = i_strdup_until(args, p);
+ parent_name = p + 1;
+
+ parent_args = strchr(parent_name, ':');
+ if (parent_args == NULL)
+ parent_args = "";
+ else
+ parent_name = t_strdup_until(parent_name, parent_args++);
+ if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0)
+ return -1;
+ return 0;
+}
+
+static void fs_sis_queue_free(struct fs *_fs)
+{
+ struct sis_queue_fs *fs = SISQUEUE_FS(_fs);
+
+ i_free(fs->queue_dir);
+ i_free(fs);
+}
+
+static struct fs_file *fs_sis_queue_file_alloc(void)
+{
+ struct sis_queue_fs_file *file = i_new(struct sis_queue_fs_file, 1);
+ return &file->file;
+}
+
+static void
+fs_sis_queue_file_init(struct fs_file *_file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags)
+{
+ struct sis_queue_fs_file *file = SISQUEUE_FILE(_file);
+ struct sis_queue_fs *fs = SISQUEUE_FS(_file->fs);
+
+ file->file.path = i_strdup(path);
+ file->fs = fs;
+
+ if (mode == FS_OPEN_MODE_APPEND)
+ fs_set_error(_file->event, ENOTSUP, "APPEND mode not supported");
+ else
+ file->file.parent = fs_file_init_parent(_file, path, mode, flags);
+}
+
+static void fs_sis_queue_file_deinit(struct fs_file *_file)
+{
+ struct sis_queue_fs_file *file = SISQUEUE_FILE(_file);
+
+ fs_file_free(_file);
+ i_free(file->file.path);
+ i_free(file);
+}
+
+static void fs_sis_queue_add(struct sis_queue_fs_file *file)
+{
+ struct sis_queue_fs *fs = SISQUEUE_FS(file->file.fs);
+ struct fs_file *queue_file;
+ const char *fname, *path, *queue_path;
+
+ path = fs_file_path(&file->file);
+ fname = strrchr(path, '/');
+ if (fname != NULL)
+ fname++;
+ else
+ fname = path;
+
+ queue_path = t_strdup_printf("%s/%s", fs->queue_dir, fname);
+ queue_file = fs_file_init_parent(&file->file, queue_path, FS_OPEN_MODE_CREATE, 0);
+ if (fs_write(queue_file, "", 0) < 0 && errno != EEXIST)
+ e_error(file->file.event, "%s", fs_file_last_error(queue_file));
+ fs_file_deinit(&queue_file);
+}
+
+static int fs_sis_queue_write(struct fs_file *_file, const void *data, size_t size)
+{
+ struct sis_queue_fs_file *file = SISQUEUE_FILE(_file);
+
+ if (_file->parent == NULL)
+ return -1;
+ if (fs_write(_file->parent, data, size) < 0)
+ return -1;
+ T_BEGIN {
+ fs_sis_queue_add(file);
+ } T_END;
+ return 0;
+}
+
+static void fs_sis_queue_write_stream(struct fs_file *_file)
+{
+ i_assert(_file->output == NULL);
+
+ if (_file->parent == NULL) {
+ _file->output = o_stream_create_error_str(EINVAL, "%s",
+ fs_file_last_error(_file));
+ } else {
+ _file->output = fs_write_stream(_file->parent);
+ }
+ o_stream_set_name(_file->output, _file->path);
+}
+
+static int fs_sis_queue_write_stream_finish(struct fs_file *_file, bool success)
+{
+ struct sis_queue_fs_file *file = SISQUEUE_FILE(_file);
+
+ if (!success) {
+ if (_file->parent != NULL)
+ fs_write_stream_abort_parent(_file, &_file->output);
+ return -1;
+ }
+
+ if (fs_write_stream_finish(_file->parent, &_file->output) < 0)
+ return -1;
+ T_BEGIN {
+ fs_sis_queue_add(file);
+ } T_END;
+ return 1;
+}
+
+static int fs_sis_queue_delete(struct fs_file *_file)
+{
+ T_BEGIN {
+ fs_sis_try_unlink_hash_file(_file, _file->parent);
+ } T_END;
+ return fs_delete(_file->parent);
+}
+
+const struct fs fs_class_sis_queue = {
+ .name = "sis-queue",
+ .v = {
+ fs_sis_queue_alloc,
+ fs_sis_queue_init,
+ NULL,
+ fs_sis_queue_free,
+ fs_wrapper_get_properties,
+ fs_sis_queue_file_alloc,
+ fs_sis_queue_file_init,
+ fs_sis_queue_file_deinit,
+ fs_wrapper_file_close,
+ fs_wrapper_file_get_path,
+ fs_wrapper_set_async_callback,
+ fs_wrapper_wait_async,
+ fs_wrapper_set_metadata,
+ fs_wrapper_get_metadata,
+ fs_wrapper_prefetch,
+ fs_wrapper_read,
+ fs_wrapper_read_stream,
+ fs_sis_queue_write,
+ fs_sis_queue_write_stream,
+ fs_sis_queue_write_stream_finish,
+ fs_wrapper_lock,
+ fs_wrapper_unlock,
+ fs_wrapper_exists,
+ fs_wrapper_stat,
+ fs_wrapper_copy,
+ fs_wrapper_rename,
+ fs_sis_queue_delete,
+ fs_wrapper_iter_alloc,
+ fs_wrapper_iter_init,
+ NULL,
+ NULL,
+ NULL,
+ fs_wrapper_get_nlinks,
+ }
+};
diff --git a/src/lib-fs/fs-sis.c b/src/lib-fs/fs-sis.c
new file mode 100644
index 0000000..6a020bc
--- /dev/null
+++ b/src/lib-fs/fs-sis.c
@@ -0,0 +1,369 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "ostream.h"
+#include "ostream-cmp.h"
+#include "fs-sis-common.h"
+
+#define FS_SIS_REQUIRED_PROPS \
+ (FS_PROPERTY_FASTCOPY | FS_PROPERTY_STAT)
+
+struct sis_fs {
+ struct fs fs;
+};
+
+struct sis_fs_file {
+ struct fs_file file;
+ struct sis_fs *fs;
+ enum fs_open_mode open_mode;
+
+ struct fs_file *hash_file;
+ struct istream *hash_input;
+ struct ostream *fs_output;
+
+ char *hash, *hash_path;
+ bool opened;
+};
+
+#define SIS_FS(ptr) container_of((ptr), struct sis_fs, fs)
+#define SIS_FILE(ptr) container_of((ptr), struct sis_fs_file, file)
+
+static struct fs *fs_sis_alloc(void)
+{
+ struct sis_fs *fs;
+
+ fs = i_new(struct sis_fs, 1);
+ fs->fs = fs_class_sis;
+ return &fs->fs;
+}
+
+static int
+fs_sis_init(struct fs *_fs, const char *args, const struct fs_settings *set,
+ const char **error_r)
+{
+ enum fs_properties props;
+ const char *parent_name, *parent_args;
+
+ if (*args == '\0') {
+ *error_r = "Parent filesystem not given as parameter";
+ return -1;
+ }
+
+ parent_args = strchr(args, ':');
+ if (parent_args == NULL) {
+ parent_name = args;
+ parent_args = "";
+ } else {
+ parent_name = t_strdup_until(args, parent_args);
+ parent_args++;
+ }
+ if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0)
+ return -1;
+ props = fs_get_properties(_fs->parent);
+ if ((props & FS_SIS_REQUIRED_PROPS) != FS_SIS_REQUIRED_PROPS) {
+ *error_r = t_strdup_printf("%s backend can't be used with SIS",
+ parent_name);
+ return -1;
+ }
+ return 0;
+}
+
+static void fs_sis_free(struct fs *_fs)
+{
+ struct sis_fs *fs = SIS_FS(_fs);
+
+ i_free(fs);
+}
+
+static struct fs_file *fs_sis_file_alloc(void)
+{
+ struct sis_fs_file *file = i_new(struct sis_fs_file, 1);
+ return &file->file;
+}
+
+static void
+fs_sis_file_init(struct fs_file *_file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags)
+{
+ struct sis_fs_file *file = SIS_FILE(_file);
+ struct sis_fs *fs = SIS_FS(_file->fs);
+ const char *dir, *hash;
+
+ file->file.path = i_strdup(path);
+ file->fs = fs;
+ file->open_mode = mode;
+ if (mode == FS_OPEN_MODE_APPEND) {
+ fs_set_error(_file->event, ENOTSUP, "APPEND mode not supported");
+ return;
+ }
+
+ if (fs_sis_path_parse(_file, path, &dir, &hash) < 0)
+ return;
+
+ /* if hashes/<hash> already exists, open it */
+ file->hash_path = i_strdup_printf("%s/"HASH_DIR_NAME"/%s", dir, hash);
+ file->hash_file = fs_file_init_parent(_file, file->hash_path,
+ FS_OPEN_MODE_READONLY, 0);
+
+ file->hash_input = fs_read_stream(file->hash_file, IO_BLOCK_SIZE);
+ if (i_stream_read(file->hash_input) == -1) {
+ /* doesn't exist */
+ if (errno != ENOENT) {
+ e_error(file->file.event, "Couldn't read hash file %s: %m",
+ file->hash_path);
+ }
+ i_stream_destroy(&file->hash_input);
+ }
+
+ file->file.parent = fs_file_init_parent(_file, path, mode, flags);
+}
+
+static void fs_sis_file_deinit(struct fs_file *_file)
+{
+ struct sis_fs_file *file = SIS_FILE(_file);
+
+ fs_file_deinit(&file->hash_file);
+ fs_file_free(_file);
+ i_free(file->hash);
+ i_free(file->hash_path);
+ i_free(file->file.path);
+ i_free(file);
+}
+
+static void fs_sis_file_close(struct fs_file *_file)
+{
+ struct sis_fs_file *file = SIS_FILE(_file);
+
+ i_stream_unref(&file->hash_input);
+ fs_file_close(file->hash_file);
+ fs_file_close(_file->parent);
+}
+
+static bool fs_sis_try_link(struct sis_fs_file *file)
+{
+ const struct stat *st;
+ struct stat st2;
+
+ if (i_stream_stat(file->hash_input, FALSE, &st) < 0)
+ return FALSE;
+
+ /* we can use the existing file */
+ if (fs_copy(file->hash_file, file->file.parent) < 0) {
+ if (errno != ENOENT && errno != EMLINK) {
+ e_error(file->file.event, "%s",
+ fs_file_last_error(file->hash_file));
+ }
+ /* failed to use link(), continue as if it hadn't been equal */
+ return FALSE;
+ }
+ if (fs_stat(file->file.parent, &st2) < 0) {
+ e_error(file->file.event, "%s",
+ fs_file_last_error(file->file.parent));
+ if (fs_delete(file->file.parent) < 0) {
+ e_error(file->file.event, "%s",
+ fs_file_last_error(file->file.parent));
+ }
+ return FALSE;
+ }
+ if (st->st_ino != st2.st_ino) {
+ /* the hashes/ file was already replaced with something else */
+ if (fs_delete(file->file.parent) < 0) {
+ e_error(file->file.event, "%s",
+ fs_file_last_error(file->file.parent));
+ }
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static void fs_sis_replace_hash_file(struct sis_fs_file *file)
+{
+ struct fs *super_fs = file->file.parent->fs;
+ struct fs_file *temp_file;
+ const char *hash_fname;
+ string_t *temp_path;
+ int ret;
+
+ if (file->hash_input == NULL) {
+ /* hash file didn't exist previously. we should be able to
+ create it with link() */
+ if (fs_copy(file->file.parent, file->hash_file) < 0) {
+ if (errno == EEXIST) {
+ /* the file was just created. it's probably
+ a duplicate, but it's too much trouble
+ trying to deduplicate it anymore */
+ } else {
+ e_error(file->file.event, "%s",
+ fs_file_last_error(file->hash_file));
+ }
+ }
+ return;
+ }
+
+ temp_path = t_str_new(256);
+ hash_fname = strrchr(file->hash_path, '/');
+ if (hash_fname == NULL)
+ hash_fname = file->hash_path;
+ else {
+ str_append_data(temp_path, file->hash_path,
+ (hash_fname-file->hash_path) + 1);
+ hash_fname++;
+ }
+ str_printfa(temp_path, "%s%s.tmp",
+ super_fs->set.temp_file_prefix, hash_fname);
+
+ /* replace existing hash file atomically */
+ temp_file = fs_file_init_parent(&file->file, str_c(temp_path),
+ FS_OPEN_MODE_READONLY, 0);
+ ret = fs_copy(file->file.parent, temp_file);
+ if (ret < 0 && errno == EEXIST) {
+ /* either someone's racing us or it's a stale file.
+ try to continue. */
+ if (fs_delete(temp_file) < 0 &&
+ errno != ENOENT)
+ e_error(file->file.event, "%s", fs_file_last_error(temp_file));
+ ret = fs_copy(file->file.parent, temp_file);
+ }
+ if (ret < 0) {
+ e_error(file->file.event, "%s", fs_file_last_error(temp_file));
+ fs_file_deinit(&temp_file);
+ return;
+ }
+
+ if (fs_rename(temp_file, file->hash_file) < 0) {
+ if (errno == ENOENT) {
+ /* apparently someone else just renamed it. ignore. */
+ } else {
+ e_error(file->file.event, "%s",
+ fs_file_last_error(file->hash_file));
+ }
+ (void)fs_delete(temp_file);
+ }
+ fs_file_deinit(&temp_file);
+}
+
+static int fs_sis_write(struct fs_file *_file, const void *data, size_t size)
+{
+ struct sis_fs_file *file = SIS_FILE(_file);
+
+ if (_file->parent == NULL)
+ return -1;
+
+ if (file->hash_input != NULL &&
+ stream_cmp_block(file->hash_input, data, size) &&
+ i_stream_read_eof(file->hash_input)) {
+ /* try to use existing file */
+ if (fs_sis_try_link(file))
+ return 0;
+ }
+
+ if (fs_write(_file->parent, data, size) < 0)
+ return -1;
+ T_BEGIN {
+ fs_sis_replace_hash_file(file);
+ } T_END;
+ return 0;
+}
+
+static void fs_sis_write_stream(struct fs_file *_file)
+{
+ struct sis_fs_file *file = SIS_FILE(_file);
+
+ i_assert(_file->output == NULL);
+
+ if (_file->parent == NULL) {
+ _file->output = o_stream_create_error_str(EINVAL, "%s",
+ fs_file_last_error(_file));
+ } else {
+ file->fs_output = fs_write_stream(_file->parent);
+ if (file->hash_input == NULL) {
+ _file->output = file->fs_output;
+ o_stream_ref(_file->output);
+ } else {
+ /* compare if files are equal */
+ _file->output = o_stream_create_cmp(file->fs_output,
+ file->hash_input);
+ }
+ }
+ o_stream_set_name(_file->output, _file->path);
+}
+
+static int fs_sis_write_stream_finish(struct fs_file *_file, bool success)
+{
+ struct sis_fs_file *file = SIS_FILE(_file);
+
+ if (!success) {
+ if (_file->parent != NULL)
+ fs_write_stream_abort_parent(_file, &file->fs_output);
+ o_stream_unref(&_file->output);
+ return -1;
+ }
+
+ if (file->hash_input != NULL &&
+ o_stream_cmp_equals(_file->output) &&
+ i_stream_read_eof(file->hash_input)) {
+ o_stream_unref(&_file->output);
+ if (fs_sis_try_link(file)) {
+ fs_write_stream_abort_parent(_file, &file->fs_output);
+ return 1;
+ }
+ }
+ if (_file->output != NULL)
+ o_stream_unref(&_file->output);
+
+ if (fs_write_stream_finish(_file->parent, &file->fs_output) < 0)
+ return -1;
+ T_BEGIN {
+ fs_sis_replace_hash_file(file);
+ } T_END;
+ return 1;
+}
+
+static int fs_sis_delete(struct fs_file *_file)
+{
+ T_BEGIN {
+ fs_sis_try_unlink_hash_file(_file, _file->parent);
+ } T_END;
+ return fs_delete(_file->parent);
+}
+
+const struct fs fs_class_sis = {
+ .name = "sis",
+ .v = {
+ fs_sis_alloc,
+ fs_sis_init,
+ NULL,
+ fs_sis_free,
+ fs_wrapper_get_properties,
+ fs_sis_file_alloc,
+ fs_sis_file_init,
+ fs_sis_file_deinit,
+ fs_sis_file_close,
+ fs_wrapper_file_get_path,
+ fs_wrapper_set_async_callback,
+ fs_wrapper_wait_async,
+ fs_wrapper_set_metadata,
+ fs_wrapper_get_metadata,
+ fs_wrapper_prefetch,
+ fs_wrapper_read,
+ fs_wrapper_read_stream,
+ fs_sis_write,
+ fs_sis_write_stream,
+ fs_sis_write_stream_finish,
+ fs_wrapper_lock,
+ fs_wrapper_unlock,
+ fs_wrapper_exists,
+ fs_wrapper_stat,
+ fs_wrapper_copy,
+ fs_wrapper_rename,
+ fs_sis_delete,
+ fs_wrapper_iter_alloc,
+ fs_wrapper_iter_init,
+ NULL,
+ NULL,
+ NULL,
+ fs_wrapper_get_nlinks,
+ }
+};
diff --git a/src/lib-fs/fs-test-async.c b/src/lib-fs/fs-test-async.c
new file mode 100644
index 0000000..a8cd3f5
--- /dev/null
+++ b/src/lib-fs/fs-test-async.c
@@ -0,0 +1,103 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "ostream.h"
+#include "fs-test.h"
+#include "test-common.h"
+
+static void test_fs_async_write(const char *test_name, struct fs *fs)
+{
+ struct fs_file *file;
+ struct test_fs_file *test_file;
+ struct ostream *output;
+ unsigned int i;
+ int ret;
+
+ test_begin(t_strdup_printf("%s: async write", test_name));
+ for (i = 0; i < 3; i++) {
+ file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE |
+ FS_OPEN_FLAG_ASYNC);
+ output = fs_write_stream(file);
+
+ o_stream_nsend_str(output, "12345");
+ if (i < 2) {
+ test_assert(fs_write_stream_finish(file, &output) == 0);
+ test_assert(output == NULL);
+ test_assert(fs_write_stream_finish_async(file) == 0);
+ }
+
+ test_file = test_fs_file_get(fs, "foo");
+ test_file->wait_async = FALSE;
+
+ switch (i) {
+ case 0:
+ while ((ret = fs_write_stream_finish_async(file)) == 0)
+ fs_wait_async(fs);
+ test_assert(ret > 0);
+ test_assert(test_file->contents->used > 0);
+ break;
+ case 1:
+ test_file->io_failure = TRUE;
+ test_assert(fs_write_stream_finish_async(file) < 0);
+ test_assert(test_file->contents->used == 0);
+ break;
+ case 2:
+ fs_write_stream_abort_error(file, &output, "test");
+ test_assert(test_file->contents->used == 0);
+ break;
+ }
+ fs_file_deinit(&file);
+ }
+ test_end();
+}
+
+static void test_fs_async_copy(const char *test_name, struct fs *fs)
+{
+ struct fs_file *src, *dest;
+ struct test_fs_file *test_file;
+ int ret;
+
+ test_begin(t_strdup_printf("%s: async copy", test_name));
+
+ src = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE);
+ test_assert(fs_write(src, "source", 6) == 0);
+
+ dest = fs_file_init(fs, "bar", FS_OPEN_MODE_REPLACE |
+ FS_OPEN_FLAG_ASYNC);
+
+ test_assert(fs_copy(src, dest) == -1 && errno == EAGAIN);
+
+ test_file = test_fs_file_get(fs, "bar");
+ test_file->wait_async = FALSE;
+
+ while ((ret = fs_copy_finish_async(dest)) < 0 && errno == EAGAIN)
+ fs_wait_async(fs);
+ test_assert(ret == 0);
+ test_assert(test_file->contents->used > 0);
+ fs_file_deinit(&dest);
+
+ fs_file_deinit(&src);
+ test_end();
+}
+
+void test_fs_async(const char *test_name, enum fs_properties properties,
+ const char *driver, const char *args)
+{
+ struct fs_settings fs_set;
+ struct fs *fs;
+ struct test_fs *test_fs;
+ const char *error;
+
+ i_zero(&fs_set);
+ if (fs_init(driver, args, &fs_set, &fs, &error) < 0)
+ i_fatal("fs_init() failed: %s", error);
+
+ test_fs = test_fs_get(fs);
+ test_fs->properties = properties;
+
+ test_fs_async_write(test_name, fs);
+ test_fs_async_copy(test_name, fs);
+
+ fs_deinit(&fs);
+}
diff --git a/src/lib-fs/fs-test.c b/src/lib-fs/fs-test.c
new file mode 100644
index 0000000..d352718
--- /dev/null
+++ b/src/lib-fs/fs-test.c
@@ -0,0 +1,443 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "ostream.h"
+#include "test-common.h"
+#include "fs-test.h"
+
+static struct fs *fs_test_alloc(void)
+{
+ struct test_fs *fs;
+
+ fs = i_new(struct test_fs, 1);
+ fs->fs = fs_class_test;
+ i_array_init(&fs->iter_files, 32);
+ return &fs->fs;
+}
+
+static int
+fs_test_init(struct fs *_fs ATTR_UNUSED, const char *args ATTR_UNUSED,
+ const struct fs_settings *set ATTR_UNUSED,
+ const char **error_r ATTR_UNUSED)
+{
+ return 0;
+}
+
+static void fs_test_free(struct fs *_fs)
+{
+ struct test_fs *fs = (struct test_fs *)_fs;
+
+ array_free(&fs->iter_files);
+ i_free(fs);
+}
+
+static enum fs_properties fs_test_get_properties(struct fs *_fs)
+{
+ struct test_fs *fs = (struct test_fs *)_fs;
+
+ return fs->properties;
+}
+
+static struct fs_file *fs_test_file_alloc(void)
+{
+ struct test_fs_file *file = i_new(struct test_fs_file, 1);
+ return &file->file;
+}
+
+static void
+fs_test_file_init(struct fs_file *_file, const char *path,
+ enum fs_open_mode mode, enum fs_open_flags flags)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ file->file.path = i_strdup(path);
+ file->file.flags = flags;
+ file->mode = mode;
+ file->contents = buffer_create_dynamic(default_pool, 1024);
+ file->exists = TRUE;
+ file->seekable = TRUE;
+ file->wait_async = (flags & FS_OPEN_FLAG_ASYNC) != 0;
+}
+
+static void fs_test_file_deinit(struct fs_file *_file)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ fs_file_free(_file);
+ buffer_free(&file->contents);
+ i_free(file->file.path);
+ i_free(file);
+}
+
+static void fs_test_file_close(struct fs_file *_file)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ file->closed = TRUE;
+}
+
+static const char *fs_test_file_get_path(struct fs_file *_file)
+{
+ return _file->path;
+}
+
+static void
+fs_test_set_async_callback(struct fs_file *_file,
+ fs_file_async_callback_t *callback,
+ void *context)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ file->async_callback = callback;
+ file->async_context = context;
+}
+
+static void fs_test_wait_async(struct fs *_fs ATTR_UNUSED)
+{
+}
+
+static void
+fs_test_set_metadata(struct fs_file *_file, const char *key,
+ const char *value)
+{
+ if (strcmp(key, FS_METADATA_WRITE_FNAME) == 0) {
+ i_free(_file->path);
+ _file->path = i_strdup(value);
+ } else {
+ fs_default_set_metadata(_file, key, value);
+ }
+}
+
+static int
+fs_test_get_metadata(struct fs_file *_file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ if ((flags & FS_GET_METADATA_FLAG_LOADED_ONLY) != 0) {
+ *metadata_r = &_file->metadata;
+ return 0;
+ }
+
+ if (file->wait_async) {
+ fs_file_set_error_async(_file);
+ return -1;
+ }
+ if (file->io_failure) {
+ errno = EIO;
+ return -1;
+ }
+ fs_metadata_init(_file);
+ *metadata_r = &_file->metadata;
+ return 0;
+}
+
+static bool fs_test_prefetch(struct fs_file *_file ATTR_UNUSED,
+ uoff_t length ATTR_UNUSED)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ file->prefetched = TRUE;
+ return TRUE;
+}
+
+static void fs_test_stream_destroyed(struct test_fs_file *file)
+{
+ i_assert(file->input != NULL);
+ file->input = NULL;
+}
+
+static struct istream *
+fs_test_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+ struct istream *input;
+
+ i_assert(file->input == NULL);
+
+ if (!file->exists)
+ return i_stream_create_error(ENOENT);
+ if (file->io_failure)
+ return i_stream_create_error(EIO);
+ input = test_istream_create_data(file->contents->data,
+ file->contents->used);
+ i_stream_add_destroy_callback(input, fs_test_stream_destroyed, file);
+ if (!file->seekable)
+ input->seekable = FALSE;
+ file->input = input;
+ return input;
+}
+
+static void fs_test_write_stream(struct fs_file *_file)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ i_assert(_file->output == NULL);
+
+ buffer_set_used_size(file->contents, 0);
+ _file->output = o_stream_create_buffer(file->contents);
+}
+
+static int fs_test_write_stream_finish(struct fs_file *_file, bool success)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ o_stream_destroy(&_file->output);
+ if (file->wait_async) {
+ fs_file_set_error_async(_file);
+ return 0;
+ }
+ if (file->io_failure)
+ success = FALSE;
+ if (!success)
+ buffer_set_used_size(file->contents, 0);
+ return success ? 1 : -1;
+}
+
+static int
+fs_test_lock(struct fs_file *_file, unsigned int secs ATTR_UNUSED,
+ struct fs_lock **lock_r)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ if (file->locked)
+ return 0;
+ file->locked = TRUE;
+ *lock_r = i_new(struct fs_lock, 1);
+ (*lock_r)->file = _file;
+ return 1;
+}
+
+static void fs_test_unlock(struct fs_lock *lock)
+{
+ struct test_fs_file *file = (struct test_fs_file *)lock->file;
+
+ file->locked = FALSE;
+ i_free(lock);
+}
+
+static int fs_test_exists(struct fs_file *_file)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ if (file->wait_async) {
+ fs_file_set_error_async(_file);
+ return -1;
+ }
+ if (file->io_failure) {
+ errno = EIO;
+ return -1;
+ }
+ return file->exists ? 1 : 0;
+}
+
+static int fs_test_stat(struct fs_file *_file, struct stat *st_r)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ if (file->wait_async) {
+ fs_file_set_error_async(_file);
+ return -1;
+ }
+ if (file->io_failure) {
+ errno = EIO;
+ return -1;
+ }
+ if (!file->exists) {
+ errno = ENOENT;
+ return -1;
+ }
+ i_zero(st_r);
+ st_r->st_size = file->contents->used;
+ return 0;
+}
+
+static int fs_test_copy(struct fs_file *_src, struct fs_file *_dest)
+{
+ struct test_fs_file *src;
+ struct test_fs_file *dest = (struct test_fs_file *)_dest;
+
+ if (_src != NULL)
+ dest->copy_src = test_fs_file_get(_src->fs, fs_file_path(_src));
+ src = dest->copy_src;
+ if (dest->wait_async) {
+ fs_file_set_error_async(_dest);
+ return -1;
+ }
+ dest->copy_src = NULL;
+
+ if (dest->io_failure) {
+ errno = EIO;
+ return -1;
+ }
+ if (!src->exists) {
+ errno = ENOENT;
+ return -1;
+ }
+ buffer_set_used_size(dest->contents, 0);
+ buffer_append_buf(dest->contents, src->contents, 0, SIZE_MAX);
+ dest->exists = TRUE;
+ return 0;
+}
+
+static int fs_test_rename(struct fs_file *_src, struct fs_file *_dest)
+{
+ struct test_fs_file *src = (struct test_fs_file *)_src;
+ struct test_fs_file *dest = (struct test_fs_file *)_dest;
+
+ if (src->wait_async || dest->wait_async) {
+ fs_file_set_error_async(_dest);
+ return -1;
+ }
+
+ if (fs_test_copy(_src, _dest) < 0)
+ return -1;
+ src->exists = FALSE;
+ return 0;
+}
+
+static int fs_test_delete(struct fs_file *_file)
+{
+ struct test_fs_file *file = (struct test_fs_file *)_file;
+
+ if (file->wait_async) {
+ fs_file_set_error_async(_file);
+ return -1;
+ }
+
+ if (!file->exists) {
+ errno = ENOENT;
+ return -1;
+ }
+ return 0;
+}
+
+static struct fs_iter *fs_test_iter_alloc(void)
+{
+ struct test_fs_iter *iter = i_new(struct test_fs_iter, 1);
+ return &iter->iter;
+}
+
+static void
+fs_test_iter_init(struct fs_iter *_iter, const char *path,
+ enum fs_iter_flags flags ATTR_UNUSED)
+{
+ struct test_fs_iter *iter = (struct test_fs_iter *)_iter;
+ struct test_fs *fs = (struct test_fs *)_iter->fs;
+
+ iter->prefix = i_strdup(path);
+ iter->prefix_len = strlen(iter->prefix);
+ iter->prev_dir = i_strdup("");
+ array_sort(&fs->iter_files, i_strcmp_p);
+}
+
+static const char *fs_test_iter_next(struct fs_iter *_iter)
+{
+ struct test_fs_iter *iter = (struct test_fs_iter *)_iter;
+ struct test_fs *fs = (struct test_fs *)_iter->fs;
+ const char *const *files, *p;
+ unsigned int count;
+ size_t len, prev_dir_len = strlen(iter->prev_dir);
+
+ files = array_get(&fs->iter_files, &count);
+ for (; iter->idx < count; iter->idx++) {
+ const char *fname = files[iter->idx];
+
+ if (strncmp(fname, iter->prefix, iter->prefix_len) != 0)
+ continue;
+ p = strrchr(fname, '/');
+ if ((_iter->flags & FS_ITER_FLAG_DIRS) == 0) {
+ if (p == NULL)
+ return fname;
+ if (p[1] == '\0')
+ continue; /* dir/ */
+ return p+1;
+ }
+
+ if (p == NULL)
+ continue;
+ len = p - fname;
+ if (len == 0)
+ continue;
+ if (len == prev_dir_len &&
+ strncmp(fname, iter->prev_dir, len) == 0)
+ continue;
+ i_free(iter->prev_dir);
+ iter->prev_dir = i_strndup(fname, len);
+ return iter->prev_dir;
+ }
+ return NULL;
+}
+
+static int fs_test_iter_deinit(struct fs_iter *_iter)
+{
+ struct test_fs_iter *iter = (struct test_fs_iter *)_iter;
+ int ret = iter->failed ? -1 : 0;
+
+ i_free(iter->prefix);
+ return ret;
+}
+
+struct test_fs *test_fs_get(struct fs *fs)
+{
+ while (strcmp(fs->name, "test") != 0) {
+ i_assert(fs->parent != NULL);
+ fs = fs->parent;
+ }
+ return (struct test_fs *)fs;
+}
+
+struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path)
+{
+ struct fs_file *file;
+
+ fs = &test_fs_get(fs)->fs;
+
+ for (file = fs->files;; file = file->next) {
+ i_assert(file != NULL);
+ if (strcmp(fs_file_path(file), path) == 0)
+ break;
+ }
+ return (struct test_fs_file *)file;
+}
+
+const struct fs fs_class_test = {
+ .name = "test",
+ .v = {
+ fs_test_alloc,
+ fs_test_init,
+ NULL,
+ fs_test_free,
+ fs_test_get_properties,
+ fs_test_file_alloc,
+ fs_test_file_init,
+ fs_test_file_deinit,
+ fs_test_file_close,
+ fs_test_file_get_path,
+ fs_test_set_async_callback,
+ fs_test_wait_async,
+ fs_test_set_metadata,
+ fs_test_get_metadata,
+ fs_test_prefetch,
+ NULL,
+ fs_test_read_stream,
+ NULL,
+ fs_test_write_stream,
+ fs_test_write_stream_finish,
+ fs_test_lock,
+ fs_test_unlock,
+ fs_test_exists,
+ fs_test_stat,
+ fs_test_copy,
+ fs_test_rename,
+ fs_test_delete,
+ fs_test_iter_alloc,
+ fs_test_iter_init,
+ fs_test_iter_next,
+ fs_test_iter_deinit,
+ NULL,
+ NULL,
+ }
+};
diff --git a/src/lib-fs/fs-test.h b/src/lib-fs/fs-test.h
new file mode 100644
index 0000000..5a2ed14
--- /dev/null
+++ b/src/lib-fs/fs-test.h
@@ -0,0 +1,45 @@
+#ifndef FS_TEST_H
+#define FS_TEST_H
+
+#include "fs-api-private.h"
+
+struct test_fs {
+ struct fs fs;
+ enum fs_properties properties;
+ ARRAY_TYPE(const_string) iter_files;
+};
+
+struct test_fs_file {
+ struct fs_file file;
+ enum fs_open_mode mode;
+
+ fs_file_async_callback_t *async_callback;
+ void *async_context;
+
+ buffer_t *contents;
+ struct istream *input;
+ struct test_fs_file *copy_src;
+
+ bool prefetched;
+ bool locked;
+ bool exists;
+ bool seekable;
+ bool closed;
+ bool io_failure;
+ bool wait_async;
+};
+
+struct test_fs_iter {
+ struct fs_iter iter;
+ char *prefix, *prev_dir;
+ unsigned int prefix_len, idx;
+ bool failed;
+};
+
+struct test_fs *test_fs_get(struct fs *fs);
+struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path);
+
+void test_fs_async(const char *test_name, enum fs_properties properties,
+ const char *driver, const char *args);
+
+#endif
diff --git a/src/lib-fs/fs-wrapper.c b/src/lib-fs/fs-wrapper.c
new file mode 100644
index 0000000..81d746e
--- /dev/null
+++ b/src/lib-fs/fs-wrapper.c
@@ -0,0 +1,172 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "fs-api-private.h"
+#include "ostream.h"
+
+struct wrapper_fs_iter {
+ struct fs_iter iter;
+ struct fs_iter *parent;
+};
+
+enum fs_properties fs_wrapper_get_properties(struct fs *fs)
+{
+ return fs_get_properties(fs->parent);
+}
+
+void fs_wrapper_file_close(struct fs_file *file)
+{
+ fs_file_close(file->parent);
+}
+
+const char *fs_wrapper_file_get_path(struct fs_file *file)
+{
+ return fs_file_path(file->parent);
+}
+
+void fs_wrapper_set_async_callback(struct fs_file *file,
+ fs_file_async_callback_t *callback,
+ void *context)
+{
+ fs_file_set_async_callback(file->parent, *callback, context);
+}
+
+void fs_wrapper_wait_async(struct fs *fs)
+{
+ fs_wait_async(fs->parent);
+}
+
+void fs_wrapper_set_metadata(struct fs_file *file, const char *key,
+ const char *value)
+{
+ fs_set_metadata(file->parent, key, value);
+}
+
+int fs_wrapper_get_metadata(struct fs_file *file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r)
+{
+ return fs_get_metadata_full(file->parent, flags, metadata_r);
+}
+
+bool fs_wrapper_prefetch(struct fs_file *file, uoff_t length)
+{
+ return fs_prefetch(file->parent, length);
+}
+
+ssize_t fs_wrapper_read(struct fs_file *file, void *buf, size_t size)
+{
+ return fs_read(file->parent, buf, size);
+}
+
+struct istream *
+fs_wrapper_read_stream(struct fs_file *file, size_t max_buffer_size)
+{
+ return fs_read_stream(file->parent, max_buffer_size);
+}
+
+int fs_wrapper_write(struct fs_file *file, const void *data, size_t size)
+{
+ return fs_write(file->parent, data, size);
+}
+
+void fs_wrapper_write_stream(struct fs_file *file)
+{
+ i_assert(file->output == NULL);
+
+ file->output = fs_write_stream(file->parent);
+}
+
+int fs_wrapper_write_stream_finish(struct fs_file *file, bool success)
+{
+ if (file->output == NULL)
+ return fs_write_stream_finish_async(file->parent);
+
+ if (!success) {
+ fs_write_stream_abort_parent(file, &file->output);
+ return -1;
+ }
+ return fs_write_stream_finish(file->parent, &file->output);
+}
+
+int fs_wrapper_lock(struct fs_file *file, unsigned int secs,
+ struct fs_lock **lock_r)
+{
+ return fs_lock(file->parent, secs, lock_r);
+}
+
+void fs_wrapper_unlock(struct fs_lock *_lock ATTR_UNUSED)
+{
+ i_unreached();
+}
+
+int fs_wrapper_exists(struct fs_file *file)
+{
+ return fs_exists(file->parent);
+}
+
+int fs_wrapper_stat(struct fs_file *file, struct stat *st_r)
+{
+ return fs_stat(file->parent, st_r);
+}
+
+int fs_wrapper_get_nlinks(struct fs_file *file, nlink_t *nlinks_r)
+{
+ return fs_get_nlinks(file->parent, nlinks_r);
+}
+
+int fs_wrapper_copy(struct fs_file *src, struct fs_file *dest)
+{
+ if (src != NULL)
+ return fs_copy(src->parent, dest->parent);
+ else
+ return fs_copy_finish_async(dest->parent);
+}
+
+int fs_wrapper_rename(struct fs_file *src, struct fs_file *dest)
+{
+ return fs_rename(src->parent, dest->parent);
+}
+
+int fs_wrapper_delete(struct fs_file *file)
+{
+ return fs_delete(file->parent);
+}
+
+struct fs_iter *fs_wrapper_iter_alloc(void)
+{
+ struct wrapper_fs_iter *iter = i_new(struct wrapper_fs_iter, 1);
+ return &iter->iter;
+}
+
+void fs_wrapper_iter_init(struct fs_iter *_iter, const char *path,
+ enum fs_iter_flags flags)
+{
+ struct wrapper_fs_iter *iter = (struct wrapper_fs_iter *)_iter;
+
+ iter->parent = fs_iter_init_parent(_iter, path, flags);
+}
+
+const char *fs_wrapper_iter_next(struct fs_iter *_iter)
+{
+ struct wrapper_fs_iter *iter = (struct wrapper_fs_iter *)_iter;
+ const char *fname;
+
+ iter->parent->async_callback = _iter->async_callback;
+ iter->parent->async_context = _iter->async_context;
+
+ fname = fs_iter_next(iter->parent);
+ _iter->async_have_more = iter->parent->async_have_more;
+ return fname;
+}
+
+int fs_wrapper_iter_deinit(struct fs_iter *_iter)
+{
+ struct wrapper_fs_iter *iter = (struct wrapper_fs_iter *)_iter;
+ const char *error;
+ int ret;
+
+ if ((ret = fs_iter_deinit(&iter->parent, &error)) < 0)
+ fs_set_error_errno(_iter->event, "%s", error);
+ return ret;
+}
diff --git a/src/lib-fs/fs-wrapper.h b/src/lib-fs/fs-wrapper.h
new file mode 100644
index 0000000..ebfd35d
--- /dev/null
+++ b/src/lib-fs/fs-wrapper.h
@@ -0,0 +1,40 @@
+#ifndef FS_WRAPPER_H
+#define FS_WRAPPER_H
+
+enum fs_get_metadata_flags;
+
+enum fs_properties fs_wrapper_get_properties(struct fs *fs);
+void fs_wrapper_file_close(struct fs_file *file);
+const char *fs_wrapper_file_get_path(struct fs_file *file);
+void fs_wrapper_set_async_callback(struct fs_file *file,
+ fs_file_async_callback_t *callback,
+ void *context);
+void fs_wrapper_wait_async(struct fs *fs);
+void fs_wrapper_set_metadata(struct fs_file *file, const char *key,
+ const char *value);
+int fs_wrapper_get_metadata(struct fs_file *file,
+ enum fs_get_metadata_flags flags,
+ const ARRAY_TYPE(fs_metadata) **metadata_r);
+bool fs_wrapper_prefetch(struct fs_file *file, uoff_t length);
+ssize_t fs_wrapper_read(struct fs_file *file, void *buf, size_t size);
+struct istream *
+fs_wrapper_read_stream(struct fs_file *file, size_t max_buffer_size);
+int fs_wrapper_write(struct fs_file *file, const void *data, size_t size);
+void fs_wrapper_write_stream(struct fs_file *file);
+int fs_wrapper_write_stream_finish(struct fs_file *file, bool success);
+int fs_wrapper_lock(struct fs_file *file, unsigned int secs,
+ struct fs_lock **lock_r);
+void fs_wrapper_unlock(struct fs_lock *_lock);
+int fs_wrapper_exists(struct fs_file *file);
+int fs_wrapper_stat(struct fs_file *file, struct stat *st_r);
+int fs_wrapper_get_nlinks(struct fs_file *file, nlink_t *nlinks_r);
+int fs_wrapper_copy(struct fs_file *src, struct fs_file *dest);
+int fs_wrapper_rename(struct fs_file *src, struct fs_file *dest);
+int fs_wrapper_delete(struct fs_file *file);
+struct fs_iter *fs_wrapper_iter_alloc(void);
+void fs_wrapper_iter_init(struct fs_iter *iter, const char *path,
+ enum fs_iter_flags flags);
+const char *fs_wrapper_iter_next(struct fs_iter *iter);
+int fs_wrapper_iter_deinit(struct fs_iter *iter);
+
+#endif
diff --git a/src/lib-fs/istream-fs-file.c b/src/lib-fs/istream-fs-file.c
new file mode 100644
index 0000000..4f61cfe
--- /dev/null
+++ b/src/lib-fs/istream-fs-file.c
@@ -0,0 +1,61 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream-private.h"
+#include "fs-api-private.h"
+#include "istream-fs-file.h"
+
+struct fs_file_istream {
+ struct istream_private istream;
+ struct fs_file *file;
+};
+
+static void i_stream_fs_file_close(struct iostream_private *stream,
+ bool close_parent ATTR_UNUSED)
+{
+ struct fs_file_istream *fstream = (struct fs_file_istream *)stream;
+
+ i_stream_destroy(&fstream->istream.parent);
+ fs_file_deinit(&fstream->file);
+}
+
+static ssize_t i_stream_fs_file_read(struct istream_private *stream)
+{
+ struct fs_file_istream *fstream = (struct fs_file_istream *)stream;
+ struct istream *input;
+
+ if (fstream->istream.parent == NULL) {
+ input = fs_read_stream(fstream->file,
+ i_stream_get_max_buffer_size(&stream->istream));
+ i_stream_init_parent(stream, input);
+ i_stream_unref(&input);
+ }
+
+ i_stream_seek(stream->parent, stream->parent_start_offset +
+ stream->istream.v_offset);
+ return i_stream_read_copy_from_parent(&stream->istream);
+}
+
+struct istream *
+i_stream_create_fs_file(struct fs_file **file, size_t max_buffer_size)
+{
+ struct fs_file_istream *fstream;
+ struct istream *input;
+
+ fstream = i_new(struct fs_file_istream, 1);
+ fstream->file = *file;
+ fstream->istream.iostream.close = i_stream_fs_file_close;
+ fstream->istream.max_buffer_size = max_buffer_size;
+ fstream->istream.read = i_stream_fs_file_read;
+ fstream->istream.stream_size_passthrough = TRUE;
+
+ fstream->istream.istream.blocking =
+ ((*file)->flags & FS_OPEN_FLAG_ASYNC) == 0;
+ fstream->istream.istream.seekable =
+ ((*file)->flags & FS_OPEN_FLAG_SEEKABLE) != 0;
+
+ input = i_stream_create(&fstream->istream, NULL, -1, 0);
+ i_stream_set_name(input, fs_file_path(*file));
+ *file = NULL;
+ return input;
+}
diff --git a/src/lib-fs/istream-fs-file.h b/src/lib-fs/istream-fs-file.h
new file mode 100644
index 0000000..2821c0e
--- /dev/null
+++ b/src/lib-fs/istream-fs-file.h
@@ -0,0 +1,13 @@
+#ifndef ISTREAM_FS_FILE_H
+#define ISTREAM_FS_FILE_H
+
+struct fs_file;
+
+/* Open the given file only when something is actually tried to be read from
+ the stream. The file is automatically deinitialized when the stream is
+ destroyed (which is why it's also set to NULL so it's not accidentally
+ double-freed). */
+struct istream *
+i_stream_create_fs_file(struct fs_file **file, size_t max_buffer_size);
+
+#endif
diff --git a/src/lib-fs/istream-fs-stats.c b/src/lib-fs/istream-fs-stats.c
new file mode 100644
index 0000000..3f77430
--- /dev/null
+++ b/src/lib-fs/istream-fs-stats.c
@@ -0,0 +1,47 @@
+/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "fs-api-private.h"
+#include "istream-private.h"
+#include "istream-fs-stats.h"
+
+struct fs_stats_istream {
+ struct istream_private istream;
+ struct fs_file *file;
+};
+
+static ssize_t
+i_stream_fs_stats_read(struct istream_private *stream)
+{
+ struct fs_stats_istream *sstream = (struct fs_stats_istream *)stream;
+ ssize_t ret;
+
+ i_stream_seek(stream->parent, stream->parent_start_offset +
+ stream->istream.v_offset);
+
+ ret = i_stream_read_copy_from_parent(&stream->istream);
+ if (ret > 0) {
+ /* count the first returned bytes as the finish time, since
+ we don't want to count the time caller spends on processing
+ this stream. (only the first fs_file_timing_end() call
+ actually does anything - the others are ignored.) */
+ fs_file_timing_end(sstream->file, FS_OP_READ);
+ }
+ return ret;
+}
+
+struct istream *
+i_stream_create_fs_stats(struct istream *input, struct fs_file *file)
+{
+ struct fs_stats_istream *sstream;
+
+ sstream = i_new(struct fs_stats_istream, 1);
+ sstream->file = file;
+ sstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+ sstream->istream.stream_size_passthrough = TRUE;
+ sstream->istream.read = i_stream_fs_stats_read;
+ sstream->istream.istream.blocking = input->blocking;
+ sstream->istream.istream.seekable = input->seekable;
+ return i_stream_create(&sstream->istream, input,
+ i_stream_get_fd(input), 0);
+}
diff --git a/src/lib-fs/istream-fs-stats.h b/src/lib-fs/istream-fs-stats.h
new file mode 100644
index 0000000..aca74b6
--- /dev/null
+++ b/src/lib-fs/istream-fs-stats.h
@@ -0,0 +1,9 @@
+#ifndef ISTREAM_FS_STATS_H
+#define ISTREAM_FS_STATS_H
+
+struct fs_file;
+
+struct istream *
+i_stream_create_fs_stats(struct istream *input, struct fs_file *file);
+
+#endif
diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c
new file mode 100644
index 0000000..ac2e8fb
--- /dev/null
+++ b/src/lib-fs/istream-metawrap.c
@@ -0,0 +1,152 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream-private.h"
+#include "istream-metawrap.h"
+
+#define METAWRAP_MAX_METADATA_LINE_LEN 8192
+
+struct metawrap_istream {
+ struct istream_private istream;
+ metawrap_callback_t *callback;
+ void *context;
+
+ uoff_t start_offset, pending_seek;
+ bool in_metadata;
+};
+
+static int metadata_header_read(struct metawrap_istream *mstream)
+{
+ char *line, *p;
+
+ while ((line = i_stream_read_next_line(mstream->istream.parent)) != NULL) {
+ if (*line == '\0') {
+ mstream->callback(NULL, NULL, mstream->context);
+ return 1;
+ }
+ p = strchr(line, ':');
+ if (p == NULL) {
+ io_stream_set_error(&mstream->istream.iostream,
+ "Metadata header line is missing ':' at offset %"PRIuUOFF_T,
+ mstream->istream.istream.v_offset);
+ mstream->istream.istream.stream_errno = EINVAL;
+ return -1;
+ }
+ *p++ = '\0';
+ mstream->callback(line, p, mstream->context);
+ }
+ if (mstream->istream.parent->eof) {
+ if (mstream->istream.parent->stream_errno != 0) {
+ mstream->istream.istream.stream_errno =
+ mstream->istream.parent->stream_errno;
+ } else {
+ io_stream_set_error(&mstream->istream.iostream,
+ "Metadata header is missing ending line at offset %"PRIuUOFF_T,
+ mstream->istream.istream.v_offset);
+ mstream->istream.istream.stream_errno = EPIPE;
+ return -1;
+ }
+ mstream->istream.istream.eof = TRUE;
+ return -1;
+ }
+ i_assert(!mstream->istream.parent->blocking);
+ return 0;
+}
+
+static ssize_t i_stream_metawrap_read(struct istream_private *stream)
+{
+ struct metawrap_istream *mstream = (struct metawrap_istream *)stream;
+ int ret;
+
+ i_stream_seek(stream->parent, mstream->start_offset +
+ stream->istream.v_offset);
+
+ if (mstream->in_metadata) {
+ size_t prev_max_size = i_stream_get_max_buffer_size(stream->parent);
+
+ i_stream_set_max_buffer_size(stream->parent, METAWRAP_MAX_METADATA_LINE_LEN);
+ ret = metadata_header_read(mstream);
+ i_stream_set_max_buffer_size(stream->parent, prev_max_size);
+
+ i_assert(stream->istream.v_offset == 0);
+ mstream->start_offset = stream->parent->v_offset;
+ if (ret <= 0)
+ return ret;
+ /* this stream is kind of silently skipping over the metadata */
+ stream->start_offset += mstream->start_offset;
+ mstream->in_metadata = FALSE;
+ if (mstream->pending_seek != 0) {
+ i_stream_seek(&stream->istream, mstream->pending_seek);
+ return i_stream_read_memarea(&stream->istream);
+ }
+ }
+ /* after metadata header it's all just passthrough */
+ return i_stream_read_copy_from_parent(&stream->istream);
+}
+
+static void
+i_stream_metawrap_seek(struct istream_private *stream,
+ uoff_t v_offset, bool mark ATTR_UNUSED)
+{
+ struct metawrap_istream *mstream = (struct metawrap_istream *)stream;
+
+ if (!mstream->in_metadata) {
+ /* already read through metadata. we can skip directly. */
+ stream->istream.v_offset = v_offset;
+ mstream->pending_seek = 0;
+ } else {
+ /* we need to read through the metadata first */
+ mstream->pending_seek = v_offset;
+ stream->istream.v_offset = 0;
+ }
+ stream->skip = stream->pos = 0;
+}
+
+static int i_stream_metawrap_stat(struct istream_private *stream, bool exact)
+{
+ struct metawrap_istream *mstream = (struct metawrap_istream *)stream;
+ const struct stat *st;
+ int ret;
+
+ if (i_stream_stat(stream->parent, exact, &st) < 0) {
+ stream->istream.stream_errno = stream->parent->stream_errno;
+ return -1;
+ }
+ stream->statbuf = *st;
+
+ if (mstream->in_metadata) {
+ ret = i_stream_read_memarea(&stream->istream);
+ if (ret < 0 && stream->istream.stream_errno != 0)
+ return -1;
+ if (ret == 0) {
+ stream->statbuf.st_size = -1;
+ return 0;
+ }
+ }
+ i_assert((uoff_t)stream->statbuf.st_size >= mstream->start_offset);
+ stream->statbuf.st_size -= mstream->start_offset;
+ return 0;
+}
+
+struct istream *
+i_stream_create_metawrap(struct istream *input,
+ metawrap_callback_t *callback, void *context)
+{
+ struct metawrap_istream *mstream;
+
+ mstream = i_new(struct metawrap_istream, 1);
+ mstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+
+ mstream->istream.read = i_stream_metawrap_read;
+ mstream->istream.seek = i_stream_metawrap_seek;
+ mstream->istream.stat = input->seekable ? i_stream_metawrap_stat : NULL;
+
+ mstream->istream.istream.readable_fd = input->readable_fd;
+ mstream->istream.istream.blocking = input->blocking;
+ mstream->istream.istream.seekable = input->seekable;
+ mstream->in_metadata = TRUE;
+ mstream->callback = callback;
+ mstream->context = context;
+ return i_stream_create(&mstream->istream, input,
+ i_stream_get_fd(input), 0);
+}
diff --git a/src/lib-fs/istream-metawrap.h b/src/lib-fs/istream-metawrap.h
new file mode 100644
index 0000000..90ca2d3
--- /dev/null
+++ b/src/lib-fs/istream-metawrap.h
@@ -0,0 +1,14 @@
+#ifndef ISTREAM_METAWRAP_H
+#define ISTREAM_METAWRAP_H
+
+typedef void
+metawrap_callback_t(const char *key, const char *value, void *context);
+
+/* Input stream is in format "key:value\nkey2:value2\n...\n\ncontents.
+ The given callback is called for each key/value metadata pair, and the
+ returned stream will skip over the metadata and return only the contents. */
+struct istream *
+i_stream_create_metawrap(struct istream *input,
+ metawrap_callback_t *callback, void *context);
+
+#endif
diff --git a/src/lib-fs/ostream-cmp.c b/src/lib-fs/ostream-cmp.c
new file mode 100644
index 0000000..8599799
--- /dev/null
+++ b/src/lib-fs/ostream-cmp.c
@@ -0,0 +1,95 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "ostream-private.h"
+#include "ostream-cmp.h"
+
+struct cmp_ostream {
+ struct ostream_private ostream;
+
+ struct istream *input;
+ bool equals;
+};
+
+static void o_stream_cmp_close(struct iostream_private *stream,
+ bool close_parent)
+{
+ struct cmp_ostream *cstream = (struct cmp_ostream *)stream;
+
+ if (cstream->input == NULL)
+ return;
+
+ i_stream_unref(&cstream->input);
+ if (close_parent)
+ o_stream_close(cstream->ostream.parent);
+}
+
+bool stream_cmp_block(struct istream *input,
+ const unsigned char *data, size_t size)
+{
+ const unsigned char *indata;
+ size_t insize, max;
+
+ while (size > 0) {
+ (void)i_stream_read_bytes(input, &indata, &insize, size);
+ max = I_MIN(insize, size);
+ if (insize == 0 || memcmp(data, indata, max) != 0)
+ return FALSE;
+ data += max;
+ size -= max;
+ i_stream_skip(input, max);
+ }
+ return TRUE;
+}
+
+static ssize_t
+o_stream_cmp_sendv(struct ostream_private *stream,
+ const struct const_iovec *iov, unsigned int iov_count)
+{
+ struct cmp_ostream *cstream = (struct cmp_ostream *)stream;
+ unsigned int i;
+ ssize_t ret;
+
+ if (cstream->equals) {
+ for (i = 0; i < iov_count; i++) {
+ if (!stream_cmp_block(cstream->input, iov[i].iov_base,
+ iov[i].iov_len)) {
+ cstream->equals = FALSE;
+ break;
+ }
+ }
+ }
+
+ if ((ret = o_stream_sendv(stream->parent, iov, iov_count)) < 0) {
+ o_stream_copy_error_from_parent(stream);
+ return -1;
+ }
+
+ stream->ostream.offset += ret;
+ return ret;
+}
+
+struct ostream *
+o_stream_create_cmp(struct ostream *output, struct istream *input)
+{
+ struct cmp_ostream *cstream;
+
+ cstream = i_new(struct cmp_ostream, 1);
+ cstream->ostream.sendv = o_stream_cmp_sendv;
+ cstream->ostream.iostream.close = o_stream_cmp_close;
+ cstream->input = input;
+ cstream->equals = TRUE;
+ i_stream_ref(input);
+
+ return o_stream_create(&cstream->ostream, output,
+ o_stream_get_fd(output));
+}
+
+bool o_stream_cmp_equals(struct ostream *_output)
+{
+ struct cmp_ostream *cstream =
+ (struct cmp_ostream *)_output->real_stream;
+
+ return cstream->equals;
+}
diff --git a/src/lib-fs/ostream-cmp.h b/src/lib-fs/ostream-cmp.h
new file mode 100644
index 0000000..cd6d567
--- /dev/null
+++ b/src/lib-fs/ostream-cmp.h
@@ -0,0 +1,15 @@
+#ifndef OSTREAM_CMP_H
+#define OSTREAM_CMP_H
+
+/* Compare given input stream to output being written to output stream. */
+struct ostream *
+o_stream_create_cmp(struct ostream *output, struct istream *input);
+/* Returns TRUE if input and output are equal so far. If the caller needs to
+ know if the files are entirely equal, it should check also if input stream
+ is at EOF. */
+bool o_stream_cmp_equals(struct ostream *output);
+
+bool stream_cmp_block(struct istream *input,
+ const unsigned char *data, size_t size);
+
+#endif
diff --git a/src/lib-fs/ostream-metawrap.c b/src/lib-fs/ostream-metawrap.c
new file mode 100644
index 0000000..3359bd3
--- /dev/null
+++ b/src/lib-fs/ostream-metawrap.c
@@ -0,0 +1,71 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "ostream-private.h"
+#include "ostream-metawrap.h"
+
+struct metawrap_ostream {
+ struct ostream_private ostream;
+ void (*write_callback)(void *);
+ void *context;
+};
+
+static void o_stream_metawrap_call_callback(struct metawrap_ostream *mstream)
+{
+ void (*write_callback)(void *) = mstream->write_callback;
+
+ if (write_callback != NULL) {
+ mstream->write_callback = NULL;
+ write_callback(mstream->context);
+ /* metadata headers aren't counted as part of the offset */
+ mstream->ostream.ostream.offset = 0;
+ }
+}
+
+static ssize_t
+o_stream_metawrap_sendv(struct ostream_private *stream,
+ const struct const_iovec *iov, unsigned int iov_count)
+{
+ struct metawrap_ostream *mstream = (struct metawrap_ostream *)stream;
+ ssize_t ret;
+
+ o_stream_metawrap_call_callback(mstream);
+ if ((ret = o_stream_sendv(stream->parent, iov, iov_count)) < 0)
+ o_stream_copy_error_from_parent(stream);
+ else
+ stream->ostream.offset += ret;
+ return ret;
+}
+
+static enum ostream_send_istream_result
+o_stream_metawrap_send_istream(struct ostream_private *_outstream,
+ struct istream *instream)
+{
+ struct metawrap_ostream *outstream =
+ (struct metawrap_ostream *)_outstream;
+ uoff_t orig_instream_offset = instream->v_offset;
+ enum ostream_send_istream_result res;
+
+ o_stream_metawrap_call_callback(outstream);
+ if ((res = o_stream_send_istream(_outstream->parent, instream)) == OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT)
+ o_stream_copy_error_from_parent(_outstream);
+ _outstream->ostream.offset += instream->v_offset - orig_instream_offset;
+ return res;
+}
+
+struct ostream *
+o_stream_create_metawrap(struct ostream *output,
+ void (*write_callback)(void *), void *context)
+{
+ struct metawrap_ostream *mstream;
+
+ mstream = i_new(struct metawrap_ostream, 1);
+ mstream->ostream.sendv = o_stream_metawrap_sendv;
+ mstream->ostream.send_istream = o_stream_metawrap_send_istream;
+ mstream->write_callback = write_callback;
+ mstream->context = context;
+
+ return o_stream_create(&mstream->ostream, output,
+ o_stream_get_fd(output));
+}
diff --git a/src/lib-fs/ostream-metawrap.h b/src/lib-fs/ostream-metawrap.h
new file mode 100644
index 0000000..09a172c
--- /dev/null
+++ b/src/lib-fs/ostream-metawrap.h
@@ -0,0 +1,8 @@
+#ifndef OSTREAM_METAWRAP_H
+#define OSTREAM_METAWRAP_H
+
+struct ostream *
+o_stream_create_metawrap(struct ostream *output,
+ void (*write_callback)(void *), void *context);
+
+#endif
diff --git a/src/lib-fs/test-fs-metawrap.c b/src/lib-fs/test-fs-metawrap.c
new file mode 100644
index 0000000..161f1c5
--- /dev/null
+++ b/src/lib-fs/test-fs-metawrap.c
@@ -0,0 +1,103 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "ostream.h"
+#include "fs-test.h"
+#include "test-common.h"
+
+static const struct fs_settings fs_set;
+
+static void test_fs_metawrap_stat(void)
+{
+ struct fs *fs;
+ struct fs_file *file;
+ struct test_fs_file *test_file;
+ struct istream *input;
+ struct stat st;
+ const char *error;
+ unsigned int i;
+
+ test_begin("fs metawrap stat");
+
+ if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0)
+ i_fatal("fs_init() failed: %s", error);
+
+ for (i = 0; i < 2; i++) {
+ file = fs_file_init(fs, "foo", FS_OPEN_MODE_READONLY);
+
+ test_file = test_fs_file_get(fs, "foo");
+ str_append(test_file->contents, "key:value\n\n12345678901234567890");
+
+ if (i == 0) {
+ input = fs_read_stream(file, 2);
+ test_istream_set_max_buffer_size(test_file->input, 2);
+ } else {
+ input = NULL;
+ }
+
+ test_assert_idx(fs_stat(file, &st) == 0 && st.st_size == 20, i);
+
+ i_stream_unref(&input);
+ fs_file_deinit(&file);
+ }
+ fs_deinit(&fs);
+ test_end();
+}
+
+static void test_fs_metawrap_async(void)
+{
+ test_fs_async("metawrap", FS_PROPERTY_METADATA, "metawrap", "test");
+ test_fs_async("metawrap passthrough", 0, "metawrap", "test");
+ test_fs_async("double-metawrap", FS_PROPERTY_METADATA, "metawrap", "metawrap:test");
+}
+
+static void test_fs_metawrap_write_empty(void)
+{
+ struct fs *fs;
+ struct stat st;
+ const char *error;
+
+ test_begin("fs metawrap write empty file");
+ if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0)
+ i_fatal("fs_init() failed: %s", error);
+ struct fs_file *file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE);
+ struct ostream *output = fs_write_stream(file);
+ test_assert(fs_write_stream_finish(file, &output) > 0);
+ test_assert(fs_stat(file, &st) == 0 && st.st_size == 0);
+ fs_file_deinit(&file);
+ fs_deinit(&fs);
+ test_end();
+}
+
+static void test_fs_metawrap_write_fname_rename(void)
+{
+ struct fs *fs;
+ const char *error;
+
+ test_begin("fs metawrap write fname rename");
+ if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0)
+ i_fatal("fs_init() failed: %s", error);
+ struct fs_file *file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE);
+ struct ostream *output = fs_write_stream(file);
+ o_stream_nsend_str(output, "test");
+ fs_set_metadata(file, FS_METADATA_WRITE_FNAME, "renamed");
+ test_assert(fs_write_stream_finish(file, &output) > 0);
+ test_assert(strcmp(fs_file_path(file), "renamed") == 0);
+ fs_file_deinit(&file);
+ fs_deinit(&fs);
+ test_end();
+}
+
+int main(void)
+{
+ static void (*const test_functions[])(void) = {
+ test_fs_metawrap_stat,
+ test_fs_metawrap_async,
+ test_fs_metawrap_write_empty,
+ test_fs_metawrap_write_fname_rename,
+ NULL
+ };
+ return test_run(test_functions);
+}
diff --git a/src/lib-fs/test-fs-posix.c b/src/lib-fs/test-fs-posix.c
new file mode 100644
index 0000000..d6fea11
--- /dev/null
+++ b/src/lib-fs/test-fs-posix.c
@@ -0,0 +1,144 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "ostream.h"
+#include "fs-api.h"
+#include "safe-mkdir.h"
+#include "safe-mkstemp.h"
+#include "test-common.h"
+#include "unlink-directory.h"
+#include <sys/stat.h>
+#include <unistd.h>
+
+static void test_fs_posix(void)
+{
+ const char testdir[] = ".test-fs-posix";
+ const char *unlink_err;
+
+ if (unlink_directory(testdir, UNLINK_DIRECTORY_FLAG_RMDIR, &unlink_err) < 0) {
+ i_error("Couldn't prepare test directory (%s): %s", testdir, unlink_err);
+ goto error_no_testdir;
+ }
+ if (safe_mkdir(testdir, 0700, (uid_t)-1, (gid_t)-1) != 1) {
+ /* Something just raced us to create this directory, bail. */
+ goto error_no_testdir;
+ }
+
+ int ret;
+ const char *error;
+ struct fs *fs;
+ struct fs_settings fs_set;
+
+ test_begin("test-fs-posix filesystem");
+ i_zero(&fs_set);
+ ret = fs_init("posix", t_strdup_printf("prefix=%s/", testdir), &fs_set, &fs, &error);
+ test_out_quiet("fs_init() failed", ret >= 0);
+ if (ret < 0) {
+ test_end();
+ goto error_no_fs;
+ }
+
+ struct fs *ref = fs;
+ fs_ref(ref);
+ fs_unref(&ref);
+ test_assert(ref == NULL);
+ test_assert(fs != NULL);
+
+ test_assert(fs_get_parent(fs) == NULL);
+ test_assert_strcmp(fs_get_driver(fs), "posix");
+ test_end();
+
+ struct fs_file *file;
+ char buf[10];
+ ssize_t count;
+ test_begin("test-fs-posix bad file read");
+ file = fs_file_init(fs, "fail_1", FS_OPEN_MODE_READONLY);
+ test_assert(fs_exists(file) == 0);
+ count = fs_read(file, buf, 1);
+ test_assert(count == -1 && errno == ENOENT &&
+ strstr(fs_file_last_error(file), "No such file or directory") != NULL);
+ fs_file_deinit(&file);
+ test_end();
+
+ test_begin("test-fs-posix good file write");
+ file = fs_file_init(fs, "good1", FS_OPEN_MODE_REPLACE);
+ test_assert(file != NULL);
+ test_assert(fs_exists(file) == 0); /* file not created until data is written */
+ test_assert(fs_write(file, "X", 1) == 0);
+ test_assert(fs_exists(file) == 1);
+ fs_file_deinit(&file);
+ test_end();
+
+ test_begin("test-fs-posix good file read");
+ file = fs_file_init(fs, "good1", FS_OPEN_MODE_READONLY);
+ test_assert(fs_exists(file) == 1);
+ errno = 0;
+ count = fs_read(file, buf, 2);
+ test_assert(count == 1 && errno == 0);
+ fs_file_deinit(&file);
+ test_end();
+
+ struct fs_iter *iter = fs_iter_init(fs, "/", 0);
+ const char *filename;
+ test_begin("test-fs-posix iterator");
+ filename = fs_iter_next(iter);
+ test_assert_strcmp(filename, "good1");
+ test_assert(fs_iter_next(iter) == NULL);
+ fs_iter_deinit(&iter, &error);
+ test_end();
+
+ struct stat st;
+ test_begin("test-fs-posix file stat and delete");
+ file = fs_file_init(fs, "good1", FS_OPEN_MODE_READONLY);
+ test_assert(fs_stat(file, &st) == 0);
+ test_assert(st.st_size == 1);
+ test_assert(fs_delete(file) == 0);
+ fs_file_deinit(&file);
+ test_end();
+
+ test_begin("test-fs-posix file write fname rename");
+ file = fs_file_init(fs, "subdir/badfname", FS_OPEN_MODE_REPLACE);
+ struct ostream *output = fs_write_stream(file);
+ o_stream_nsend_str(output, "hello");
+ fs_set_metadata(file, FS_METADATA_WRITE_FNAME, "subdir/rename1");
+ test_assert(fs_write_stream_finish(file, &output) == 1);
+ test_assert(strcmp(fs_file_path(file), "subdir/rename1") == 0);
+ fs_file_deinit(&file);
+ file = fs_file_init(fs, "subdir/rename1", FS_OPEN_MODE_READONLY);
+ test_assert(fs_stat(file, &st) == 0);
+ test_assert(st.st_size == 5);
+ fs_file_deinit(&file);
+ test_end();
+
+ test_begin("test-fs-posix file copy fname rename");
+ struct fs_file *src = fs_file_init(fs, "subdir/rename1", FS_OPEN_MODE_READONLY);
+ struct fs_file *dest = fs_file_init(fs, "subdir/badfname", FS_OPEN_MODE_REPLACE);
+ fs_set_metadata(dest, FS_METADATA_WRITE_FNAME, "subdir/rename2");
+ test_assert(fs_copy(src, dest) == 0);
+ test_assert(strcmp(fs_file_path(dest), "subdir/rename2") == 0);
+ fs_file_deinit(&src);
+ fs_file_deinit(&dest);
+ file = fs_file_init(fs, "subdir/rename2", FS_OPEN_MODE_READONLY);
+ test_assert(fs_stat(file, &st) == 0);
+ test_assert(st.st_size == 5);
+ fs_file_deinit(&file);
+ test_end();
+
+ fs_deinit(&fs);
+
+error_no_fs:
+ if (unlink_directory(testdir, UNLINK_DIRECTORY_FLAG_RMDIR, &unlink_err) < 0)
+ i_error("Couldn't clean up test directory (%s): %s", testdir, unlink_err);
+error_no_testdir:
+ return;
+}
+
+int main(void)
+{
+ static void (*const test_functions[])(void) = {
+ test_fs_posix,
+ NULL
+ };
+ return test_run(test_functions);
+}