diff options
Diffstat (limited to 'src/lib-fs')
30 files changed, 7717 insertions, 0 deletions
diff --git a/src/lib-fs/Makefile.am b/src/lib-fs/Makefile.am new file mode 100644 index 0000000..e5ae056 --- /dev/null +++ b/src/lib-fs/Makefile.am @@ -0,0 +1,70 @@ +noinst_LTLIBRARIES = libfs.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -DMODULE_DIR=\""$(moduledir)"\" + +libfs_la_SOURCES = \ + fs-api.c \ + fs-dict.c \ + fs-metawrap.c \ + fs-randomfail.c \ + fs-posix.c \ + fs-test.c \ + fs-test-async.c \ + fs-sis.c \ + fs-sis-common.c \ + fs-sis-queue.c \ + fs-wrapper.c \ + istream-fs-file.c \ + istream-fs-stats.c \ + istream-metawrap.c \ + ostream-metawrap.c \ + ostream-cmp.c + +headers = \ + fs-api.h \ + fs-api-private.h \ + fs-sis-common.h \ + fs-wrapper.h \ + fs-test.h \ + istream-fs-file.h \ + istream-fs-stats.h \ + istream-metawrap.h \ + ostream-metawrap.h \ + ostream-cmp.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + +noinst_PROGRAMS = $(test_programs) + +test_programs = \ + test-fs-metawrap \ + test-fs-posix + +test_deps = \ + $(noinst_LTLIBRARIES) \ + ../lib-dict/libdict.la \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_libs = \ + $(test_deps) \ + $(MODULE_LIBS) + +test_fs_metawrap_SOURCES = test-fs-metawrap.c +test_fs_metawrap_LDADD = $(test_libs) +test_fs_metawrap_DEPENDENCIES = $(test_deps) + +test_fs_posix_SOURCES = test-fs-posix.c +test_fs_posix_LDADD = $(test_libs) +test_fs_posix_DEPENDENCIES = $(test_deps) + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/lib-fs/Makefile.in b/src/lib-fs/Makefile.in new file mode 100644 index 0000000..19cbe6b --- /dev/null +++ b/src/lib-fs/Makefile.in @@ -0,0 +1,955 @@ +# Makefile.in generated by automake 1.16.3 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/lib-fs +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-fs-metawrap$(EXEEXT) test-fs-posix$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +LTLIBRARIES = $(noinst_LTLIBRARIES) +libfs_la_LIBADD = +am_libfs_la_OBJECTS = fs-api.lo fs-dict.lo fs-metawrap.lo \ + fs-randomfail.lo fs-posix.lo fs-test.lo fs-test-async.lo \ + fs-sis.lo fs-sis-common.lo fs-sis-queue.lo fs-wrapper.lo \ + istream-fs-file.lo istream-fs-stats.lo istream-metawrap.lo \ + ostream-metawrap.lo ostream-cmp.lo +libfs_la_OBJECTS = $(am_libfs_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am_test_fs_metawrap_OBJECTS = test-fs-metawrap.$(OBJEXT) +test_fs_metawrap_OBJECTS = $(am_test_fs_metawrap_OBJECTS) +am__DEPENDENCIES_1 = +am__DEPENDENCIES_2 = $(test_deps) $(am__DEPENDENCIES_1) +am_test_fs_posix_OBJECTS = test-fs-posix.$(OBJEXT) +test_fs_posix_OBJECTS = $(am_test_fs_posix_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/fs-api.Plo ./$(DEPDIR)/fs-dict.Plo \ + ./$(DEPDIR)/fs-metawrap.Plo ./$(DEPDIR)/fs-posix.Plo \ + ./$(DEPDIR)/fs-randomfail.Plo ./$(DEPDIR)/fs-sis-common.Plo \ + ./$(DEPDIR)/fs-sis-queue.Plo ./$(DEPDIR)/fs-sis.Plo \ + ./$(DEPDIR)/fs-test-async.Plo ./$(DEPDIR)/fs-test.Plo \ + ./$(DEPDIR)/fs-wrapper.Plo ./$(DEPDIR)/istream-fs-file.Plo \ + ./$(DEPDIR)/istream-fs-stats.Plo \ + ./$(DEPDIR)/istream-metawrap.Plo ./$(DEPDIR)/ostream-cmp.Plo \ + ./$(DEPDIR)/ostream-metawrap.Plo \ + ./$(DEPDIR)/test-fs-metawrap.Po ./$(DEPDIR)/test-fs-posix.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libfs_la_SOURCES) $(test_fs_metawrap_SOURCES) \ + $(test_fs_posix_SOURCES) +DIST_SOURCES = $(libfs_la_SOURCES) $(test_fs_metawrap_SOURCES) \ + $(test_fs_posix_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libfs.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -DMODULE_DIR=\""$(moduledir)"\" + +libfs_la_SOURCES = \ + fs-api.c \ + fs-dict.c \ + fs-metawrap.c \ + fs-randomfail.c \ + fs-posix.c \ + fs-test.c \ + fs-test-async.c \ + fs-sis.c \ + fs-sis-common.c \ + fs-sis-queue.c \ + fs-wrapper.c \ + istream-fs-file.c \ + istream-fs-stats.c \ + istream-metawrap.c \ + ostream-metawrap.c \ + ostream-cmp.c + +headers = \ + fs-api.h \ + fs-api-private.h \ + fs-sis-common.h \ + fs-wrapper.h \ + fs-test.h \ + istream-fs-file.h \ + istream-fs-stats.h \ + istream-metawrap.h \ + ostream-metawrap.h \ + ostream-cmp.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +test_programs = \ + test-fs-metawrap \ + test-fs-posix + +test_deps = \ + $(noinst_LTLIBRARIES) \ + ../lib-dict/libdict.la \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_libs = \ + $(test_deps) \ + $(MODULE_LIBS) + +test_fs_metawrap_SOURCES = test-fs-metawrap.c +test_fs_metawrap_LDADD = $(test_libs) +test_fs_metawrap_DEPENDENCIES = $(test_deps) +test_fs_posix_SOURCES = test-fs-posix.c +test_fs_posix_LDADD = $(test_libs) +test_fs_posix_DEPENDENCIES = $(test_deps) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-fs/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-fs/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libfs.la: $(libfs_la_OBJECTS) $(libfs_la_DEPENDENCIES) $(EXTRA_libfs_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libfs_la_OBJECTS) $(libfs_la_LIBADD) $(LIBS) + +test-fs-metawrap$(EXEEXT): $(test_fs_metawrap_OBJECTS) $(test_fs_metawrap_DEPENDENCIES) $(EXTRA_test_fs_metawrap_DEPENDENCIES) + @rm -f test-fs-metawrap$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_metawrap_OBJECTS) $(test_fs_metawrap_LDADD) $(LIBS) + +test-fs-posix$(EXEEXT): $(test_fs_posix_OBJECTS) $(test_fs_posix_DEPENDENCIES) $(EXTRA_test_fs_posix_DEPENDENCIES) + @rm -f test-fs-posix$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_posix_OBJECTS) $(test_fs_posix_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-api.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-dict.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-metawrap.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-posix.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-randomfail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-sis-common.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-sis-queue.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-sis.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-test-async.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-test.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs-wrapper.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-fs-file.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-fs-stats.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/istream-metawrap.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ostream-cmp.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ostream-metawrap.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-fs-metawrap.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-fs-posix.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/fs-api.Plo + -rm -f ./$(DEPDIR)/fs-dict.Plo + -rm -f ./$(DEPDIR)/fs-metawrap.Plo + -rm -f ./$(DEPDIR)/fs-posix.Plo + -rm -f ./$(DEPDIR)/fs-randomfail.Plo + -rm -f ./$(DEPDIR)/fs-sis-common.Plo + -rm -f ./$(DEPDIR)/fs-sis-queue.Plo + -rm -f ./$(DEPDIR)/fs-sis.Plo + -rm -f ./$(DEPDIR)/fs-test-async.Plo + -rm -f ./$(DEPDIR)/fs-test.Plo + -rm -f ./$(DEPDIR)/fs-wrapper.Plo + -rm -f ./$(DEPDIR)/istream-fs-file.Plo + -rm -f ./$(DEPDIR)/istream-fs-stats.Plo + -rm -f ./$(DEPDIR)/istream-metawrap.Plo + -rm -f ./$(DEPDIR)/ostream-cmp.Plo + -rm -f ./$(DEPDIR)/ostream-metawrap.Plo + -rm -f ./$(DEPDIR)/test-fs-metawrap.Po + -rm -f ./$(DEPDIR)/test-fs-posix.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/fs-api.Plo + -rm -f ./$(DEPDIR)/fs-dict.Plo + -rm -f ./$(DEPDIR)/fs-metawrap.Plo + -rm -f ./$(DEPDIR)/fs-posix.Plo + -rm -f ./$(DEPDIR)/fs-randomfail.Plo + -rm -f ./$(DEPDIR)/fs-sis-common.Plo + -rm -f ./$(DEPDIR)/fs-sis-queue.Plo + -rm -f ./$(DEPDIR)/fs-sis.Plo + -rm -f ./$(DEPDIR)/fs-test-async.Plo + -rm -f ./$(DEPDIR)/fs-test.Plo + -rm -f ./$(DEPDIR)/fs-wrapper.Plo + -rm -f ./$(DEPDIR)/istream-fs-file.Plo + -rm -f ./$(DEPDIR)/istream-fs-stats.Plo + -rm -f ./$(DEPDIR)/istream-metawrap.Plo + -rm -f ./$(DEPDIR)/ostream-cmp.Plo + -rm -f ./$(DEPDIR)/ostream-metawrap.Plo + -rm -f ./$(DEPDIR)/test-fs-metawrap.Po + -rm -f ./$(DEPDIR)/test-fs-posix.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-pkginc_libHEADERS install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h new file mode 100644 index 0000000..76eb2c5 --- /dev/null +++ b/src/lib-fs/fs-api-private.h @@ -0,0 +1,213 @@ +#ifndef FS_API_PRIVATE_H +#define FS_API_PRIVATE_H + +#include "fs-api.h" +#include "fs-wrapper.h" +#include "module-context.h" + +#include <sys/time.h> + +#define FS_EVENT_FIELD_FS "lib-fs#fs" +#define FS_EVENT_FIELD_FILE "lib-fs#file" +#define FS_EVENT_FIELD_ITER "lib-fs#iter" + +enum fs_get_metadata_flags { + FS_GET_METADATA_FLAG_LOADED_ONLY = BIT(0), +}; + +struct fs_api_module_register { + unsigned int id; +}; + +union fs_api_module_context { + struct fs_api_module_register *reg; +}; + +extern struct fs_api_module_register fs_api_module_register; + +struct fs_vfuncs { + struct fs *(*alloc)(void); + int (*init)(struct fs *fs, const char *args, + const struct fs_settings *set, const char **error_r); + void (*deinit)(struct fs *fs); + void (*free)(struct fs *fs); + + enum fs_properties (*get_properties)(struct fs *fs); + + struct fs_file *(*file_alloc)(void); + void (*file_init)(struct fs_file *file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags); + void (*file_deinit)(struct fs_file *file); + void (*file_close)(struct fs_file *file); + const char *(*get_path)(struct fs_file *file); + + void (*set_async_callback)(struct fs_file *file, + fs_file_async_callback_t *callback, + void *context); + void (*wait_async)(struct fs *fs); + + void (*set_metadata)(struct fs_file *file, const char *key, + const char *value); + int (*get_metadata)(struct fs_file *file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r); + + bool (*prefetch)(struct fs_file *file, uoff_t length); + ssize_t (*read)(struct fs_file *file, void *buf, size_t size); + struct istream *(*read_stream)(struct fs_file *file, + size_t max_buffer_size); + + int (*write)(struct fs_file *file, const void *data, size_t size); + void (*write_stream)(struct fs_file *file); + /* After write_stream_finish() is called once, all the following + (async) calls will have success==TRUE. */ + int (*write_stream_finish)(struct fs_file *file, bool success); + + int (*lock)(struct fs_file *file, unsigned int secs, + struct fs_lock **lock_r); + void (*unlock)(struct fs_lock *lock); + + int (*exists)(struct fs_file *file); + int (*stat)(struct fs_file *file, struct stat *st_r); + int (*copy)(struct fs_file *src, struct fs_file *dest); + int (*rename)(struct fs_file *src, struct fs_file *dest); + int (*delete_file)(struct fs_file *file); + + struct fs_iter *(*iter_alloc)(void); + void (*iter_init)(struct fs_iter *iter, const char *path, + enum fs_iter_flags flags); + const char *(*iter_next)(struct fs_iter *iter); + int (*iter_deinit)(struct fs_iter *iter); + + bool (*switch_ioloop)(struct fs *fs); + int (*get_nlinks)(struct fs_file *file, nlink_t *nlinks_r); +}; + +struct fs { + struct fs *parent; /* for wrapper filesystems */ + const char *name; + struct fs_vfuncs v; + char *temp_path_prefix; + int refcount; + + char *username, *session_id; + + struct fs_settings set; + + /* may be used by fs_wait_async() to do the waiting */ + struct ioloop *wait_ioloop, *prev_ioloop; + + unsigned int files_open_count; + struct fs_file *files; + struct fs_iter *iters; + struct event *event; + + struct fs_stats stats; + + ARRAY(union fs_api_module_context *) module_contexts; +}; + +struct fs_file { + /* linked list of all files */ + struct fs_file *prev, *next; + + struct fs_file *parent; /* for wrapper filesystems */ + struct fs *fs; + struct ostream *output; + struct event *event; + char *path; + char *last_error; + enum fs_open_flags flags; + + struct istream *seekable_input; + struct istream *pending_read_input; + + const struct hash_method *write_digest_method; + void *write_digest; + + pool_t metadata_pool; + ARRAY_TYPE(fs_metadata) metadata; + + struct fs_file *copy_src; + struct istream *copy_input; + struct ostream *copy_output; + + struct timeval timing_start[FS_OP_COUNT]; + + bool write_pending:1; + bool writing_stream:1; + bool metadata_changed:1; + + bool read_or_prefetch_counted:1; + bool lookup_metadata_counted:1; + bool stat_counted:1; + bool copy_counted:1; + bool istream_open:1; + bool last_error_changed:1; +}; + +struct fs_lock { + struct fs_file *file; +}; + +struct fs_iter { + /* linked list of all iters */ + struct fs_iter *prev, *next; + + struct fs *fs; + struct event *event; + char *path; + enum fs_iter_flags flags; + struct timeval start_time; + char *last_error; + + bool async_have_more; + fs_file_async_callback_t *async_callback; + void *async_context; +}; + +extern const struct fs fs_class_dict; +extern const struct fs fs_class_posix; +extern const struct fs fs_class_randomfail; +extern const struct fs fs_class_metawrap; +extern const struct fs fs_class_sis; +extern const struct fs fs_class_sis_queue; +extern const struct fs fs_class_test; + +void fs_class_register(const struct fs *fs_class); + +/* Event must be fs_file or fs_iter events. Set errno from err. */ +void fs_set_error(struct event *event, int err, + const char *fmt, ...) ATTR_FORMAT(3, 4); +/* Like fs_set_error(), but use the existing errno. */ +void fs_set_error_errno(struct event *event, const char *fmt, ...) ATTR_FORMAT(2, 3); +void fs_file_set_error_async(struct fs_file *file); + +ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size); +int fs_write_via_stream(struct fs_file *file, const void *data, size_t size); +void fs_metadata_init(struct fs_file *file); +void fs_metadata_init_or_clear(struct fs_file *file); +void fs_default_set_metadata(struct fs_file *file, + const char *key, const char *value); +int fs_get_metadata_full(struct fs_file *file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r); +const char *fs_metadata_find(const ARRAY_TYPE(fs_metadata) *metadata, + const char *key); +int fs_default_copy(struct fs_file *src, struct fs_file *dest); + +void fs_file_timing_end(struct fs_file *file, enum fs_op op); + +struct fs_file * +fs_file_init_parent(struct fs_file *parent, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags); +struct fs_iter * +fs_iter_init_parent(struct fs_iter *parent, + const char *path, enum fs_iter_flags flags); +void fs_file_free(struct fs_file *file); + +/* Same as fs_write_stream_abort_error(), except it closes the *parent* file + and error is left untouched */ +void fs_write_stream_abort_parent(struct fs_file *file, struct ostream **output); + +#endif diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c new file mode 100644 index 0000000..c53e142 --- /dev/null +++ b/src/lib-fs/fs-api.c @@ -0,0 +1,1412 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "module-dir.h" +#include "llist.h" +#include "str.h" +#include "hash-method.h" +#include "istream.h" +#include "istream-seekable.h" +#include "ostream.h" +#include "stats-dist.h" +#include "time-util.h" +#include "istream-fs-stats.h" +#include "fs-api-private.h" + +static struct event_category event_category_fs = { + .name = "fs" +}; + +struct fs_api_module_register fs_api_module_register = { 0 }; + +static struct module *fs_modules = NULL; +static ARRAY(const struct fs *) fs_classes; + +static void fs_classes_init(void); + +static struct event *fs_create_event(struct fs *fs, struct event *parent) +{ + struct event *event; + + event = event_create(parent); + event_add_category(event, &event_category_fs); + event_set_append_log_prefix(event, + t_strdup_printf("fs-%s: ", fs->name)); + return event; +} + +static int +fs_alloc(const struct fs *fs_class, const char *args, + const struct fs_settings *set, struct fs **fs_r, const char **error_r) +{ + struct fs *fs; + const char *error; + int ret; + + fs = fs_class->v.alloc(); + fs->refcount = 1; + fs->set.debug = set->debug; + fs->set.enable_timing = set->enable_timing; + i_array_init(&fs->module_contexts, 5); + fs->event = fs_create_event(fs, set->event_parent); + event_set_forced_debug(fs->event, fs->set.debug); + + T_BEGIN { + ret = fs_class->v.init(fs, args, set, &error); + } T_END_PASS_STR_IF(ret < 0, &error); + if (ret < 0) { + /* a bit kludgy way to allow data stack frame usage in normal + conditions but still be able to return error message from + data stack. */ + *error_r = t_strdup_printf("%s: %s", fs_class->name, error); + fs_unref(&fs); + return -1; + } + fs->username = i_strdup(set->username); + fs->session_id = i_strdup(set->session_id); + *fs_r = fs; + return 0; +} + +void fs_class_register(const struct fs *fs_class) +{ + if (!array_is_created(&fs_classes)) + fs_classes_init(); + array_push_back(&fs_classes, &fs_class); +} + +static void fs_classes_deinit(void) +{ + array_free(&fs_classes); +} + +static void fs_classes_init(void) +{ + i_array_init(&fs_classes, 8); + fs_class_register(&fs_class_dict); + fs_class_register(&fs_class_posix); + fs_class_register(&fs_class_randomfail); + fs_class_register(&fs_class_metawrap); + fs_class_register(&fs_class_sis); + fs_class_register(&fs_class_sis_queue); + fs_class_register(&fs_class_test); + lib_atexit(fs_classes_deinit); +} + +static const struct fs *fs_class_find(const char *driver) +{ + const struct fs *class; + + if (!array_is_created(&fs_classes)) + fs_classes_init(); + + array_foreach_elem(&fs_classes, class) { + if (strcmp(class->name, driver) == 0) + return class; + } + return NULL; +} + +static void fs_class_deinit_modules(void) +{ + module_dir_unload(&fs_modules); +} + +static const char *fs_driver_module_name(const char *driver) +{ + return t_str_replace(driver, '-', '_'); +} + +static void fs_class_try_load_plugin(const char *driver) +{ + const char *module_name = + t_strdup_printf("fs_%s", fs_driver_module_name(driver)); + struct module *module; + struct module_dir_load_settings mod_set; + const struct fs *fs_class; + + i_zero(&mod_set); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.ignore_missing = TRUE; + + fs_modules = module_dir_load_missing(fs_modules, MODULE_DIR, + module_name, &mod_set); + module_dir_init(fs_modules); + + module = module_dir_find(fs_modules, module_name); + fs_class = module == NULL ? NULL : + module_get_symbol(module, t_strdup_printf( + "fs_class_%s", fs_driver_module_name(driver))); + if (fs_class != NULL) + fs_class_register(fs_class); + + lib_atexit(fs_class_deinit_modules); +} + +int fs_init(const char *driver, const char *args, + const struct fs_settings *set, + struct fs **fs_r, const char **error_r) +{ + const struct fs *fs_class; + const char *temp_file_prefix; + + fs_class = fs_class_find(driver); + if (fs_class == NULL) { + T_BEGIN { + fs_class_try_load_plugin(driver); + } T_END; + fs_class = fs_class_find(driver); + } + if (fs_class == NULL) { + *error_r = t_strdup_printf("Unknown fs driver: %s", driver); + return -1; + } + if (fs_alloc(fs_class, args, set, fs_r, error_r) < 0) + return -1; + event_set_ptr((*fs_r)->event, FS_EVENT_FIELD_FS, *fs_r); + + temp_file_prefix = set->temp_file_prefix != NULL ? + set->temp_file_prefix : ".temp.dovecot"; + if(set->temp_dir == NULL) + (*fs_r)->temp_path_prefix = i_strconcat("/tmp/", + temp_file_prefix, NULL); + else + (*fs_r)->temp_path_prefix = i_strconcat(set->temp_dir, "/", + temp_file_prefix, NULL); + return 0; +} + +int fs_init_from_string(const char *str, const struct fs_settings *set, + struct fs **fs_r, const char **error_r) +{ + const char *args = strpbrk(str, " :"); + if (args == NULL) + args = ""; + else + str = t_strdup_until(str, args++); + return fs_init(str, args, set, fs_r, error_r); +} + +void fs_deinit(struct fs **fs) +{ + fs_unref(fs); +} + +void fs_ref(struct fs *fs) +{ + i_assert(fs->refcount > 0); + + fs->refcount++; +} + +void fs_unref(struct fs **_fs) +{ + struct fs *fs = *_fs; + struct array module_contexts_arr; + unsigned int i; + + if (fs == NULL) + return; + + module_contexts_arr = fs->module_contexts.arr; + + i_assert(fs->refcount > 0); + + *_fs = NULL; + + if (--fs->refcount > 0) + return; + + if (fs->files_open_count > 0) { + i_panic("fs-%s: %u files still open (first = %s)", + fs->name, fs->files_open_count, fs_file_path(fs->files)); + } + i_assert(fs->files == NULL); + + if (fs->v.deinit != NULL) + fs->v.deinit(fs); + + fs_deinit(&fs->parent); + event_unref(&fs->event); + i_free(fs->username); + i_free(fs->session_id); + i_free(fs->temp_path_prefix); + for (i = 0; i < FS_OP_COUNT; i++) { + if (fs->stats.timings[i] != NULL) + stats_dist_deinit(&fs->stats.timings[i]); + } + T_BEGIN { + fs->v.free(fs); + } T_END; + array_free_i(&module_contexts_arr); +} + +struct fs *fs_get_parent(struct fs *fs) +{ + return fs->parent; +} + +const char *fs_get_driver(struct fs *fs) +{ + return fs->name; +} + +const char *fs_get_root_driver(struct fs *fs) +{ + while (fs->parent != NULL) + fs = fs->parent; + return fs->name; +} + +struct fs_file *fs_file_init(struct fs *fs, const char *path, int mode_flags) +{ + return fs_file_init_with_event(fs, fs->event, path, mode_flags); +} + +struct fs_file *fs_file_init_with_event(struct fs *fs, struct event *event, + const char *path, int mode_flags) +{ + struct fs_file *file; + + i_assert(path != NULL); + i_assert((mode_flags & FS_OPEN_FLAG_ASYNC_NOQUEUE) == 0 || + (mode_flags & FS_OPEN_FLAG_ASYNC) != 0); + + T_BEGIN { + file = fs->v.file_alloc(); + file->fs = fs; + file->flags = mode_flags & ENUM_NEGATE(FS_OPEN_MODE_MASK); + file->event = fs_create_event(fs, event); + event_set_ptr(file->event, FS_EVENT_FIELD_FS, fs); + event_set_ptr(file->event, FS_EVENT_FIELD_FILE, file); + fs->v.file_init(file, path, mode_flags & FS_OPEN_MODE_MASK, + mode_flags & ENUM_NEGATE(FS_OPEN_MODE_MASK)); + } T_END; + + fs->files_open_count++; + DLLIST_PREPEND(&fs->files, file); + + fs_set_metadata(file, FS_METADATA_ORIG_PATH, path); + return file; +} + +void fs_file_deinit(struct fs_file **_file) +{ + struct fs_file *file = *_file; + + if (file == NULL) + return; + + i_assert(file->fs->files_open_count > 0); + + *_file = NULL; + + fs_file_close(file); + + DLLIST_REMOVE(&file->fs->files, file); + file->fs->files_open_count--; + T_BEGIN { + file->fs->v.file_deinit(file); + } T_END; +} + +void fs_file_free(struct fs_file *file) +{ + if (file->last_error_changed) { + /* fs_set_error() used without ever accessing it via + fs_file_last_error(). Log it to make sure it's not lost. + Note that the errors are always set only to the file at + the root of the parent hierarchy. */ + e_error(file->event, "%s (in file %s deinit)", + file->last_error, fs_file_path(file)); + } + + fs_file_deinit(&file->parent); + event_unref(&file->event); + pool_unref(&file->metadata_pool); + i_free(file->last_error); +} + +void fs_file_set_flags(struct fs_file *file, + enum fs_open_flags add_flags, + enum fs_open_flags remove_flags) +{ + file->flags |= add_flags; + file->flags &= ENUM_NEGATE(remove_flags); + + if (file->parent != NULL) + fs_file_set_flags(file->parent, add_flags, remove_flags); +} + +void fs_file_close(struct fs_file *file) +{ + if (file == NULL) + return; + + i_assert(!file->writing_stream); + i_assert(file->output == NULL); + + if (file->pending_read_input != NULL) + i_stream_unref(&file->pending_read_input); + if (file->seekable_input != NULL) + i_stream_unref(&file->seekable_input); + + if (file->copy_input != NULL) { + i_stream_unref(&file->copy_input); + fs_write_stream_abort_error(file, &file->copy_output, "fs_file_close(%s)", + o_stream_get_name(file->copy_output)); + } + i_free_and_null(file->write_digest); + if (file->fs->v.file_close != NULL) T_BEGIN { + file->fs->v.file_close(file); + } T_END; + + /* check this only after closing, because some of the fs backends keep + the istream internally open and don't call the destroy-callback + until after file_close() */ + i_assert(!file->istream_open); +} + +enum fs_properties fs_get_properties(struct fs *fs) +{ + return fs->v.get_properties(fs); +} + +void fs_metadata_init(struct fs_file *file) +{ + if (file->metadata_pool == NULL) { + i_assert(!array_is_created(&file->metadata)); + file->metadata_pool = pool_alloconly_create("fs metadata", 1024); + p_array_init(&file->metadata, file->metadata_pool, 8); + } +} + +void fs_metadata_init_or_clear(struct fs_file *file) +{ + if (file->metadata_pool == NULL) + fs_metadata_init(file); + else T_BEGIN { + const struct fs_metadata *md; + ARRAY_TYPE(fs_metadata) internal_metadata; + + t_array_init(&internal_metadata, 4); + array_foreach(&file->metadata, md) { + if (strncmp(md->key, FS_METADATA_INTERNAL_PREFIX, + strlen(FS_METADATA_INTERNAL_PREFIX)) == 0) + array_push_back(&internal_metadata, md); + } + array_clear(&file->metadata); + array_append_array(&file->metadata, &internal_metadata); + } T_END; +} + +static struct fs_metadata * +fs_metadata_find_md(const ARRAY_TYPE(fs_metadata) *metadata, + const char *key) +{ + struct fs_metadata *md; + + array_foreach_modifiable(metadata, md) { + if (strcmp(md->key, key) == 0) + return md; + } + return NULL; +} + +void fs_default_set_metadata(struct fs_file *file, + const char *key, const char *value) +{ + struct fs_metadata *metadata; + + fs_metadata_init(file); + metadata = fs_metadata_find_md(&file->metadata, key); + if (metadata == NULL) { + metadata = array_append_space(&file->metadata); + metadata->key = p_strdup(file->metadata_pool, key); + } + metadata->value = p_strdup(file->metadata_pool, value); +} + +const char *fs_metadata_find(const ARRAY_TYPE(fs_metadata) *metadata, + const char *key) +{ + const struct fs_metadata *md; + + if (!array_is_created(metadata)) + return NULL; + + md = fs_metadata_find_md(metadata, key); + return md == NULL ? NULL : md->value; +} + +void fs_set_metadata(struct fs_file *file, const char *key, const char *value) +{ + i_assert(key != NULL); + i_assert(value != NULL); + + if (file->fs->v.set_metadata != NULL) T_BEGIN { + file->fs->v.set_metadata(file, key, value); + if (strncmp(key, FS_METADATA_INTERNAL_PREFIX, + strlen(FS_METADATA_INTERNAL_PREFIX)) == 0) { + /* internal metadata change, which isn't stored. */ + } else { + file->metadata_changed = TRUE; + } + } T_END; +} + +static void fs_file_timing_start(struct fs_file *file, enum fs_op op) +{ + if (!file->fs->set.enable_timing) + return; + if (file->timing_start[op].tv_sec == 0) + i_gettimeofday(&file->timing_start[op]); +} + +static void +fs_timing_end(struct stats_dist **timing, const struct timeval *start_tv) +{ + struct timeval now; + long long diff; + + i_gettimeofday(&now); + + diff = timeval_diff_usecs(&now, start_tv); + if (diff > 0) { + if (*timing == NULL) + *timing = stats_dist_init(); + stats_dist_add(*timing, diff); + } +} + +void fs_file_timing_end(struct fs_file *file, enum fs_op op) +{ + if (!file->fs->set.enable_timing || file->timing_start[op].tv_sec == 0) + return; + + fs_timing_end(&file->fs->stats.timings[op], &file->timing_start[op]); + /* don't count this again */ + file->timing_start[op].tv_sec = 0; +} + +int fs_get_metadata_full(struct fs_file *file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + int ret; + + if (file->fs->v.get_metadata == NULL) { + if (array_is_created(&file->metadata)) { + /* Return internal metadata. */ + *metadata_r = &file->metadata; + return 0; + } + fs_set_error(file->event, ENOTSUP, "Metadata not supported by backend"); + return -1; + } + if (!file->read_or_prefetch_counted && + !file->lookup_metadata_counted) { + if ((flags & FS_GET_METADATA_FLAG_LOADED_ONLY) == 0) { + file->lookup_metadata_counted = TRUE; + file->fs->stats.lookup_metadata_count++; + } + fs_file_timing_start(file, FS_OP_METADATA); + } + T_BEGIN { + ret = file->fs->v.get_metadata(file, flags, metadata_r); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_METADATA); + return ret; +} + +int fs_get_metadata(struct fs_file *file, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + return fs_get_metadata_full(file, 0, metadata_r); +} + +int fs_lookup_metadata(struct fs_file *file, const char *key, + const char **value_r) +{ + const ARRAY_TYPE(fs_metadata) *metadata; + + if (fs_get_metadata(file, &metadata) < 0) + return -1; + *value_r = fs_metadata_find(metadata, key); + return *value_r != NULL ? 1 : 0; +} + +const char *fs_lookup_loaded_metadata(struct fs_file *file, const char *key) +{ + const ARRAY_TYPE(fs_metadata) *metadata; + + if (fs_get_metadata_full(file, FS_GET_METADATA_FLAG_LOADED_ONLY, &metadata) < 0) + i_panic("FS_GET_METADATA_FLAG_LOADED_ONLY lookup can't fail"); + return fs_metadata_find(metadata, key); +} + +const char *fs_file_path(struct fs_file *file) +{ + return file->fs->v.get_path == NULL ? file->path : + file->fs->v.get_path(file); +} + +struct fs *fs_file_fs(struct fs_file *file) +{ + return file->fs; +} + +struct event *fs_file_event(struct fs_file *file) +{ + return file->event; +} + +static struct fs_file *fs_file_get_error_file(struct fs_file *file) +{ + /* the error is always kept in the parentmost file */ + while (file->parent != NULL) + file = file->parent; + return file; +} + +static void ATTR_FORMAT(2, 0) +fs_set_verror(struct event *event, const char *fmt, va_list args) +{ + struct event *fs_event = event; + struct fs_file *file; + struct fs_iter *iter; + + /* NOTE: the event might be a passthrough event. We must log it exactly + once so it gets freed. */ + + /* figure out if the error is for a file or iter */ + while ((file = event_get_ptr(fs_event, FS_EVENT_FIELD_FILE)) == NULL && + (iter = event_get_ptr(fs_event, FS_EVENT_FIELD_ITER)) == NULL) { + fs_event = event_get_parent(fs_event); + i_assert(fs_event != NULL); + } + + char *new_error = i_strdup_vprintf(fmt, args); + /* Don't flood the debug log with "Asynchronous operation in progress" + messages. They tell nothing useful. */ + if (errno != EAGAIN) + e_debug(event, "%s", new_error); + else + event_send_abort(event); + + /* free old error after strdup in case args point to the old error */ + if (file != NULL) { + file = fs_file_get_error_file(file); + char *old_error = file->last_error; + + if (old_error == NULL) { + i_assert(!file->last_error_changed); + } else if (strcmp(old_error, new_error) == 0) { + /* identical error - ignore */ + } else if (file->last_error_changed) { + /* multiple fs_set_error() calls used without + fs_file_last_error() in the middle. */ + e_error(file->event, "%s (overwriting error for file %s)", + old_error, fs_file_path(file)); + } + if (errno == EAGAIN || errno == ENOENT || errno == EEXIST || + errno == ENOTEMPTY) { + /* These are (or can be) expected errors - don't log + them if they have a missing fs_file_last_error() + call */ + file->last_error_changed = FALSE; + } else { + file->last_error_changed = TRUE; + } + + i_free(file->last_error); + file->last_error = new_error; + } else { + i_assert(iter != NULL); + if (iter->last_error != NULL && + strcmp(iter->last_error, new_error) == 0) { + /* identical error - ignore */ + } else if (iter->last_error != NULL) { + /* multiple fs_set_error() calls before the iter + finishes */ + e_error(iter->fs->event, "%s (overwriting error for file %s)", + iter->last_error, iter->path); + } + i_free(iter->last_error); + iter->last_error = new_error; + } +} + +const char *fs_file_last_error(struct fs_file *file) +{ + struct fs_file *error_file = fs_file_get_error_file(file); + + error_file->last_error_changed = FALSE; + if (error_file->last_error == NULL) + return "BUG: Unknown file error"; + return error_file->last_error; +} + +bool fs_prefetch(struct fs_file *file, uoff_t length) +{ + bool ret; + + if (!file->read_or_prefetch_counted) { + file->read_or_prefetch_counted = TRUE; + file->fs->stats.prefetch_count++; + fs_file_timing_start(file, FS_OP_PREFETCH); + } + T_BEGIN { + ret = file->fs->v.prefetch(file, length); + } T_END; + fs_file_timing_end(file, FS_OP_PREFETCH); + return ret; +} + +ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size) +{ + const unsigned char *data; + size_t data_size; + ssize_t ret; + + i_assert(size > 0); + + if (file->pending_read_input == NULL) + file->pending_read_input = fs_read_stream(file, size+1); + ret = i_stream_read_bytes(file->pending_read_input, &data, + &data_size, size); + if (ret == 0) { + fs_file_set_error_async(file); + return -1; + } + if (ret < 0 && file->pending_read_input->stream_errno != 0) { + fs_set_error(file->event, + file->pending_read_input->stream_errno, + "read(%s) failed: %s", + i_stream_get_name(file->pending_read_input), + i_stream_get_error(file->pending_read_input)); + } else { + ret = I_MIN(size, data_size); + memcpy(buf, data, ret); + } + i_stream_unref(&file->pending_read_input); + return ret; +} + +ssize_t fs_read(struct fs_file *file, void *buf, size_t size) +{ + int ret; + + if (!file->read_or_prefetch_counted) { + file->read_or_prefetch_counted = TRUE; + file->fs->stats.read_count++; + fs_file_timing_start(file, FS_OP_READ); + } + + if (file->fs->v.read != NULL) { + T_BEGIN { + ret = file->fs->v.read(file, buf, size); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_READ); + return ret; + } + + /* backend didn't bother to implement read(), but we can do it with + streams. */ + return fs_read_via_stream(file, buf, size); +} + +static void fs_file_istream_destroyed(struct fs_file *file) +{ + i_assert(file->istream_open); + + file->istream_open = FALSE; +} + +struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size) +{ + struct istream *input, *inputs[2]; + const unsigned char *data; + size_t size; + ssize_t ret; + bool want_seekable = FALSE; + + if (!file->read_or_prefetch_counted) { + file->read_or_prefetch_counted = TRUE; + file->fs->stats.read_count++; + fs_file_timing_start(file, FS_OP_READ); + } + + if (file->seekable_input != NULL) { + /* allow multiple open streams, each in a different position */ + input = i_stream_create_limit(file->seekable_input, UOFF_T_MAX); + i_stream_seek(input, 0); + return input; + } + i_assert(!file->istream_open); + T_BEGIN { + input = file->fs->v.read_stream(file, max_buffer_size); + } T_END; + if (input->stream_errno != 0) { + /* read failed already */ + fs_file_timing_end(file, FS_OP_READ); + return input; + } + if (file->fs->set.enable_timing) { + struct istream *input2 = i_stream_create_fs_stats(input, file); + + i_stream_unref(&input); + input = input2; + } + + if ((file->flags & FS_OPEN_FLAG_SEEKABLE) != 0) + want_seekable = TRUE; + else if ((file->flags & FS_OPEN_FLAG_ASYNC) == 0 && !input->blocking) + want_seekable = TRUE; + + if (want_seekable && !input->seekable) { + /* need to make the stream seekable */ + inputs[0] = input; + inputs[1] = NULL; + input = i_stream_create_seekable_path(inputs, max_buffer_size, + file->fs->temp_path_prefix); + i_stream_set_name(input, i_stream_get_name(inputs[0])); + i_stream_unref(&inputs[0]); + } + file->seekable_input = input; + i_stream_ref(file->seekable_input); + + if ((file->flags & FS_OPEN_FLAG_ASYNC) == 0 && !input->blocking) { + /* read the whole input stream before returning */ + while ((ret = i_stream_read_more(input, &data, &size)) >= 0) { + i_stream_skip(input, size); + if (ret == 0) + fs_wait_async(file->fs); + } + i_stream_seek(input, 0); + } + file->istream_open = TRUE; + i_stream_add_destroy_callback(input, fs_file_istream_destroyed, file); + return input; +} + +int fs_write_via_stream(struct fs_file *file, const void *data, size_t size) +{ + struct ostream *output; + ssize_t ret; + int err; + + if (!file->write_pending) { + output = fs_write_stream(file); + if ((ret = o_stream_send(output, data, size)) < 0) { + err = errno; + fs_write_stream_abort_error(file, &output, "fs_write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); + errno = err; + return -1; + } + i_assert((size_t)ret == size); + ret = fs_write_stream_finish(file, &output); + } else { + ret = fs_write_stream_finish_async(file); + } + if (ret == 0) { + fs_file_set_error_async(file); + file->write_pending = TRUE; + return -1; + } + file->write_pending = FALSE; + return ret < 0 ? -1 : 0; +} + +int fs_write(struct fs_file *file, const void *data, size_t size) +{ + int ret; + + if (file->fs->v.write != NULL) { + fs_file_timing_start(file, FS_OP_WRITE); + T_BEGIN { + ret = file->fs->v.write(file, data, size); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) { + file->fs->stats.write_count++; + file->fs->stats.write_bytes += size; + fs_file_timing_end(file, FS_OP_WRITE); + } + return ret; + } + + /* backend didn't bother to implement write(), but we can do it with + streams. */ + return fs_write_via_stream(file, data, size); +} + +struct ostream *fs_write_stream(struct fs_file *file) +{ + i_assert(!file->writing_stream); + i_assert(file->output == NULL); + + file->writing_stream = TRUE; + file->fs->stats.write_count++; + T_BEGIN { + file->fs->v.write_stream(file); + } T_END; + i_assert(file->output != NULL); + o_stream_cork(file->output); + return file->output; +} + +static int fs_write_stream_finish_int(struct fs_file *file, bool success) +{ + int ret; + + i_assert(file->writing_stream); + + fs_file_timing_start(file, FS_OP_WRITE); + T_BEGIN { + ret = file->fs->v.write_stream_finish(file, success); + } T_END; + if (ret != 0) { + fs_file_timing_end(file, FS_OP_WRITE); + file->metadata_changed = FALSE; + } else { + /* write didn't finish yet. this shouldn't happen if we + indicated a failure. */ + i_assert(success); + } + if (ret != 0) { + i_assert(file->output == NULL); + file->writing_stream = FALSE; + } + return ret; +} + +int fs_write_stream_finish(struct fs_file *file, struct ostream **output) +{ + bool success = TRUE; + int ret; + + i_assert(*output == file->output || *output == NULL); + i_assert(output != &file->output); + + *output = NULL; + if (file->output != NULL) { + o_stream_uncork(file->output); + if ((ret = o_stream_finish(file->output)) <= 0) { + i_assert(ret < 0); + fs_set_error(file->event, file->output->stream_errno, + "write(%s) failed: %s", + o_stream_get_name(file->output), + o_stream_get_error(file->output)); + success = FALSE; + } + file->fs->stats.write_bytes += file->output->offset; + } + return fs_write_stream_finish_int(file, success); +} + +int fs_write_stream_finish_async(struct fs_file *file) +{ + return fs_write_stream_finish_int(file, TRUE); +} + +static void fs_write_stream_abort(struct fs_file *file, struct ostream **output) +{ + int ret; + + i_assert(*output == file->output); + i_assert(file->output != NULL); + i_assert(output != &file->output); + + *output = NULL; + o_stream_abort(file->output); + /* make sure we don't have an old error lying around */ + ret = fs_write_stream_finish_int(file, FALSE); + i_assert(ret != 0); +} + +void fs_write_stream_abort_error(struct fs_file *file, struct ostream **output, const char *error_fmt, ...) +{ + va_list args; + va_start(args, error_fmt); + fs_set_verror(file->event, error_fmt, args); + /* the error shouldn't be automatically logged if + fs_file_last_error() is no longer used */ + fs_file_get_error_file(file)->last_error_changed = FALSE; + fs_write_stream_abort(file, output); + va_end(args); +} + +void fs_write_stream_abort_parent(struct fs_file *file, struct ostream **output) +{ + i_assert(file->parent != NULL); + i_assert(fs_file_last_error(file->parent) != NULL); + fs_write_stream_abort(file->parent, output); +} + +void fs_write_set_hash(struct fs_file *file, const struct hash_method *method, + const void *digest) +{ + file->write_digest_method = method; + + i_free(file->write_digest); + file->write_digest = i_malloc(method->digest_size); + memcpy(file->write_digest, digest, method->digest_size); +} + +#undef fs_file_set_async_callback +void fs_file_set_async_callback(struct fs_file *file, + fs_file_async_callback_t *callback, + void *context) +{ + if (file->fs->v.set_async_callback != NULL) + file->fs->v.set_async_callback(file, callback, context); + else + callback(context); +} + +void fs_wait_async(struct fs *fs) +{ + /* recursion not allowed */ + i_assert(fs->prev_ioloop == NULL); + + if (fs->v.wait_async != NULL) T_BEGIN { + fs->prev_ioloop = current_ioloop; + fs->v.wait_async(fs); + i_assert(current_ioloop == fs->prev_ioloop); + fs->prev_ioloop = NULL; + } T_END; +} + +bool fs_switch_ioloop(struct fs *fs) +{ + bool ret = FALSE; + + if (fs->v.switch_ioloop != NULL) { + T_BEGIN { + ret = fs->v.switch_ioloop(fs); + } T_END; + } else if (fs->parent != NULL) { + ret = fs_switch_ioloop(fs->parent); + } + return ret; +} + +int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r) +{ + int ret; + + T_BEGIN { + ret = file->fs->v.lock(file, secs, lock_r); + } T_END; + return ret; +} + +void fs_unlock(struct fs_lock **_lock) +{ + struct fs_lock *lock = *_lock; + + if (lock == NULL) + return; + + *_lock = NULL; + T_BEGIN { + lock->file->fs->v.unlock(lock); + } T_END; +} + +int fs_exists(struct fs_file *file) +{ + struct stat st; + int ret; + + if (file->fs->v.exists == NULL) { + /* fallback to stat() */ + if (fs_stat(file, &st) == 0) + return 1; + else + return errno == ENOENT ? 0 : -1; + } + fs_file_timing_start(file, FS_OP_EXISTS); + T_BEGIN { + ret = file->fs->v.exists(file); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) { + file->fs->stats.exists_count++; + fs_file_timing_end(file, FS_OP_EXISTS); + } + return ret; +} + +int fs_stat(struct fs_file *file, struct stat *st_r) +{ + int ret; + + if (file->fs->v.stat == NULL) { + fs_set_error(file->event, ENOTSUP, "fs_stat() not supported"); + return -1; + } + + if (!file->read_or_prefetch_counted && + !file->lookup_metadata_counted && !file->stat_counted) { + file->stat_counted = TRUE; + file->fs->stats.stat_count++; + fs_file_timing_start(file, FS_OP_STAT); + } + T_BEGIN { + ret = file->fs->v.stat(file, st_r); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_STAT); + return ret; +} + +int fs_get_nlinks(struct fs_file *file, nlink_t *nlinks_r) +{ + int ret; + + if (file->fs->v.get_nlinks == NULL) { + struct stat st; + + if (fs_stat(file, &st) < 0) + return -1; + *nlinks_r = st.st_nlink; + return 0; + } + + if (!file->read_or_prefetch_counted && + !file->lookup_metadata_counted && !file->stat_counted) { + file->stat_counted = TRUE; + file->fs->stats.stat_count++; + fs_file_timing_start(file, FS_OP_STAT); + } + T_BEGIN { + ret = file->fs->v.get_nlinks(file, nlinks_r); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_STAT); + return ret; +} + +int fs_default_copy(struct fs_file *src, struct fs_file *dest) +{ + int tmp_errno; + /* we're going to be counting this as read+write, so don't update + copy_count */ + dest->copy_counted = TRUE; + + if (dest->copy_src != NULL) { + i_assert(src == NULL || src == dest->copy_src); + if (dest->copy_output == NULL) { + i_assert(dest->copy_input == NULL); + if (fs_write_stream_finish_async(dest) <= 0) + return -1; + dest->copy_src = NULL; + return 0; + } + } else { + dest->copy_src = src; + dest->copy_input = fs_read_stream(src, IO_BLOCK_SIZE); + dest->copy_output = fs_write_stream(dest); + } + switch (o_stream_send_istream(dest->copy_output, dest->copy_input)) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + break; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + fs_file_set_error_async(dest); + return -1; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + fs_write_stream_abort_error(dest, &dest->copy_output, + "read(%s) failed: %s", + i_stream_get_name(dest->copy_input), + i_stream_get_error(dest->copy_input)); + errno = dest->copy_input->stream_errno; + i_stream_unref(&dest->copy_input); + return -1; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + /* errno might not survive abort error */ + tmp_errno = dest->copy_output->stream_errno; + fs_write_stream_abort_error(dest, &dest->copy_output, + "write(%s) failed: %s", + o_stream_get_name(dest->copy_output), + o_stream_get_error(dest->copy_output)); + errno = tmp_errno; + i_stream_unref(&dest->copy_input); + return -1; + } + i_stream_unref(&dest->copy_input); + if (fs_write_stream_finish(dest, &dest->copy_output) <= 0) + return -1; + dest->copy_src = NULL; + return 0; +} + +int fs_copy(struct fs_file *src, struct fs_file *dest) +{ + int ret; + + i_assert(src->fs == dest->fs); + + if (src->fs->v.copy == NULL) { + fs_set_error(src->event, ENOTSUP, "fs_copy() not supported"); + return -1; + } + + fs_file_timing_start(dest, FS_OP_COPY); + T_BEGIN { + ret = src->fs->v.copy(src, dest); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) { + fs_file_timing_end(dest, FS_OP_COPY); + if (dest->copy_counted) + dest->copy_counted = FALSE; + else + dest->fs->stats.copy_count++; + dest->metadata_changed = FALSE; + } + return ret; +} + +int fs_copy_finish_async(struct fs_file *dest) +{ + int ret; + + T_BEGIN { + ret = dest->fs->v.copy(NULL, dest); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) { + fs_file_timing_end(dest, FS_OP_COPY); + if (dest->copy_counted) + dest->copy_counted = FALSE; + else + dest->fs->stats.copy_count++; + dest->metadata_changed = FALSE; + } + return ret; +} + +int fs_rename(struct fs_file *src, struct fs_file *dest) +{ + int ret; + + i_assert(src->fs == dest->fs); + + fs_file_timing_start(dest, FS_OP_RENAME); + T_BEGIN { + ret = src->fs->v.rename(src, dest); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) { + dest->fs->stats.rename_count++; + fs_file_timing_end(dest, FS_OP_RENAME); + } + return ret; +} + +int fs_delete(struct fs_file *file) +{ + int ret; + + i_assert(!file->writing_stream); + + fs_file_timing_start(file, FS_OP_DELETE); + T_BEGIN { + ret = file->fs->v.delete_file(file); + } T_END; + if (!(ret < 0 && errno == EAGAIN)) { + file->fs->stats.delete_count++; + fs_file_timing_end(file, FS_OP_DELETE); + } + return ret; +} + +struct fs_iter * +fs_iter_init(struct fs *fs, const char *path, enum fs_iter_flags flags) +{ + return fs_iter_init_with_event(fs, fs->event, path, flags); +} + +struct fs_iter * +fs_iter_init_with_event(struct fs *fs, struct event *event, + const char *path, enum fs_iter_flags flags) +{ + struct fs_iter *iter; + struct timeval now = ioloop_timeval; + + i_assert((flags & FS_ITER_FLAG_OBJECTIDS) == 0 || + (fs_get_properties(fs) & FS_PROPERTY_OBJECTIDS) != 0); + + fs->stats.iter_count++; + if (fs->set.enable_timing) + i_gettimeofday(&now); + if (fs->v.iter_init == NULL) { + iter = i_new(struct fs_iter, 1); + iter->fs = fs; + } else T_BEGIN { + iter = fs->v.iter_alloc(); + iter->fs = fs; + iter->flags = flags; + iter->path = i_strdup(path); + iter->event = fs_create_event(fs, event); + event_set_ptr(iter->event, FS_EVENT_FIELD_FS, fs); + event_set_ptr(iter->event, FS_EVENT_FIELD_ITER, iter); + fs->v.iter_init(iter, path, flags); + } T_END; + iter->start_time = now; + DLLIST_PREPEND(&fs->iters, iter); + return iter; +} + +int fs_iter_deinit(struct fs_iter **_iter, const char **error_r) +{ + struct fs_iter *iter = *_iter; + struct fs *fs; + struct event *event; + int ret; + + if (iter == NULL) + return 0; + + fs = iter->fs; + event = iter->event; + + *_iter = NULL; + DLLIST_REMOVE(&fs->iters, iter); + + if (fs->v.iter_deinit == NULL) { + fs_set_error(event, ENOTSUP, "FS iteration not supported"); + ret = -1; + } else T_BEGIN { + ret = iter->fs->v.iter_deinit(iter); + } T_END; + if (ret < 0) + *error_r = t_strdup(iter->last_error); + i_free(iter->last_error); + i_free(iter->path); + i_free(iter); + event_unref(&event); + return ret; +} + +const char *fs_iter_next(struct fs_iter *iter) +{ + const char *ret; + + if (iter->fs->v.iter_next == NULL) + return NULL; + T_BEGIN { + ret = iter->fs->v.iter_next(iter); + } T_END; + if (iter->start_time.tv_sec != 0 && + (ret != NULL || !fs_iter_have_more(iter))) { + /* first result returned - count this as the finish time, since + we don't want to count the time caller spends on this + iteration. */ + fs_timing_end(&iter->fs->stats.timings[FS_OP_ITER], &iter->start_time); + /* don't count this again */ + iter->start_time.tv_sec = 0; + } + return ret; +} + +#undef fs_iter_set_async_callback +void fs_iter_set_async_callback(struct fs_iter *iter, + fs_file_async_callback_t *callback, + void *context) +{ + iter->async_callback = callback; + iter->async_context = context; +} + +bool fs_iter_have_more(struct fs_iter *iter) +{ + return iter->async_have_more; +} + +const struct fs_stats *fs_get_stats(struct fs *fs) +{ + return &fs->stats; +} + +void fs_set_error(struct event *event, int err, const char *fmt, ...) +{ + va_list args; + + i_assert(err != 0); + + errno = err; + va_start(args, fmt); + fs_set_verror(event, fmt, args); + va_end(args); +} + +void fs_set_error_errno(struct event *event, const char *fmt, ...) +{ + va_list args; + + i_assert(errno != 0); + + va_start(args, fmt); + fs_set_verror(event, fmt, args); + va_end(args); +} + +void fs_file_set_error_async(struct fs_file *file) +{ + fs_set_error(file->event, EAGAIN, "Asynchronous operation in progress"); +} + +static uint64_t +fs_stats_count_ops(const struct fs_stats *stats, const enum fs_op ops[], + unsigned int ops_count) +{ + uint64_t ret = 0; + + for (unsigned int i = 0; i < ops_count; i++) { + if (stats->timings[ops[i]] != NULL) + ret += stats_dist_get_sum(stats->timings[ops[i]]); + } + return ret; +} + +uint64_t fs_stats_get_read_usecs(const struct fs_stats *stats) +{ + const enum fs_op read_ops[] = { + FS_OP_METADATA, FS_OP_PREFETCH, FS_OP_READ, FS_OP_EXISTS, + FS_OP_STAT, FS_OP_ITER + }; + return fs_stats_count_ops(stats, read_ops, N_ELEMENTS(read_ops)); +} + +uint64_t fs_stats_get_write_usecs(const struct fs_stats *stats) +{ + const enum fs_op write_ops[] = { + FS_OP_WRITE, FS_OP_COPY, FS_OP_DELETE + }; + return fs_stats_count_ops(stats, write_ops, N_ELEMENTS(write_ops)); +} + +struct fs_file * +fs_file_init_parent(struct fs_file *parent, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + return fs_file_init_with_event(parent->fs->parent, parent->event, + path, (int)mode | (int)flags); +} + +struct fs_iter * +fs_iter_init_parent(struct fs_iter *parent, + const char *path, enum fs_iter_flags flags) +{ + return fs_iter_init_with_event(parent->fs->parent, parent->event, + path, flags); +} + +struct event *fs_get_event(struct fs *fs) +{ + return fs->event; +} diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h new file mode 100644 index 0000000..f69cc79 --- /dev/null +++ b/src/lib-fs/fs-api.h @@ -0,0 +1,399 @@ +#ifndef FS_API_H +#define FS_API_H + +struct stat; +struct fs; +struct fs_file; +struct fs_lock; +struct hash_method; + +/* Metadata with this prefix shouldn't actually be sent to storage. */ +#define FS_METADATA_INTERNAL_PREFIX ":/X-Dovecot-fs-api-" +/* fs_write*() may return a hex-encoded object ID after write is finished. + This can be later on used to optimize reads by setting it before reading + the file. */ +#define FS_METADATA_OBJECTID FS_METADATA_INTERNAL_PREFIX"ObjectID" +/* Calling this before fs_write_stream_finish() allows renaming the filename. + This can be useful if you don't know the final filename before writing it + (e.g. filename contains the file size). The given filename must include the + full path also. */ +#define FS_METADATA_WRITE_FNAME FS_METADATA_INTERNAL_PREFIX"WriteFilename" +/* Original path of the file. The path that's eventually visible to a fs + backend may be something different, e.g. object ID. This allows the backend + to still access the original path. */ +#define FS_METADATA_ORIG_PATH FS_METADATA_INTERNAL_PREFIX"OrigPath" + +enum fs_properties { + FS_PROPERTY_METADATA = 0x01, + FS_PROPERTY_LOCKS = 0x02, + FS_PROPERTY_FASTCOPY = 0x04, + FS_PROPERTY_RENAME = 0x08, + FS_PROPERTY_STAT = 0x10, + /* Iteration is possible */ + FS_PROPERTY_ITER = 0x20, + /* Iteration always returns all of the files (instead of possibly + slightly out of date view) */ + FS_PROPERTY_RELIABLEITER= 0x40, + /* Backend uses directories, which aren't automatically deleted + when its children are deleted. */ + FS_PROPERTY_DIRECTORIES = 0x80, + FS_PROPERTY_WRITE_HASH_MD5 = 0x100, + FS_PROPERTY_WRITE_HASH_SHA256 = 0x200, + /* fs_copy() will copy the metadata if fs_set_metadata() hasn't + been explicitly called. */ + FS_PROPERTY_COPY_METADATA = 0x400, + /* Backend support asynchronous file operations. */ + FS_PROPERTY_ASYNC = 0x800, + /* Backend supports FS_ITER_FLAG_OBJECTIDS. */ + FS_PROPERTY_OBJECTIDS = 0x1000, + /* fs_copy() is fast even when file's metadata is changed */ + FS_PROPERTY_FASTCOPY_CHANGED_METADATA = 0x2000, +}; + +enum fs_open_mode { + /* Open only for reading, or fail with ENOENT if it doesn't exist */ + FS_OPEN_MODE_READONLY, + /* Create a new file, fail with EEXIST if it already exists */ + FS_OPEN_MODE_CREATE, + /* Create a new file with a new unique name. The generated name is a + 128bit hex-encoded string. The fs_open()'s path parameter specifies + only the directory where the file is created to. */ + FS_OPEN_MODE_CREATE_UNIQUE_128, + /* Create or replace a file */ + FS_OPEN_MODE_REPLACE, + /* Append to existing file, fail with ENOENT if it doesn't exist */ + FS_OPEN_MODE_APPEND + +#define FS_OPEN_MODE_MASK 0x0f +}; + +enum fs_open_flags { + /* File is important and writing must call fsync() or have equivalent + behavior. */ + FS_OPEN_FLAG_FSYNC = 0x10, + /* Asynchronous writes: fs_write() will fail with EAGAIN if it needs to + be called again (the retries can use size=0). For streams + fs_write_stream_finish() may request retrying with 0. + + Asynchronous reads: fs_read() will fail with EAGAIN if it's not + finished and fs_read_stream() returns a nonblocking stream. */ + FS_OPEN_FLAG_ASYNC = 0x20, + /* fs_read_stream() must return a seekable input stream */ + FS_OPEN_FLAG_SEEKABLE = 0x40, + /* Backend should handle this file's operations immediately without + any additional command queueing. The caller is assumed to be the one + doing any rate limiting if needed. This flag can only be used with + ASYNC flag, synchronous requests are never queued. */ + FS_OPEN_FLAG_ASYNC_NOQUEUE = 0x80 +}; + +enum fs_iter_flags { + /* Iterate only directories, not files */ + FS_ITER_FLAG_DIRS = 0x01, + /* Request asynchronous iteration. */ + FS_ITER_FLAG_ASYNC = 0x02, + /* Instead of returning object names, return <objectid>/<object name>. + If this isn't supported, the <objectid> is returned empty. The + object IDs are always hex-encoded data. This flag can be used only + if FS_PROPERTY_OBJECTIDS is enabled. */ + FS_ITER_FLAG_OBJECTIDS = 0x04, + /* Explicitly disable all caching for this iteration (if anything + happens to be enabled). This should be used only in situations where + the iteration is used to fix something that is broken, e.g. doveadm + force-resync. */ + FS_ITER_FLAG_NOCACHE = 0x08 +}; + +enum fs_op { + FS_OP_WAIT, + FS_OP_METADATA, + FS_OP_PREFETCH, + FS_OP_READ, + FS_OP_WRITE, + FS_OP_LOCK, + FS_OP_EXISTS, + FS_OP_STAT, + FS_OP_COPY, + FS_OP_RENAME, + FS_OP_DELETE, + FS_OP_ITER, + + FS_OP_COUNT +}; + +struct fs_settings { + /* Username and session ID are mainly used for debugging/logging, + but may also be useful for other purposes if they exist (they + may be NULL). */ + const char *username; + const char *session_id; + + /* Dovecot instance's base_dir */ + const char *base_dir; + /* Directory where temporary files can be created at any time + (e.g. /tmp or mail_temp_dir) */ + const char *temp_dir; + /* SSL client settings. */ + const struct ssl_iostream_settings *ssl_client_set; + + /* Automatically try to rmdir() directories up to this path when + deleting files. */ + const char *root_path; + /* When creating temporary files, use this prefix + (to avoid conflicts with existing files). */ + const char *temp_file_prefix; + /* If the backend needs to do DNS lookups, use this dns_client for + them. */ + struct dns_client *dns_client; + + /* Parent event to use, unless overridden by + fs_file_init_with_event() */ + struct event *event_parent; + + /* Enable debugging */ + bool debug; + /* Enable timing statistics */ + bool enable_timing; +}; + +struct fs_stats { + /* Number of fs_prefetch() calls. Counted only if fs_read*() hasn't + already been called for the file (which would be pretty pointless + to do). */ + unsigned int prefetch_count; + /* Number of fs_read*() calls. Counted only if fs_prefetch() hasn't + already been called for the file. */ + unsigned int read_count; + /* Number of fs_lookup_metadata() calls. Counted only if neither + fs_read*() nor fs_prefetch() has been called for the file. */ + unsigned int lookup_metadata_count; + /* Number of fs_stat() calls. Counted only if none of the above + has been called (because the stat result should be cached). */ + unsigned int stat_count; + + /* Number of fs_write*() calls. */ + unsigned int write_count; + /* Number of fs_exists() calls, which actually went to the backend + instead of being handled by fs_stat() call due to fs_exists() not + being implemented. */ + unsigned int exists_count; + /* Number of fs_delete() calls. */ + unsigned int delete_count; + /* Number of fs_copy() calls. If backend doesn't implement copying + operation but falls back to regular read+write instead, this count + isn't increased but the read+write counters are. */ + unsigned int copy_count; + /* Number of fs_rename() calls. */ + unsigned int rename_count; + /* Number of fs_iter_init() calls. */ + unsigned int iter_count; + + /* Number of bytes written by fs_write*() calls. */ + uint64_t write_bytes; + + /* Cumulative sum of usecs spent on calls - set only if + fs_settings.enable_timing=TRUE */ + struct stats_dist *timings[FS_OP_COUNT]; +}; + +struct fs_metadata { + const char *key; + const char *value; +}; +ARRAY_DEFINE_TYPE(fs_metadata, struct fs_metadata); + +typedef void fs_file_async_callback_t(void *context); + +int fs_init(const char *driver, const char *args, + const struct fs_settings *set, + struct fs **fs_r, const char **error_r); +/* helper for fs_init, accepts a filesystem string + that can come directly from config */ +int fs_init_from_string(const char *str, const struct fs_settings *set, + struct fs **fs_r, const char **error_r); +/* same as fs_unref() */ +void fs_deinit(struct fs **fs); + +void fs_ref(struct fs *fs); +void fs_unref(struct fs **fs); + +/* Returns the parent filesystem (if this is a wrapper fs) or NULL if + there's no parent. */ +struct fs *fs_get_parent(struct fs *fs); +/* Returns the filesystem's driver name. */ +const char *fs_get_driver(struct fs *fs); +/* Returns the root fs's driver name (bypassing all wrapper fses) */ +const char *fs_get_root_driver(struct fs *fs); +/* Returns the fs's event. */ +struct event *fs_get_event(struct fs *fs); + +struct fs_file *fs_file_init(struct fs *fs, const char *path, int mode_flags); +struct fs_file *fs_file_init_with_event(struct fs *fs, struct event *event, + const char *path, int mode_flags); +void fs_file_deinit(struct fs_file **file); + +/* Change flags for a file (and its parents). */ +void fs_file_set_flags(struct fs_file *file, + enum fs_open_flags add_flags, + enum fs_open_flags remove_flags); +/* If the file has an input streams open, close them. */ +void fs_file_close(struct fs_file *file); + +/* Return properties supported by backend. */ +enum fs_properties fs_get_properties(struct fs *fs); + +/* Add/replace metadata when saving a file. This makes sense only when the + file is being created/replaced. */ +void fs_set_metadata(struct fs_file *file, const char *key, const char *value); +/* Return file's all metadata. */ +int fs_get_metadata(struct fs_file *file, + const ARRAY_TYPE(fs_metadata) **metadata_r); +/* Wrapper to fs_get_metadata() to lookup a specific key. Returns 1 if value_r + is set, 0 if key wasn't found, -1 if error. */ +int fs_lookup_metadata(struct fs_file *file, const char *key, + const char **value_r); +/* Try to find key from the currently set metadata (without refreshing it). + This is typically used e.g. after writing or copying a file to find some + extra metadata they may have set. It can also be used after + fs_get_metadata() or fs_lookup_metadata() to search within the looked up + metadata. */ +const char *fs_lookup_loaded_metadata(struct fs_file *file, const char *key); + +/* Returns the path given to fs_open(). If file was opened with + FS_OPEN_MODE_CREATE_UNIQUE_128 and the write has already finished, + return the path including the generated filename. */ +const char *fs_file_path(struct fs_file *file); +/* Returns the file's fs. */ +struct fs *fs_file_fs(struct fs_file *file); +/* Returns the file's event. */ +struct event *fs_file_event(struct fs_file *file); + +/* Return the error message for the last failed file operation. Each file + keeps track of its own errors. For failed copy/rename operations the "dest" + file contains the error. */ +const char *fs_file_last_error(struct fs_file *file); + +/* Try to asynchronously prefetch file into memory. Returns TRUE if file is + already in memory (i.e. caller should handle this file before prefetching + more), FALSE if not. The length is a hint of how much the caller expects + to read, but it may be more or less (0=whole file). */ +bool fs_prefetch(struct fs_file *file, uoff_t length); +/* Returns >0 if something was read, -1 if error (errno is set). */ +ssize_t fs_read(struct fs_file *file, void *buf, size_t size); +/* Returns a stream for reading from file. Multiple streams can be opened, + and caller must destroy the streams before closing the file. */ +struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size); + +/* Returns 0 if ok, -1 if error (errno is set). Note: With CREATE/REPLACE mode + files you can call fs_write() only once, the file creation is finished by it. + CREATE can return EEXIST here, if the destination file was already created. + With APPEND mode each fs_write() atomically appends the given data to + file. */ +int fs_write(struct fs_file *file, const void *data, size_t size); + +/* Write to file via output stream. The stream will be destroyed by + fs_write_stream_finish/abort. The returned ostream is already corked and + it doesn't need to be uncorked. */ +struct ostream *fs_write_stream(struct fs_file *file); +/* Finish writing via stream, calling also o_stream_flush() on the stream and + handling any pending errors. The file will be created/replaced/appended only + after this call, same as with fs_write(). Anything written to the stream + won't be visible earlier. Returns 1 if ok, 0 if async write isn't finished + yet (retry calling fs_write_stream_finish_async()), -1 if error */ +int fs_write_stream_finish(struct fs_file *file, struct ostream **output); +int fs_write_stream_finish_async(struct fs_file *file); +/* Abort writing via stream. Anything written to the stream is discarded. + o_stream_ignore_last_errors() is called on the output stream so the caller + doesn't need to do it. This must not be called after + fs_write_stream_finish(), i.e. it can't be used to abort a pending async + write. */ +void fs_write_stream_abort_error(struct fs_file *file, struct ostream **output, const char *error_fmt, ...) ATTR_FORMAT(3, 4); + +/* Set a hash to the following write. The storage can then verify that the + input data matches the specified hash, or fail if it doesn't. Typically + implemented by Content-MD5 header. */ +void fs_write_set_hash(struct fs_file *file, const struct hash_method *method, + const void *digest); + +/* Call the specified callback whenever the file can be read/written to. + May call the callback immediately. */ +void fs_file_set_async_callback(struct fs_file *file, + fs_file_async_callback_t *callback, + void *context); +#define fs_file_set_async_callback(file, callback, context) \ + fs_file_set_async_callback(file, (fs_file_async_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) +/* Wait until some file can be read/written to more before returning. + It's an error to call this when there are no pending async operations. */ +void fs_wait_async(struct fs *fs); +/* Switch the fs to the current ioloop. This can be used to do fs_wait_async() + among other IO work. Returns TRUE if there is actually some work that can + be waited on. */ +bool fs_switch_ioloop(struct fs *fs) ATTR_NOWARN_UNUSED_RESULT; + +/* Returns 1 if file exists, 0 if not, -1 if error occurred. */ +int fs_exists(struct fs_file *file); +/* Delete a file. Returns 0 if file was actually deleted by us, -1 if error. */ +int fs_delete(struct fs_file *file); + +/* Returns 0 if ok, -1 if error occurred (e.g. errno=ENOENT). + All fs backends may not support all stat fields. */ +int fs_stat(struct fs_file *file, struct stat *st_r); +/* Get number of links to the file. This is the same as using fs_stat()'s + st_nlinks field, except not all backends support returning it via fs_stat(). + Returns 0 if ok, -1 if error occurred. */ +int fs_get_nlinks(struct fs_file *file, nlink_t *nlinks_r); +/* Copy an object with possibly updated metadata. Destination parent + directories are created automatically. Returns 0 if ok, -1 if error + occurred. The "dest" file contains the error. */ +int fs_copy(struct fs_file *src, struct fs_file *dest); +/* Try to finish asynchronous fs_copy(). Returns the same as fs_copy(). */ +int fs_copy_finish_async(struct fs_file *dest); +/* Atomically rename a file. Destination parent directories are created + automatically. Returns 0 if ok, -1 if error occurred. The "dest" file + contains the error. */ +int fs_rename(struct fs_file *src, struct fs_file *dest); + +/* Exclusively lock a file. If file is already locked, wait for it for given + number of seconds (0 = fail immediately). Returns 1 if locked, 0 if wait + timed out, -1 if error. */ +int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r); +void fs_unlock(struct fs_lock **lock); + +/* Iterate through all files or directories in the given directory. + Doesn't recurse to child directories. It's not an error to iterate a + nonexistent directory. */ +struct fs_iter * +fs_iter_init(struct fs *fs, const char *path, enum fs_iter_flags flags); +struct fs_iter * +fs_iter_init_with_event(struct fs *fs, struct event *event, + const char *path, enum fs_iter_flags flags); +/* Returns 0 if ok, -1 if iteration failed. */ +int fs_iter_deinit(struct fs_iter **iter, const char **error_r); +/* Returns the next filename. */ +const char *fs_iter_next(struct fs_iter *iter); + +/* For asynchronous iterations: Specify the callback that is called whenever + there's more data available for reading. */ +void fs_iter_set_async_callback(struct fs_iter *iter, + fs_file_async_callback_t *callback, + void *context); +#define fs_iter_set_async_callback(iter, callback, context) \ + fs_iter_set_async_callback(iter, (fs_file_async_callback_t *)(callback), \ + 1 ? (context) : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) +/* For asynchronous iterations: If fs_iter_next() returns NULL, use this + function to determine if you should wait for more data or finish up. */ +bool fs_iter_have_more(struct fs_iter *iter); + +/* Return the filesystem's fs_stats. Note that each wrapper filesystem keeps + track of its own fs_stats calls. You can use fs_get_parent() to get to the + filesystem whose stats you want to see. */ +const struct fs_stats *fs_get_stats(struct fs *fs); + +/* Helper functions to count number of usecs for read/write operations. */ +uint64_t fs_stats_get_read_usecs(const struct fs_stats *stats); +uint64_t fs_stats_get_write_usecs(const struct fs_stats *stats); + +#endif diff --git a/src/lib-fs/fs-dict.c b/src/lib-fs/fs-dict.c new file mode 100644 index 0000000..936e9b6 --- /dev/null +++ b/src/lib-fs/fs-dict.c @@ -0,0 +1,372 @@ +/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "guid.h" +#include "hex-binary.h" +#include "base64.h" +#include "istream.h" +#include "ostream.h" +#include "dict.h" +#include "fs-api-private.h" + +enum fs_dict_value_encoding { + FS_DICT_VALUE_ENCODING_RAW, + FS_DICT_VALUE_ENCODING_HEX, + FS_DICT_VALUE_ENCODING_BASE64 +}; + +struct dict_fs { + struct fs fs; + struct dict *dict; + char *path_prefix; + enum fs_dict_value_encoding encoding; +}; + +struct dict_fs_file { + struct fs_file file; + pool_t pool; + const char *key, *value; + buffer_t *write_buffer; +}; + +struct dict_fs_iter { + struct fs_iter iter; + struct dict_iterate_context *dict_iter; +}; + +static struct fs *fs_dict_alloc(void) +{ + struct dict_fs *fs; + + fs = i_new(struct dict_fs, 1); + fs->fs = fs_class_dict; + return &fs->fs; +} + +static int +fs_dict_init(struct fs *_fs, const char *args, const struct fs_settings *set, + const char **error_r) +{ + struct dict_fs *fs = (struct dict_fs *)_fs; + struct dict_settings dict_set; + const char *p, *encoding_str, *error; + + p = strchr(args, ':'); + if (p == NULL) { + *error_r = "':' missing in args"; + return -1; + } + encoding_str = t_strdup_until(args, p++); + if (strcmp(encoding_str, "raw") == 0) + fs->encoding = FS_DICT_VALUE_ENCODING_RAW; + else if (strcmp(encoding_str, "hex") == 0) + fs->encoding = FS_DICT_VALUE_ENCODING_HEX; + else if (strcmp(encoding_str, "base64") == 0) + fs->encoding = FS_DICT_VALUE_ENCODING_BASE64; + else { + *error_r = t_strdup_printf("Unknown value encoding '%s'", + encoding_str); + return -1; + } + + i_zero(&dict_set); + dict_set.base_dir = set->base_dir; + dict_set.event_parent = set->event_parent; + + if (dict_init(p, &dict_set, &fs->dict, &error) < 0) { + *error_r = t_strdup_printf("dict_init(%s) failed: %s", + args, error); + return -1; + } + return 0; +} + +static void fs_dict_free(struct fs *_fs) +{ + struct dict_fs *fs = (struct dict_fs *)_fs; + + if (fs->dict != NULL) dict_deinit(&fs->dict); + i_free(fs); +} + +static enum fs_properties fs_dict_get_properties(struct fs *fs ATTR_UNUSED) +{ + return FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER; +} + +static struct fs_file *fs_dict_file_alloc(void) +{ + struct dict_fs_file *file; + pool_t pool; + + pool = pool_alloconly_create("fs dict file", 128); + file = p_new(pool, struct dict_fs_file, 1); + file->pool = pool; + return &file->file; +} + +static void +fs_dict_file_init(struct fs_file *_file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags ATTR_UNUSED) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + struct dict_fs *fs = (struct dict_fs *)_file->fs; + guid_128_t guid; + + i_assert(mode != FS_OPEN_MODE_APPEND); /* not supported */ + i_assert(mode != FS_OPEN_MODE_CREATE); /* not supported */ + + if (mode != FS_OPEN_MODE_CREATE_UNIQUE_128) + file->file.path = p_strdup(file->pool, path); + else { + guid_128_generate(guid); + file->file.path = p_strdup_printf(file->pool, "%s/%s", path, + guid_128_to_string(guid)); + } + file->key = fs->path_prefix == NULL ? + p_strdup(file->pool, file->file.path) : + p_strconcat(file->pool, fs->path_prefix, file->file.path, NULL); +} + +static void fs_dict_file_deinit(struct fs_file *_file) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + + i_assert(_file->output == NULL); + + fs_file_free(_file); + pool_unref(&file->pool); +} + +static bool fs_dict_prefetch(struct fs_file *_file ATTR_UNUSED, + uoff_t length ATTR_UNUSED) +{ + /* once async dict_lookup() is implemented, we want to start it here */ + return TRUE; +} + +static int fs_dict_lookup(struct dict_fs_file *file) +{ + struct dict_fs *fs = (struct dict_fs *)file->file.fs; + const char *error; + int ret; + + if (file->value != NULL) + return 0; + + struct dict_op_settings set = { + .username = file->file.fs->username, + }; + ret = dict_lookup(fs->dict, &set, file->pool, file->key, &file->value, &error); + if (ret > 0) + return 0; + else if (ret < 0) { + fs_set_error(file->file.event, EIO, + "dict_lookup(%s) failed: %s", file->key, error); + return -1; + } else { + fs_set_error(file->file.event, ENOENT, + "Dict key %s doesn't exist", file->key); + return -1; + } +} + +static struct istream * +fs_dict_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + struct istream *input; + + if (fs_dict_lookup(file) < 0) + input = i_stream_create_error_str(errno, "%s", fs_file_last_error(_file)); + else + input = i_stream_create_from_data(file->value, strlen(file->value)); + i_stream_set_name(input, file->key); + return input; +} + +static void fs_dict_write_stream(struct fs_file *_file) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + + i_assert(_file->output == NULL); + + file->write_buffer = buffer_create_dynamic(file->pool, 128); + _file->output = o_stream_create_buffer(file->write_buffer); + o_stream_set_name(_file->output, file->key); +} + +static void fs_dict_write_rename_if_needed(struct dict_fs_file *file) +{ + struct dict_fs *fs = (struct dict_fs *)file->file.fs; + const char *new_fname; + + new_fname = fs_metadata_find(&file->file.metadata, FS_METADATA_WRITE_FNAME); + if (new_fname == NULL) + return; + + file->file.path = p_strdup(file->pool, new_fname); + file->key = fs->path_prefix == NULL ? p_strdup(file->pool, new_fname) : + p_strconcat(file->pool, fs->path_prefix, new_fname, NULL); +} + +static int fs_dict_write_stream_finish(struct fs_file *_file, bool success) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + struct dict_fs *fs = (struct dict_fs *)_file->fs; + struct dict_transaction_context *trans; + const char *error; + + o_stream_destroy(&_file->output); + if (!success) + return -1; + + struct dict_op_settings set = { + .username = _file->fs->username, + }; + fs_dict_write_rename_if_needed(file); + trans = dict_transaction_begin(fs->dict, &set); + switch (fs->encoding) { + case FS_DICT_VALUE_ENCODING_RAW: + dict_set(trans, file->key, str_c(file->write_buffer)); + break; + case FS_DICT_VALUE_ENCODING_HEX: { + string_t *hex = t_str_new(file->write_buffer->used * 2 + 1); + binary_to_hex_append(hex, file->write_buffer->data, + file->write_buffer->used); + dict_set(trans, file->key, str_c(hex)); + break; + } + case FS_DICT_VALUE_ENCODING_BASE64: { + const size_t base64_size = + MAX_BASE64_ENCODED_SIZE(file->write_buffer->used); + string_t *base64 = t_str_new(base64_size); + base64_encode(file->write_buffer->data, + file->write_buffer->used, base64); + dict_set(trans, file->key, str_c(base64)); + } + } + if (dict_transaction_commit(&trans, &error) < 0) { + fs_set_error(_file->event, EIO, + "Dict transaction commit failed: %s", error); + return -1; + } + return 1; +} + +static int fs_dict_stat(struct fs_file *_file, struct stat *st_r) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + + i_zero(st_r); + + if (fs_dict_lookup(file) < 0) + return -1; + st_r->st_size = strlen(file->value); + return 0; +} + +static int fs_dict_delete(struct fs_file *_file) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + struct dict_fs *fs = (struct dict_fs *)_file->fs; + struct dict_transaction_context *trans; + const char *error; + + struct dict_op_settings set = { + .username = fs->fs.username, + }; + trans = dict_transaction_begin(fs->dict, &set); + dict_unset(trans, file->key); + if (dict_transaction_commit(&trans, &error) < 0) { + fs_set_error(_file->event, EIO, + "Dict transaction commit failed: %s", error); + return -1; + } + return 0; +} + +static struct fs_iter *fs_dict_iter_alloc(void) +{ + struct dict_fs_iter *iter = i_new(struct dict_fs_iter, 1); + return &iter->iter; +} + +static void +fs_dict_iter_init(struct fs_iter *_iter, const char *path, + enum fs_iter_flags flags ATTR_UNUSED) +{ + struct dict_fs_iter *iter = (struct dict_fs_iter *)_iter; + struct dict_fs *fs = (struct dict_fs *)_iter->fs; + + if (fs->path_prefix != NULL) + path = t_strconcat(fs->path_prefix, path, NULL); + + struct dict_op_settings set = { + .username = iter->iter.fs->username, + }; + iter->dict_iter = dict_iterate_init(fs->dict, &set, path, 0); +} + +static const char *fs_dict_iter_next(struct fs_iter *_iter) +{ + struct dict_fs_iter *iter = (struct dict_fs_iter *)_iter; + const char *key, *value; + + if (!dict_iterate(iter->dict_iter, &key, &value)) + return NULL; + return key; +} + +static int fs_dict_iter_deinit(struct fs_iter *_iter) +{ + struct dict_fs_iter *iter = (struct dict_fs_iter *)_iter; + const char *error; + int ret; + + ret = dict_iterate_deinit(&iter->dict_iter, &error); + if (ret < 0) + fs_set_error(_iter->event, EIO, + "Dict iteration failed: %s", error); + return ret; +} + +const struct fs fs_class_dict = { + .name = "dict", + .v = { + fs_dict_alloc, + fs_dict_init, + NULL, + fs_dict_free, + fs_dict_get_properties, + fs_dict_file_alloc, + fs_dict_file_init, + fs_dict_file_deinit, + NULL, + NULL, + NULL, NULL, + fs_default_set_metadata, + NULL, + fs_dict_prefetch, + NULL, + fs_dict_read_stream, + NULL, + fs_dict_write_stream, + fs_dict_write_stream_finish, + NULL, + NULL, + NULL, + fs_dict_stat, + fs_default_copy, + NULL, + fs_dict_delete, + fs_dict_iter_alloc, + fs_dict_iter_init, + fs_dict_iter_next, + fs_dict_iter_deinit, + NULL, + NULL + } +}; diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c new file mode 100644 index 0000000..22517e7 --- /dev/null +++ b/src/lib-fs/fs-metawrap.c @@ -0,0 +1,526 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "istream.h" +#include "istream-private.h" +#include "istream-concat.h" +#include "istream-metawrap.h" +#include "ostream.h" +#include "ostream-metawrap.h" +#include "iostream-temp.h" +#include "fs-api-private.h" + +struct metawrap_fs { + struct fs fs; + bool wrap_metadata; +}; + +struct metawrap_fs_file { + struct fs_file file; + struct metawrap_fs *fs; + struct fs_file *super_read; + enum fs_open_mode open_mode; + struct istream *input; + bool metadata_read; + + struct ostream *super_output; + struct ostream *temp_output; + string_t *metadata_header; + uoff_t metadata_write_size; + bool metadata_changed_since_write; +}; + +#define METAWRAP_FS(ptr) container_of((ptr), struct metawrap_fs, fs) +#define METAWRAP_FILE(ptr) container_of((ptr), struct metawrap_fs_file, file) + +static struct fs *fs_metawrap_alloc(void) +{ + struct metawrap_fs *fs; + + fs = i_new(struct metawrap_fs, 1); + fs->fs = fs_class_metawrap; + return &fs->fs; +} + +static int +fs_metawrap_init(struct fs *_fs, const char *args, + const struct fs_settings *set, const char **error_r) +{ + struct metawrap_fs *fs = METAWRAP_FS(_fs); + const char *parent_name, *parent_args; + + if (*args == '\0') { + *error_r = "Parent filesystem not given as parameter"; + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; + } + if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0) + return -1; + if ((fs_get_properties(_fs->parent) & FS_PROPERTY_METADATA) == 0) + fs->wrap_metadata = TRUE; + return 0; +} + +static void fs_metawrap_free(struct fs *_fs) +{ + struct metawrap_fs *fs = METAWRAP_FS(_fs); + + i_free(fs); +} + +static enum fs_properties fs_metawrap_get_properties(struct fs *_fs) +{ + const struct metawrap_fs *fs = METAWRAP_FS(_fs); + enum fs_properties props; + + props = fs_get_properties(_fs->parent); + if (fs->wrap_metadata) { + /* we don't have a quick stat() to see the file's size, + because of the metadata header */ + props &= ENUM_NEGATE(FS_PROPERTY_STAT); + /* Copying can copy the whole metadata. */ + props |= FS_PROPERTY_COPY_METADATA; + } + return props; +} + +static struct fs_file *fs_metawrap_file_alloc(void) +{ + struct metawrap_fs_file *file = i_new(struct metawrap_fs_file, 1); + return &file->file; +} + +static void +fs_metawrap_file_init(struct fs_file *_file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + struct metawrap_fs *fs = METAWRAP_FS(_file->fs); + + file->file.path = i_strdup(path); + file->fs = fs; + file->open_mode = mode; + + /* avoid unnecessarily creating two seekable streams */ + flags &= ENUM_NEGATE(FS_OPEN_FLAG_SEEKABLE); + + file->file.parent = fs_file_init_parent(_file, path, mode, flags); + if (file->fs->wrap_metadata && mode == FS_OPEN_MODE_READONLY && + (flags & FS_OPEN_FLAG_ASYNC) == 0) { + /* use async stream for parent, so fs_read_stream() won't create + another seekable stream needlessly */ + file->super_read = fs_file_init_parent(_file, path, + mode, flags | FS_OPEN_FLAG_ASYNC | + FS_OPEN_FLAG_ASYNC_NOQUEUE); + } else { + file->super_read = file->file.parent; + } + fs_metadata_init(&file->file); +} + +static void fs_metawrap_file_deinit(struct fs_file *_file) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + + if (file->super_read != _file->parent) + fs_file_deinit(&file->super_read); + str_free(&file->metadata_header); + fs_file_free(_file); + i_free(file->file.path); + i_free(file); +} + +static void fs_metawrap_file_close(struct fs_file *_file) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + + i_stream_unref(&file->input); + fs_file_close(file->super_read); + fs_file_close(_file->parent); +} + +static void +fs_metawrap_set_metadata(struct fs_file *_file, const char *key, + const char *value) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + + if (!file->fs->wrap_metadata || + strcmp(key, FS_METADATA_WRITE_FNAME) == 0) + fs_set_metadata(_file->parent, key, value); + else { + fs_default_set_metadata(_file, key, value); + file->metadata_changed_since_write = TRUE; + } +} + +static int +fs_metawrap_get_metadata(struct fs_file *_file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + ssize_t ret; + char c; + + if (!file->fs->wrap_metadata) + return fs_get_metadata_full(_file->parent, flags, metadata_r); + + if (file->metadata_read) { + /* we have the metadata */ + } else if ((flags & FS_GET_METADATA_FLAG_LOADED_ONLY) != 0) { + /* use the existing metadata only */ + } else if (file->input == NULL) { + if (fs_read(_file, &c, 1) < 0) + return -1; + } else { + /* use the existing istream to read it */ + while ((ret = i_stream_read(file->input)) == 0) { + if (file->metadata_read) + break; + + i_assert(!file->input->blocking); + fs_wait_async(_file->fs); + } + if (ret == -1 && file->input->stream_errno != 0) { + fs_set_error(_file->event, file->input->stream_errno, + "read(%s) failed: %s", + i_stream_get_name(file->input), + i_stream_get_error(file->input)); + return -1; + } + i_assert(file->metadata_read); + } + *metadata_r = &_file->metadata; + return 0; +} + +static bool fs_metawrap_prefetch(struct fs_file *_file, uoff_t length) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + + if (!file->fs->wrap_metadata) + return fs_prefetch(_file->parent, length); + else + return fs_prefetch(file->super_read, length); +} + +static ssize_t fs_metawrap_read(struct fs_file *_file, void *buf, size_t size) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + + if (!file->fs->wrap_metadata) + return fs_read(_file->parent, buf, size); + return fs_read_via_stream(_file, buf, size); +} + +static void +fs_metawrap_callback(const char *key, const char *value, void *context) +{ + struct metawrap_fs_file *file = context; + + if (key == NULL) { + file->metadata_read = TRUE; + return; + } + + T_BEGIN { + key = str_tabunescape(t_strdup_noconst(key)); + value = str_tabunescape(t_strdup_noconst(value)); + fs_default_set_metadata(&file->file, key, value); + } T_END; +} + +static struct istream * +fs_metawrap_read_stream(struct fs_file *_file, size_t max_buffer_size) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + struct istream *input; + + if (!file->fs->wrap_metadata) + return fs_read_stream(_file->parent, max_buffer_size); + + if (file->input != NULL) { + i_stream_ref(file->input); + i_stream_seek(file->input, 0); + return file->input; + } + + input = fs_read_stream(file->super_read, max_buffer_size); + file->input = i_stream_create_metawrap(input, fs_metawrap_callback, file); + i_stream_unref(&input); + i_stream_ref(file->input); + return file->input; +} + +static int fs_metawrap_write(struct fs_file *_file, const void *data, size_t size) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + + if (!file->fs->wrap_metadata) + return fs_write(_file->parent, data, size); + return fs_write_via_stream(_file, data, size); +} + +static void +fs_metawrap_append_metadata(struct metawrap_fs_file *file, string_t *str) +{ + const struct fs_metadata *metadata; + + array_foreach(&file->file.metadata, metadata) { + if (str_begins(metadata->key, FS_METADATA_INTERNAL_PREFIX)) + continue; + + str_append_tabescaped(str, metadata->key); + str_append_c(str, ':'); + str_append_tabescaped(str, metadata->value); + str_append_c(str, '\n'); + } + str_append_c(str, '\n'); +} + +static void +fs_metawrap_write_metadata_to(struct metawrap_fs_file *file, + struct ostream *output) +{ + string_t *str = t_str_new(256); + ssize_t ret; + + fs_metawrap_append_metadata(file, str); + file->metadata_write_size = str_len(str); + + ret = o_stream_send(output, str_data(str), str_len(str)); + if (ret < 0) + o_stream_close(output); + else + i_assert((size_t)ret == str_len(str)); + file->metadata_changed_since_write = FALSE; +} + +static void fs_metawrap_write_metadata(void *context) +{ + struct metawrap_fs_file *file = context; + + fs_metawrap_write_metadata_to(file, file->file.output); +} + +static void fs_metawrap_write_stream(struct fs_file *_file) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + i_assert(_file->output == NULL); + + if (!file->fs->wrap_metadata) { + file->super_output = fs_write_stream(_file->parent); + _file->output = file->super_output; + } else { + file->temp_output = + iostream_temp_create_named(_file->fs->temp_path_prefix, + IOSTREAM_TEMP_FLAG_TRY_FD_DUP, + fs_file_path(_file)); + _file->output = o_stream_create_metawrap(file->temp_output, + fs_metawrap_write_metadata, file); + } +} + +static struct istream * +fs_metawrap_create_updated_istream(struct metawrap_fs_file *file, + struct istream *input) +{ + struct istream *input2, *inputs[3]; + + if (file->metadata_header != NULL) + str_truncate(file->metadata_header, 0); + else + file->metadata_header = str_new(default_pool, 1024); + fs_metawrap_append_metadata(file, file->metadata_header); + inputs[0] = i_stream_create_from_data(str_data(file->metadata_header), + str_len(file->metadata_header)); + + i_stream_seek(input, file->metadata_write_size); + inputs[1] = i_stream_create_limit(input, UOFF_T_MAX); + inputs[2] = NULL; + input2 = i_stream_create_concat(inputs); + i_stream_unref(&inputs[0]); + i_stream_unref(&inputs[1]); + + file->metadata_write_size = str_len(file->metadata_header); + return input2; +} + +static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + struct istream *input; + int ret; + + if (_file->output != NULL) { + if (_file->output == file->super_output) + _file->output = NULL; + else + o_stream_unref(&_file->output); + } + if (!success) { + if (file->super_output != NULL) { + /* no metawrap */ + i_assert(file->temp_output == NULL); + fs_write_stream_abort_parent(_file, &file->super_output); + } else { + i_assert(file->temp_output != NULL); + o_stream_destroy(&file->temp_output); + } + return -1; + } + + if (file->super_output != NULL) { + /* no metawrap */ + i_assert(file->temp_output == NULL); + return fs_write_stream_finish(_file->parent, &file->super_output); + } + if (file->temp_output == NULL) { + /* finishing up */ + i_assert(file->super_output == NULL); + return fs_write_stream_finish_async(_file->parent); + } + /* finish writing the temporary file */ + if (file->temp_output->offset == 0) { + /* empty file - temp_output is already finished, + so we can't write to it. To make sure metadata is still + appended/written to file use metadata_changed_since_write */ + file->metadata_changed_since_write = TRUE; + } + input = iostream_temp_finish(&file->temp_output, IO_BLOCK_SIZE); + if (file->metadata_changed_since_write) { + /* we'll need to recreate the metadata. do this by creating a + new istream combining the new metadata header and the + old body. */ + struct istream *input2 = input; + + input = fs_metawrap_create_updated_istream(file, input); + i_stream_unref(&input2); + } + file->super_output = fs_write_stream(_file->parent); + o_stream_nsend_istream(file->super_output, input); + /* because of the "end of metadata" LF, there's always at least + 1 byte */ + i_assert(file->super_output->offset > 0 || + file->super_output->stream_errno != 0); + ret = fs_write_stream_finish(_file->parent, &file->super_output); + i_stream_unref(&input); + return ret; +} + +static int fs_metawrap_stat(struct fs_file *_file, struct stat *st_r) +{ + struct metawrap_fs_file *file = METAWRAP_FILE(_file); + struct istream *input; + uoff_t input_size; + ssize_t ret; + + if (!file->fs->wrap_metadata) + return fs_stat(_file->parent, st_r); + + if (file->metadata_write_size != 0) { + /* fs_stat() after a write. we can do this quickly. */ + if (fs_stat(_file->parent, st_r) < 0) + return -1; + if ((uoff_t)st_r->st_size < file->metadata_write_size) { + fs_set_error(_file->event, EIO, + "Just-written %s shrank unexpectedly " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", + fs_file_path(_file), st_r->st_size, + file->metadata_write_size); + return -1; + } + st_r->st_size -= file->metadata_write_size; + return 0; + } + + if (file->input == NULL) + input = fs_read_stream(_file, IO_BLOCK_SIZE); + else { + input = file->input; + i_stream_ref(input); + } + if ((ret = i_stream_get_size(input, TRUE, &input_size)) < 0) { + fs_set_error(_file->event, input->stream_errno, + "i_stream_get_size(%s) failed: %s", + fs_file_path(_file), i_stream_get_error(input)); + i_stream_unref(&input); + return -1; + } + i_stream_unref(&input); + if (ret == 0) { + /* we shouldn't get here */ + fs_set_error(_file->event, EIO, "i_stream_get_size(%s) returned size as unknown", + fs_file_path(_file)); + return -1; + } + + if (fs_stat(_file->parent, st_r) < 0) { + i_assert(errno != EAGAIN); /* read should have caught this */ + return -1; + } + st_r->st_size = input_size; + return 0; +} + +static int fs_metawrap_copy(struct fs_file *_src, struct fs_file *_dest) +{ + struct metawrap_fs_file *dest = METAWRAP_FILE(_dest); + + if (!dest->fs->wrap_metadata || !_dest->metadata_changed) + return fs_wrapper_copy(_src, _dest); + else + return fs_default_copy(_src, _dest); +} + +const struct fs fs_class_metawrap = { + .name = "metawrap", + .v = { + fs_metawrap_alloc, + fs_metawrap_init, + NULL, + fs_metawrap_free, + fs_metawrap_get_properties, + fs_metawrap_file_alloc, + fs_metawrap_file_init, + fs_metawrap_file_deinit, + fs_metawrap_file_close, + fs_wrapper_file_get_path, + fs_wrapper_set_async_callback, + fs_wrapper_wait_async, + fs_metawrap_set_metadata, + fs_metawrap_get_metadata, + fs_metawrap_prefetch, + fs_metawrap_read, + fs_metawrap_read_stream, + fs_metawrap_write, + fs_metawrap_write_stream, + fs_metawrap_write_stream_finish, + fs_wrapper_lock, + fs_wrapper_unlock, + fs_wrapper_exists, + fs_metawrap_stat, + fs_metawrap_copy, + fs_wrapper_rename, + fs_wrapper_delete, + fs_wrapper_iter_alloc, + fs_wrapper_iter_init, + fs_wrapper_iter_next, + fs_wrapper_iter_deinit, + NULL, + fs_wrapper_get_nlinks, + } +}; diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c new file mode 100644 index 0000000..657938e --- /dev/null +++ b/src/lib-fs/fs-posix.c @@ -0,0 +1,1028 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "guid.h" +#include "istream.h" +#include "ostream.h" +#include "safe-mkstemp.h" +#include "mkdir-parents.h" +#include "write-full.h" +#include "file-lock.h" +#include "file-dotlock.h" +#include "time-util.h" +#include "fs-api-private.h" + +#include <stdio.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <sys/stat.h> + +#define FS_POSIX_DOTLOCK_STALE_TIMEOUT_SECS (60*10) +#define MAX_MKDIR_RETRY_COUNT 5 +#define FS_DEFAULT_MODE 0600 + +enum fs_posix_lock_method { + FS_POSIX_LOCK_METHOD_FLOCK, + FS_POSIX_LOCK_METHOD_DOTLOCK +}; + +struct posix_fs { + struct fs fs; + char *temp_file_prefix, *root_path, *path_prefix; + size_t temp_file_prefix_len; + enum fs_posix_lock_method lock_method; + mode_t mode; + bool mode_auto; + bool have_dirs; + bool disable_fsync; + bool accurate_mtime; +}; + +struct posix_fs_file { + struct fs_file file; + char *temp_path, *full_path; + int fd; + enum fs_open_mode open_mode; + enum fs_open_flags open_flags; + + buffer_t *write_buf; + + bool seek_to_beginning; +}; + +struct posix_fs_lock { + struct fs_lock lock; + struct file_lock *file_lock; + struct dotlock *dotlock; +}; + +struct posix_fs_iter { + struct fs_iter iter; + char *path; + DIR *dir; + int err; +}; + +static struct fs *fs_posix_alloc(void) +{ + struct posix_fs *fs; + + fs = i_new(struct posix_fs, 1); + fs->fs = fs_class_posix; + return &fs->fs; +} + +static int +fs_posix_init(struct fs *_fs, const char *args, const struct fs_settings *set, + const char **error_r) +{ + struct posix_fs *fs = container_of(_fs, struct posix_fs, fs); + const char *const *tmp; + + fs->temp_file_prefix = set->temp_file_prefix != NULL ? + i_strdup(set->temp_file_prefix) : i_strdup("temp.dovecot."); + fs->temp_file_prefix_len = strlen(fs->temp_file_prefix); + fs->root_path = i_strdup(set->root_path); + fs->fs.set.temp_file_prefix = fs->temp_file_prefix; + fs->fs.set.root_path = fs->root_path; + + fs->lock_method = FS_POSIX_LOCK_METHOD_FLOCK; + fs->mode = FS_DEFAULT_MODE; + + tmp = t_strsplit_spaces(args, ":"); + for (; *tmp != NULL; tmp++) { + const char *arg = *tmp; + + if (strcmp(arg, "lock=flock") == 0) + fs->lock_method = FS_POSIX_LOCK_METHOD_FLOCK; + else if (strcmp(arg, "lock=dotlock") == 0) + fs->lock_method = FS_POSIX_LOCK_METHOD_DOTLOCK; + else if (str_begins(arg, "prefix=")) { + i_free(fs->path_prefix); + fs->path_prefix = i_strdup(arg + 7); + } else if (strcmp(arg, "mode=auto") == 0) { + fs->mode_auto = TRUE; + } else if (strcmp(arg, "dirs") == 0) { + fs->have_dirs = TRUE; + } else if (strcmp(arg, "no-fsync") == 0) { + fs->disable_fsync = TRUE; + } else if (strcmp(arg, "accurate-mtime") == 0) { + fs->accurate_mtime = TRUE; + } else if (str_begins(arg, "mode=")) { + unsigned int mode; + if (str_to_uint_oct(arg+5, &mode) < 0) { + *error_r = t_strdup_printf("Invalid mode value: %s", arg+5); + return -1; + } + fs->mode = mode & 0666; + if (fs->mode == 0) { + *error_r = t_strdup_printf("Invalid mode: %s", arg+5); + return -1; + } + } else { + *error_r = t_strdup_printf("Unknown arg '%s'", arg); + return -1; + } + } + return 0; +} + +static void fs_posix_free(struct fs *_fs) +{ + struct posix_fs *fs = container_of(_fs, struct posix_fs, fs); + + i_free(fs->temp_file_prefix); + i_free(fs->root_path); + i_free(fs->path_prefix); + i_free(fs); +} + +static enum fs_properties fs_posix_get_properties(struct fs *_fs) +{ + struct posix_fs *fs = container_of(_fs, struct posix_fs, fs); + enum fs_properties props = + FS_PROPERTY_LOCKS | FS_PROPERTY_FASTCOPY | FS_PROPERTY_RENAME | + FS_PROPERTY_STAT | FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER; + + /* FS_PROPERTY_DIRECTORIES is not returned normally because fs_delete() + automatically rmdir()s parents. For backwards compatibility + (especially with SIS code) we'll do it that way, but optionally with + "dirs" parameter enable them. This is especially important to be + able to use doveadm fs commands to delete empty directories. */ + if (fs->have_dirs) + props |= FS_PROPERTY_DIRECTORIES; + return props; +} + +static int +fs_posix_get_mode(struct posix_fs_file *file, const char *path, mode_t *mode_r) +{ + struct posix_fs *fs = (struct posix_fs *)file->file.fs; + struct stat st; + const char *p; + + *mode_r = fs->mode; + + /* This function is used to get mode of the parent directory, so path + is never the same as file->path. The file is used just to set the + errors. */ + while (stat(path, &st) < 0) { + if (errno != ENOENT) { + fs_set_error_errno(file->file.event, + "stat(%s) failed: %m", path); + return -1; + } + p = strrchr(path, '/'); + if (p != NULL) + path = t_strdup_until(path, p); + else if (strcmp(path, ".") != 0) + path = "."; + else + return 0; + } + if ((st.st_mode & S_ISGID) != 0) { + /* setgid set - copy mode from parent */ + *mode_r = st.st_mode & 0666; + } + return 0; +} + +static int fs_posix_mkdir_parents(struct posix_fs_file *file, const char *path) +{ + const char *dir, *fname; + mode_t mode, dir_mode; + + fname = strrchr(path, '/'); + if (fname == NULL) + return 1; + dir = t_strdup_until(path, fname); + + if (fs_posix_get_mode(file, dir, &mode) < 0) + return -1; + dir_mode = mode; + if ((dir_mode & 0600) != 0) dir_mode |= 0100; + if ((dir_mode & 0060) != 0) dir_mode |= 0010; + if ((dir_mode & 0006) != 0) dir_mode |= 0001; + + if (mkdir_parents(dir, dir_mode) == 0) + return 0; + else if (errno == EEXIST) + return 1; + else { + fs_set_error_errno(file->file.event, + "mkdir_parents(%s) failed: %m", dir); + return -1; + } +} + +static int fs_posix_rmdir_parents(struct posix_fs_file *file, const char *path) +{ + struct posix_fs *fs = (struct posix_fs *)file->file.fs; + const char *p; + + if (fs->have_dirs) + return 0; + if (fs->root_path == NULL && fs->path_prefix == NULL) + return 0; + + while ((p = strrchr(path, '/')) != NULL) { + path = t_strdup_until(path, p); + if ((fs->root_path != NULL && strcmp(path, fs->root_path) == 0) || + (fs->path_prefix != NULL && str_begins(fs->path_prefix, path))) + break; + if (rmdir(path) == 0) { + /* success, continue to parent */ + } else if (errno == ENOTEMPTY || errno == EEXIST) { + /* there are other entries in this directory */ + break; + } else if (errno == EBUSY || errno == ENOENT) { + /* some other not-unexpected error */ + break; + } else { + fs_set_error_errno(file->file.event, + "rmdir(%s) failed: %m", path); + return -1; + } + } + return 0; +} + +static int fs_posix_create(struct posix_fs_file *file) +{ + struct posix_fs *fs = container_of(file->file.fs, struct posix_fs, fs); + string_t *str = t_str_new(256); + const char *slash; + unsigned int try_count = 0; + mode_t mode; + int fd; + + i_assert(file->temp_path == NULL); + + if ((slash = strrchr(file->full_path, '/')) != NULL) { + str_append_data(str, file->full_path, slash - file->full_path); + if (fs_posix_get_mode(file, str_c(str), &mode) < 0) + return -1; + str_append_c(str, '/'); + } else { + if (fs_posix_get_mode(file, ".", &mode) < 0) + return -1; + } + str_append(str, fs->temp_file_prefix); + + fd = safe_mkstemp_hostpid(str, mode, (uid_t)-1, (gid_t)-1); + while (fd == -1 && errno == ENOENT && + try_count <= MAX_MKDIR_RETRY_COUNT) { + if (fs_posix_mkdir_parents(file, str_c(str)) < 0) + return -1; + fd = safe_mkstemp_hostpid(str, mode, (uid_t)-1, (gid_t)-1); + try_count++; + } + if (fd == -1) { + fs_set_error_errno(file->file.event, + "safe_mkstemp(%s) failed: %m", str_c(str)); + return -1; + } + file->temp_path = i_strdup(str_c(str)); + return fd; +} + +static int fs_posix_open(struct posix_fs_file *file) +{ + const char *path = file->full_path; + + i_assert(file->fd == -1); + + switch (file->open_mode) { + case FS_OPEN_MODE_READONLY: + file->fd = open(path, O_RDONLY); + if (file->fd == -1) + fs_set_error_errno(file->file.event, + "open(%s) failed: %m", path); + break; + case FS_OPEN_MODE_APPEND: + file->fd = open(path, O_RDWR | O_APPEND); + if (file->fd == -1) + fs_set_error_errno(file->file.event, + "open(%s) failed: %m", path); + break; + case FS_OPEN_MODE_CREATE_UNIQUE_128: + case FS_OPEN_MODE_CREATE: + case FS_OPEN_MODE_REPLACE: + T_BEGIN { + file->fd = fs_posix_create(file); + } T_END; + break; + } + if (file->fd == -1) + return -1; + return 0; +} + +static struct fs_file *fs_posix_file_alloc(void) +{ + struct posix_fs_file *file = i_new(struct posix_fs_file, 1); + return &file->file; +} + +static void +fs_posix_file_init(struct fs_file *_file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + struct posix_fs *fs = container_of(_file->fs, struct posix_fs, fs); + guid_128_t guid; + size_t path_len = strlen(path); + + if (path_len > 0 && path[path_len-1] == '/') { + /* deleting "path/" (used e.g. by doveadm fs delete) - strip + out the trailing "/" since it doesn't work well with NFS. */ + path = t_strndup(path, path_len-1); + } + + if (mode != FS_OPEN_MODE_CREATE_UNIQUE_128) + file->file.path = i_strdup(path); + else { + guid_128_generate(guid); + file->file.path = i_strdup_printf("%s/%s", path, + guid_128_to_string(guid)); + } + file->full_path = fs->path_prefix == NULL ? i_strdup(file->file.path) : + i_strconcat(fs->path_prefix, file->file.path, NULL); + file->open_mode = mode; + file->open_flags = flags; + file->fd = -1; +} + +static void fs_posix_file_close(struct fs_file *_file) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + + if (file->fd != -1 && file->file.output == NULL) { + if (close(file->fd) < 0) { + e_error(_file->event, "close(%s) failed: %m", + file->full_path); + } + file->fd = -1; + } +} + +static void fs_posix_file_deinit(struct fs_file *_file) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + + i_assert(_file->output == NULL); + + switch (file->open_mode) { + case FS_OPEN_MODE_READONLY: + case FS_OPEN_MODE_APPEND: + break; + case FS_OPEN_MODE_CREATE_UNIQUE_128: + case FS_OPEN_MODE_CREATE: + case FS_OPEN_MODE_REPLACE: + if (file->temp_path == NULL) + break; + /* failed to create/replace this. delete the temp file */ + if (unlink(file->temp_path) < 0) { + e_error(_file->event, "unlink(%s) failed: %m", + file->temp_path); + } + break; + } + + fs_file_free(_file); + i_free(file->temp_path); + i_free(file->full_path); + i_free(file->file.path); + i_free(file); +} + +static int fs_posix_open_for_read(struct posix_fs_file *file) +{ + i_assert(file->file.output == NULL); + i_assert(file->temp_path == NULL); + + if (file->fd == -1) { + if (fs_posix_open(file) < 0) + return -1; + } + return 0; +} + +static bool fs_posix_prefetch(struct fs_file *_file, uoff_t length ATTR_UNUSED) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + + if (fs_posix_open_for_read(file) < 0) + return TRUE; + +/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */ +#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) + if (posix_fadvise(file->fd, 0, length, POSIX_FADV_WILLNEED) < 0) { + e_error(_file->event, "posix_fadvise(%s) failed: %m", file->full_path); + return TRUE; + } +#endif + return FALSE; +} + +static ssize_t fs_posix_read(struct fs_file *_file, void *buf, size_t size) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + ssize_t ret; + + if (fs_posix_open_for_read(file) < 0) + return -1; + + if (file->seek_to_beginning) { + file->seek_to_beginning = FALSE; + if (lseek(file->fd, 0, SEEK_SET) < 0) { + fs_set_error_errno(_file->event, + "lseek(%s, 0) failed: %m", + file->full_path); + return -1; + } + } + + ret = read(file->fd, buf, size); + if (ret < 0) + fs_set_error_errno(_file->event, "read(%s) failed: %m", + file->full_path); + fs_posix_file_close(_file); + return ret; +} + +static struct istream * +fs_posix_read_stream(struct fs_file *_file, size_t max_buffer_size) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + struct istream *input; + int fd_dup; + + if (fs_posix_open_for_read(file) < 0) + input = i_stream_create_error_str(errno, "%s", fs_file_last_error(_file)); + else if ((fd_dup = dup(file->fd)) == -1) + input = i_stream_create_error_str(errno, "dup() failed: %m"); + else { + /* The stream could live even after the fs_file. + Don't use file->fd directly, because the fd may still be + used for other purposes. It's especially important for files + that were just created. */ + input = i_stream_create_fd_autoclose(&fd_dup, max_buffer_size); + } + i_stream_set_name(input, file->full_path); + return input; +} + +static void fs_posix_write_rename_if_needed(struct posix_fs_file *file) +{ + struct posix_fs *fs = container_of(file->file.fs, struct posix_fs, fs); + const char *new_fname; + + new_fname = fs_metadata_find(&file->file.metadata, FS_METADATA_WRITE_FNAME); + if (new_fname == NULL) + return; + + i_free(file->file.path); + file->file.path = i_strdup(new_fname); + + i_free(file->full_path); + file->full_path = fs->path_prefix == NULL ? i_strdup(file->file.path) : + i_strconcat(fs->path_prefix, file->file.path, NULL); +} + +static int fs_posix_write_finish_link(struct posix_fs_file *file) +{ + unsigned int try_count = 0; + int ret; + + ret = link(file->temp_path, file->full_path); + while (ret < 0 && errno == ENOENT && + try_count <= MAX_MKDIR_RETRY_COUNT) { + if (fs_posix_mkdir_parents(file, file->full_path) < 0) + return -1; + ret = link(file->temp_path, file->full_path); + try_count++; + } + if (ret < 0) { + fs_set_error_errno(file->file.event, "link(%s, %s) failed: %m", + file->temp_path, file->full_path); + } + return ret; +} + +static int fs_posix_write_finish(struct posix_fs_file *file) +{ + struct posix_fs *fs = container_of(file->file.fs, struct posix_fs, fs); + unsigned int try_count = 0; + int ret, old_errno; + + if ((file->open_flags & FS_OPEN_FLAG_FSYNC) != 0 && + !fs->disable_fsync) { + if (fdatasync(file->fd) < 0) { + fs_set_error_errno(file->file.event, + "fdatasync(%s) failed: %m", + file->full_path); + return -1; + } + } + if (fs->accurate_mtime) { + /* Linux updates the mtime timestamp only on timer interrupts. + This isn't anywhere close to being microsecond precision. + If requested, use utimes() to explicitly set a more accurate + mtime. */ + struct timeval tv[2]; + i_gettimeofday(&tv[0]); + tv[1] = tv[0]; + if ((utimes(file->temp_path, tv)) < 0) { + fs_set_error_errno(file->file.event, + "utimes(%s) failed: %m", + file->temp_path); + return -1; + } + } + + fs_posix_write_rename_if_needed(file); + switch (file->open_mode) { + case FS_OPEN_MODE_CREATE_UNIQUE_128: + case FS_OPEN_MODE_CREATE: + ret = fs_posix_write_finish_link(file); + old_errno = errno; + if (unlink(file->temp_path) < 0) { + fs_set_error_errno(file->file.event, + "unlink(%s) failed: %m", + file->temp_path); + } + errno = old_errno; + if (ret < 0) { + fs_posix_file_close(&file->file); + i_free_and_null(file->temp_path); + return -1; + } + break; + case FS_OPEN_MODE_REPLACE: + ret = rename(file->temp_path, file->full_path); + while (ret < 0 && errno == ENOENT && + try_count <= MAX_MKDIR_RETRY_COUNT) { + if (fs_posix_mkdir_parents(file, file->full_path) < 0) + return -1; + ret = rename(file->temp_path, file->full_path); + try_count++; + } + if (ret < 0) { + fs_set_error_errno(file->file.event, + "rename(%s, %s) failed: %m", + file->temp_path, file->full_path); + return -1; + } + break; + default: + i_unreached(); + } + i_free_and_null(file->temp_path); + file->seek_to_beginning = TRUE; + /* allow opening the file after writing to it */ + file->open_mode = FS_OPEN_MODE_READONLY; + return 0; +} + +static int fs_posix_write(struct fs_file *_file, const void *data, size_t size) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + ssize_t ret; + + if (file->fd == -1) { + if (fs_posix_open(file) < 0) + return -1; + i_assert(file->fd != -1); + } + + if (file->open_mode != FS_OPEN_MODE_APPEND) { + if (write_full(file->fd, data, size) < 0) { + fs_set_error_errno(_file->event, "write(%s) failed: %m", + file->full_path); + return -1; + } + return fs_posix_write_finish(file); + } + + /* atomic append - it should either succeed or fail */ + ret = write(file->fd, data, size); + if (ret < 0) { + fs_set_error_errno(_file->event, "write(%s) failed: %m", + file->full_path); + return -1; + } + if ((size_t)ret != size) { + fs_set_error(_file->event, ENOSPC, + "write(%s) returned %zu/%zu", + file->full_path, (size_t)ret, size); + return -1; + } + return 0; +} + +static void fs_posix_write_stream(struct fs_file *_file) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + + i_assert(_file->output == NULL); + + if (file->open_mode == FS_OPEN_MODE_APPEND) { + file->write_buf = buffer_create_dynamic(default_pool, 1024*32); + _file->output = o_stream_create_buffer(file->write_buf); + } else if (file->fd == -1 && fs_posix_open(file) < 0) { + _file->output = o_stream_create_error_str(errno, "%s", + fs_file_last_error(_file)); + } else { + i_assert(file->fd != -1); + _file->output = o_stream_create_fd_file(file->fd, + UOFF_T_MAX, FALSE); + } + o_stream_set_name(_file->output, file->full_path); +} + +static int fs_posix_write_stream_finish(struct fs_file *_file, bool success) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + int ret = success ? 0 : -1; + + o_stream_destroy(&_file->output); + + switch (file->open_mode) { + case FS_OPEN_MODE_APPEND: + if (ret == 0) { + ret = fs_posix_write(_file, file->write_buf->data, + file->write_buf->used); + } + buffer_free(&file->write_buf); + break; + case FS_OPEN_MODE_CREATE: + case FS_OPEN_MODE_CREATE_UNIQUE_128: + case FS_OPEN_MODE_REPLACE: + if (ret == 0) + ret = fs_posix_write_finish(file); + break; + case FS_OPEN_MODE_READONLY: + i_unreached(); + } + return ret < 0 ? -1 : 1; +} + +static int +fs_posix_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lock_r) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + struct posix_fs *fs = container_of(_file->fs, struct posix_fs, fs); + struct dotlock_settings dotlock_set; + struct posix_fs_lock fs_lock, *ret_lock; + const char *error; + int ret = -1; + + i_zero(&fs_lock); + fs_lock.lock.file = _file; + + struct file_lock_settings lock_set = { + .lock_method = FILE_LOCK_METHOD_FLOCK, + }; + switch (fs->lock_method) { + case FS_POSIX_LOCK_METHOD_FLOCK: +#ifndef HAVE_FLOCK + fs_set_error(_file->event, ENOTSUP, + "flock() not supported by OS (for file %s)", + file->full_path); +#else + if (secs == 0) { + ret = file_try_lock(file->fd, file->full_path, F_WRLCK, + &lock_set, &fs_lock.file_lock, + &error); + } else { + ret = file_wait_lock(file->fd, file->full_path, F_WRLCK, + &lock_set, secs, + &fs_lock.file_lock, &error); + } + if (ret < 0) { + fs_set_error_errno(_file->event, "flock(%s) failed: %s", + file->full_path, error); + } +#endif + break; + case FS_POSIX_LOCK_METHOD_DOTLOCK: + i_zero(&dotlock_set); + dotlock_set.stale_timeout = FS_POSIX_DOTLOCK_STALE_TIMEOUT_SECS; + dotlock_set.use_excl_lock = TRUE; + dotlock_set.timeout = secs; + + ret = file_dotlock_create(&dotlock_set, file->full_path, + secs == 0 ? 0 : + DOTLOCK_CREATE_FLAG_NONBLOCK, + &fs_lock.dotlock); + if (ret < 0) { + fs_set_error_errno(_file->event, + "file_dotlock_create(%s) failed: %m", + file->full_path); + } + break; + } + if (ret <= 0) + return ret; + + ret_lock = i_new(struct posix_fs_lock, 1); + *ret_lock = fs_lock; + *lock_r = &ret_lock->lock; + return 1; +} + +static void fs_posix_unlock(struct fs_lock *_lock) +{ + struct posix_fs_lock *lock = + container_of(_lock, struct posix_fs_lock, lock); + + if (lock->file_lock != NULL) + file_unlock(&lock->file_lock); + if (lock->dotlock != NULL) + file_dotlock_delete(&lock->dotlock); + i_free(lock); +} + +static int fs_posix_exists(struct fs_file *_file) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + struct stat st; + + if (stat(file->full_path, &st) < 0) { + if (errno != ENOENT) { + fs_set_error_errno(_file->event, "stat(%s) failed: %m", + file->full_path); + return -1; + } + return 0; + } + return 1; +} + +static int fs_posix_stat(struct fs_file *_file, struct stat *st_r) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + + /* in case output != NULL it means that we're still writing to the file + and fs_stat() shouldn't stat the unfinished file. this is done by + fs-sis after fs_copy(). */ + if (file->fd != -1 && _file->output == NULL) { + if (fstat(file->fd, st_r) < 0) { + fs_set_error_errno(_file->event, "fstat(%s) failed: %m", + file->full_path); + return -1; + } + } else { + if (stat(file->full_path, st_r) < 0) { + fs_set_error_errno(_file->event, "stat(%s) failed: %m", + file->full_path); + return -1; + } + } + return 0; +} + +static int fs_posix_copy(struct fs_file *_src, struct fs_file *_dest) +{ + struct posix_fs_file *src = + container_of(_src, struct posix_fs_file, file); + struct posix_fs_file *dest = + container_of(_dest, struct posix_fs_file, file); + unsigned int try_count = 0; + int ret; + + fs_posix_write_rename_if_needed(dest); + ret = link(src->full_path, dest->full_path); + if (errno == EEXIST && dest->open_mode == FS_OPEN_MODE_REPLACE) { + /* destination file already exists - replace it */ + i_unlink_if_exists(dest->full_path); + ret = link(src->full_path, dest->full_path); + } + while (ret < 0 && errno == ENOENT && + try_count <= MAX_MKDIR_RETRY_COUNT) { + if (fs_posix_mkdir_parents(dest, dest->full_path) < 0) + return -1; + ret = link(src->full_path, dest->full_path); + try_count++; + } + if (ret < 0) { + fs_set_error_errno(_src->event, "link(%s, %s) failed: %m", + src->full_path, dest->full_path); + return -1; + } + return 0; +} + +static int fs_posix_rename(struct fs_file *_src, struct fs_file *_dest) +{ + struct posix_fs_file *src = + container_of(_src, struct posix_fs_file, file); + struct posix_fs_file *dest = + container_of(_dest, struct posix_fs_file, file); + unsigned int try_count = 0; + int ret; + + ret = rename(src->full_path, dest->full_path); + while (ret < 0 && errno == ENOENT && + try_count <= MAX_MKDIR_RETRY_COUNT) { + if (fs_posix_mkdir_parents(dest, dest->full_path) < 0) + return -1; + ret = rename(src->full_path, dest->full_path); + try_count++; + } + if (ret < 0) { + fs_set_error_errno(_src->event, "rename(%s, %s) failed: %m", + src->full_path, dest->full_path); + return -1; + } + return 0; +} + +static int fs_posix_delete(struct fs_file *_file) +{ + struct posix_fs_file *file = + container_of(_file, struct posix_fs_file, file); + + if (unlink(file->full_path) < 0) { + if (!UNLINK_EISDIR(errno)) { + fs_set_error_errno(_file->event, "unlink(%s) failed: %m", + file->full_path); + return -1; + } + /* attempting to delete a directory. convert it to rmdir() + automatically. */ + if (rmdir(file->full_path) < 0) { + fs_set_error_errno(_file->event, "rmdir(%s) failed: %m", + file->full_path); + return -1; + } + } + (void)fs_posix_rmdir_parents(file, file->full_path); + return 0; +} + +static struct fs_iter *fs_posix_iter_alloc(void) +{ + struct posix_fs_iter *iter = i_new(struct posix_fs_iter, 1); + return &iter->iter; +} + +static void +fs_posix_iter_init(struct fs_iter *_iter, const char *path, + enum fs_iter_flags flags ATTR_UNUSED) +{ + struct posix_fs_iter *iter = + container_of(_iter, struct posix_fs_iter, iter); + struct posix_fs *fs = container_of(_iter->fs, struct posix_fs, fs); + + iter->path = fs->path_prefix == NULL ? i_strdup(path) : + i_strconcat(fs->path_prefix, path, NULL); + if (iter->path[0] == '\0') { + i_free(iter->path); + iter->path = i_strdup("."); + } + iter->dir = opendir(iter->path); + if (iter->dir == NULL && errno != ENOENT) { + iter->err = errno; + fs_set_error_errno(_iter->event, + "opendir(%s) failed: %m", iter->path); + } +} + +static bool fs_posix_iter_want(struct posix_fs_iter *iter, const char *fname) +{ + bool ret; + + T_BEGIN { + const char *path = t_strdup_printf("%s/%s", iter->path, fname); + struct stat st; + + if (stat(path, &st) < 0 && + lstat(path, &st) < 0) + ret = FALSE; + else if (!S_ISDIR(st.st_mode)) + ret = (iter->iter.flags & FS_ITER_FLAG_DIRS) == 0; + else + ret = (iter->iter.flags & FS_ITER_FLAG_DIRS) != 0; + } T_END; + return ret; +} + +static const char *fs_posix_iter_next(struct fs_iter *_iter) +{ + struct posix_fs_iter *iter = + container_of(_iter, struct posix_fs_iter, iter); + struct posix_fs *fs = container_of(_iter->fs, struct posix_fs, fs); + struct dirent *d; + + if (iter->dir == NULL) + return NULL; + + errno = 0; + for (; (d = readdir(iter->dir)) != NULL; errno = 0) { + if (strcmp(d->d_name, ".") == 0 || + strcmp(d->d_name, "..") == 0) + continue; + if (strncmp(d->d_name, fs->temp_file_prefix, + fs->temp_file_prefix_len) == 0) + continue; +#ifdef HAVE_DIRENT_D_TYPE + switch (d->d_type) { + case DT_UNKNOWN: + if (fs_posix_iter_want(iter, d->d_name)) + return d->d_name; + break; + case DT_DIR: + if ((iter->iter.flags & FS_ITER_FLAG_DIRS) != 0) + return d->d_name; + break; + default: + if ((iter->iter.flags & FS_ITER_FLAG_DIRS) == 0) + return d->d_name; + break; + } +#else + if (fs_posix_iter_want(iter, d->d_name)) + return d->d_name; +#endif + } + if (errno != 0) { + iter->err = errno; + fs_set_error_errno(_iter->event, + "readdir(%s) failed: %m", iter->path); + } + return NULL; +} + +static int fs_posix_iter_deinit(struct fs_iter *_iter) +{ + struct posix_fs_iter *iter = + container_of(_iter, struct posix_fs_iter, iter); + int ret = 0; + + if (iter->dir != NULL && closedir(iter->dir) < 0 && iter->err == 0) { + iter->err = errno; + fs_set_error_errno(_iter->event, + "closedir(%s) failed: %m", iter->path); + } + if (iter->err != 0) { + errno = iter->err; + ret = -1; + } + i_free(iter->path); + return ret; +} + +const struct fs fs_class_posix = { + .name = "posix", + .v = { + fs_posix_alloc, + fs_posix_init, + NULL, + fs_posix_free, + fs_posix_get_properties, + fs_posix_file_alloc, + fs_posix_file_init, + fs_posix_file_deinit, + fs_posix_file_close, + NULL, + NULL, NULL, + fs_default_set_metadata, + NULL, + fs_posix_prefetch, + fs_posix_read, + fs_posix_read_stream, + fs_posix_write, + fs_posix_write_stream, + fs_posix_write_stream_finish, + fs_posix_lock, + fs_posix_unlock, + fs_posix_exists, + fs_posix_stat, + fs_posix_copy, + fs_posix_rename, + fs_posix_delete, + fs_posix_iter_alloc, + fs_posix_iter_init, + fs_posix_iter_next, + fs_posix_iter_deinit, + NULL, + NULL, + } +}; diff --git a/src/lib-fs/fs-randomfail.c b/src/lib-fs/fs-randomfail.c new file mode 100644 index 0000000..48b1bf0 --- /dev/null +++ b/src/lib-fs/fs-randomfail.c @@ -0,0 +1,555 @@ +/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "istream-private.h" +#include "istream-concat.h" +#include "istream-failure-at.h" +#include "ostream-failure-at.h" +#include "ostream.h" +#include "fs-api-private.h" + + +#define RANDOMFAIL_ERROR "Random failure injection" + +static const char *fs_op_names[FS_OP_COUNT] = { + "wait", "metadata", "prefetch", "read", "write", "lock", "exists", + "stat", "copy", "rename", "delete", "iter" +}; + +struct randomfail_fs { + struct fs fs; + unsigned int op_probability[FS_OP_COUNT]; + uoff_t range_start[FS_OP_COUNT], range_end[FS_OP_COUNT]; +}; + +struct randomfail_fs_file { + struct fs_file file; + struct fs_file *super_read; + struct istream *input; + bool op_pending[FS_OP_COUNT]; + + struct ostream *super_output; +}; + +struct randomfail_fs_iter { + struct fs_iter iter; + struct fs_iter *super; + unsigned int fail_pos; +}; + +#define RANDOMFAIL_FS(ptr) container_of((ptr), struct randomfail_fs, fs) +#define RANDOMFAIL_FILE(ptr) container_of((ptr), struct randomfail_fs_file, file) +#define RANDOMFAIL_ITER(ptr) container_of((ptr), struct randomfail_fs_iter, iter) + +static struct fs *fs_randomfail_alloc(void) +{ + struct randomfail_fs *fs; + + fs = i_new(struct randomfail_fs, 1); + fs->fs = fs_class_randomfail; + return &fs->fs; +} + +static bool fs_op_find(const char *str, enum fs_op *op_r) +{ + enum fs_op op; + + for (op = 0; op < FS_OP_COUNT; op++) { + if (strcmp(fs_op_names[op], str) == 0) { + *op_r = op; + return TRUE; + } + } + return FALSE; +} + +static int +fs_randomfail_add_probability(struct randomfail_fs *fs, + const char *key, const char *value, + const char **error_r) +{ + unsigned int num; + enum fs_op op; + bool invalid_value = FALSE; + + if (str_to_uint(value, &num) < 0 || num > 100) + invalid_value = TRUE; + if (fs_op_find(key, &op)) { + if (invalid_value) { + *error_r = "Invalid probability value"; + return -1; + } + fs->op_probability[op] = num; + return 1; + } + if (strcmp(key, "all") == 0) { + if (invalid_value) { + *error_r = "Invalid probability value"; + return -1; + } + for (op = 0; op < FS_OP_COUNT; op++) + fs->op_probability[op] = num; + return 1; + } + return 0; +} + +static int +fs_randomfail_add_probability_range(struct randomfail_fs *fs, + const char *key, const char *value, + const char **error_r) +{ + enum fs_op op; + const char *p; + uoff_t num1, num2; + + if (strcmp(key, "read-range") == 0) + op = FS_OP_READ; + else if (strcmp(key, "write-range") == 0) + op = FS_OP_WRITE; + else if (strcmp(key, "iter-range") == 0) + op = FS_OP_ITER; + else + return 0; + + p = strchr(value, '-'); + if (p == NULL) { + if (str_to_uoff(value, &num1) < 0) { + *error_r = "Invalid range value"; + return -1; + } + num2 = num1; + } else if (str_to_uoff(t_strdup_until(value, p), &num1) < 0 || + str_to_uoff(p+1, &num2) < 0 || num1 > num2) { + *error_r = "Invalid range values"; + return -1; + } + fs->range_start[op] = num1; + fs->range_end[op] = num2; + return 1; +} + +static int fs_randomfail_parse_params(struct randomfail_fs *fs, + const char *params, const char **error_r) +{ + const char *const *tmp; + int ret; + + for (tmp = t_strsplit_spaces(params, ","); *tmp != NULL; tmp++) { + const char *key = *tmp; + const char *value = strchr(key, '='); + + if (value == NULL) { + *error_r = "Missing '='"; + return -1; + } + key = t_strdup_until(key, value++); + if ((ret = fs_randomfail_add_probability(fs, key, value, error_r)) != 0) { + if (ret < 0) + return -1; + continue; + } + if ((ret = fs_randomfail_add_probability_range(fs, key, value, error_r)) != 0) { + if (ret < 0) + return -1; + continue; + } + *error_r = t_strdup_printf("Unknown key '%s'", key); + return -1; + } + return 0; +} + +static int +fs_randomfail_init(struct fs *_fs, const char *args, + const struct fs_settings *set, const char **error_r) +{ + struct randomfail_fs *fs = RANDOMFAIL_FS(_fs); + const char *p, *parent_name, *parent_args, *error; + + p = strchr(args, ':'); + if (p == NULL) { + *error_r = "Randomfail parameters missing"; + return -1; + } + if (fs_randomfail_parse_params(fs, t_strdup_until(args, p++), &error) < 0) { + *error_r = t_strdup_printf( + "Invalid randomfail parameters: %s", error); + return -1; + } + args = p; + + if (*args == '\0') { + *error_r = "Parent filesystem not given as parameter"; + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; + } + if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0) + return -1; + return 0; +} + +static void fs_randomfail_free(struct fs *_fs) +{ + struct randomfail_fs *fs = RANDOMFAIL_FS(_fs); + + i_free(fs); +} + +static enum fs_properties fs_randomfail_get_properties(struct fs *_fs) +{ + return fs_get_properties(_fs->parent); +} + +static struct fs_file *fs_randomfail_file_alloc(void) +{ + struct randomfail_fs_file *file = i_new(struct randomfail_fs_file, 1); + return &file->file; +} + +static void +fs_randomfail_file_init(struct fs_file *_file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + + file->file.path = i_strdup(path); + file->file.parent = fs_file_init_parent(_file, path, mode, flags); +} + +static void fs_randomfail_file_deinit(struct fs_file *_file) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + + fs_file_free(_file); + i_free(file->file.path); + i_free(file); +} + +static bool fs_random_fail(struct fs *_fs, struct event *event, + int divider, enum fs_op op) +{ + struct randomfail_fs *fs = RANDOMFAIL_FS(_fs); + + if (fs->op_probability[op] == 0) + return FALSE; + if ((unsigned int)i_rand_limit(100 * divider) <= fs->op_probability[op]) { + fs_set_error(event, EIO, RANDOMFAIL_ERROR); + return TRUE; + } + return FALSE; +} + +static bool +fs_file_random_fail_begin(struct randomfail_fs_file *file, enum fs_op op) +{ + if (!file->op_pending[op]) { + if (fs_random_fail(file->file.fs, file->file.event, 2, op)) + return TRUE; + } + file->op_pending[op] = TRUE; + return FALSE; +} + +static int +fs_file_random_fail_end(struct randomfail_fs_file *file, + int ret, enum fs_op op) +{ + if (ret == 0 || errno != EAGAIN) { + if (fs_random_fail(file->file.fs, file->file.event, 2, op)) + return -1; + file->op_pending[op] = FALSE; + } + return ret; +} + +static bool +fs_random_fail_range(struct fs *_fs, struct event *event, + enum fs_op op, uoff_t *offset_r) +{ + struct randomfail_fs *fs = RANDOMFAIL_FS(_fs); + + if (!fs_random_fail(_fs, event, 1, op)) + return FALSE; + *offset_r = i_rand_minmax(fs->range_start[op], fs->range_end[op]); + return TRUE; +} + +static int +fs_randomfail_get_metadata(struct fs_file *_file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + int ret; + + if (fs_file_random_fail_begin(file, FS_OP_METADATA)) + return -1; + ret = fs_get_metadata_full(_file->parent, flags, metadata_r); + return fs_file_random_fail_end(file, ret, FS_OP_METADATA); +} + +static bool fs_randomfail_prefetch(struct fs_file *_file, uoff_t length) +{ + if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_PREFETCH)) + return TRUE; + return fs_prefetch(_file->parent, length); +} + +static ssize_t fs_randomfail_read(struct fs_file *_file, void *buf, size_t size) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + int ret; + + if (fs_file_random_fail_begin(file, FS_OP_READ)) + return -1; + ret = fs_read(_file->parent, buf, size); + if (fs_file_random_fail_end(file, ret < 0 ? -1 : 0, FS_OP_READ) < 0) + return -1; + return ret; +} + +static struct istream * +fs_randomfail_read_stream(struct fs_file *_file, size_t max_buffer_size) +{ + struct istream *input, *input2; + uoff_t offset; + + input = fs_read_stream(_file->parent, max_buffer_size); + if (!fs_random_fail_range(_file->fs, _file->event, FS_OP_READ, &offset)) + return input; + input2 = i_stream_create_failure_at(input, offset, EIO, RANDOMFAIL_ERROR); + i_stream_unref(&input); + return input2; +} + +static int fs_randomfail_write(struct fs_file *_file, const void *data, size_t size) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + int ret; + + if (fs_file_random_fail_begin(file, FS_OP_WRITE)) + return -1; + ret = fs_write(_file->parent, data, size); + return fs_file_random_fail_end(file, ret, FS_OP_EXISTS); +} + +static void fs_randomfail_write_stream(struct fs_file *_file) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + uoff_t offset; + + i_assert(_file->output == NULL); + + file->super_output = fs_write_stream(_file->parent); + if (!fs_random_fail_range(_file->fs, _file->event, FS_OP_WRITE, &offset)) + _file->output = file->super_output; + else { + _file->output = o_stream_create_failure_at(file->super_output, offset, + RANDOMFAIL_ERROR); + } +} + +static int fs_randomfail_write_stream_finish(struct fs_file *_file, bool success) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + + if (_file->output != NULL) { + if (_file->output == file->super_output) + _file->output = NULL; + else + o_stream_unref(&_file->output); + if (!success) { + fs_write_stream_abort_parent(_file, &file->super_output); + return -1; + } + if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_WRITE)) { + fs_write_stream_abort_error(_file->parent, &file->super_output, RANDOMFAIL_ERROR); + return -1; + } + } + return fs_write_stream_finish(_file->parent, &file->super_output); +} + +static int +fs_randomfail_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lock_r) +{ + if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_LOCK)) + return -1; + return fs_lock(_file->parent, secs, lock_r); +} + +static void fs_randomfail_unlock(struct fs_lock *_lock ATTR_UNUSED) +{ + i_unreached(); +} + +static int fs_randomfail_exists(struct fs_file *_file) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + int ret; + + if (fs_file_random_fail_begin(file, FS_OP_EXISTS)) + return -1; + ret = fs_exists(_file->parent); + return fs_file_random_fail_end(file, ret, FS_OP_EXISTS); +} + +static int fs_randomfail_stat(struct fs_file *_file, struct stat *st_r) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + int ret; + + if (fs_file_random_fail_begin(file, FS_OP_STAT)) + return -1; + ret = fs_stat(_file->parent, st_r); + return fs_file_random_fail_end(file, ret, FS_OP_STAT); +} + +static int fs_randomfail_get_nlinks(struct fs_file *_file, nlink_t *nlinks_r) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + int ret; + + if (fs_file_random_fail_begin(file, FS_OP_STAT)) + return -1; + ret = fs_get_nlinks(_file->parent, nlinks_r); + return fs_file_random_fail_end(file, ret, FS_OP_STAT); +} + +static int fs_randomfail_copy(struct fs_file *_src, struct fs_file *_dest) +{ + struct randomfail_fs_file *dest = RANDOMFAIL_FILE(_dest); + int ret; + + if (fs_file_random_fail_begin(dest, FS_OP_COPY)) + return -1; + + if (_src != NULL) + ret = fs_copy(_src->parent, _dest->parent); + else + ret = fs_copy_finish_async(_dest->parent); + return fs_file_random_fail_end(dest, ret, FS_OP_COPY); +} + +static int fs_randomfail_rename(struct fs_file *_src, struct fs_file *_dest) +{ + struct randomfail_fs_file *dest = RANDOMFAIL_FILE(_dest); + int ret; + + if (fs_file_random_fail_begin(dest, FS_OP_RENAME)) + return -1; + ret = fs_rename(_src->parent, _dest->parent); + return fs_file_random_fail_end(dest, ret, FS_OP_RENAME); +} + +static int fs_randomfail_delete(struct fs_file *_file) +{ + struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file); + int ret; + + if (fs_file_random_fail_begin(file, FS_OP_DELETE)) + return -1; + ret = fs_delete(_file->parent); + return fs_file_random_fail_end(file, ret, FS_OP_DELETE); +} + +static struct fs_iter *fs_randomfail_iter_alloc(void) +{ + struct randomfail_fs_iter *iter = i_new(struct randomfail_fs_iter, 1); + return &iter->iter; +} + +static void +fs_randomfail_iter_init(struct fs_iter *_iter, const char *path, + enum fs_iter_flags flags) +{ + struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter); + uoff_t pos; + + iter->super = fs_iter_init_parent(_iter, path, flags); + if (fs_random_fail_range(_iter->fs, _iter->event, FS_OP_ITER, &pos)) + iter->fail_pos = pos + 1; +} + +static const char *fs_randomfail_iter_next(struct fs_iter *_iter) +{ + struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter); + const char *fname; + + if (iter->fail_pos > 0) { + if (iter->fail_pos == 1) + return NULL; + iter->fail_pos--; + } + + iter->super->async_callback = _iter->async_callback; + iter->super->async_context = _iter->async_context; + + fname = fs_iter_next(iter->super); + _iter->async_have_more = iter->super->async_have_more; + return fname; +} + +static int fs_randomfail_iter_deinit(struct fs_iter *_iter) +{ + struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter); + const char *error; + int ret; + + if ((ret = fs_iter_deinit(&iter->super, &error)) < 0) + fs_set_error_errno(_iter->event, "%s", error); + if (iter->fail_pos == 1) { + fs_set_error(_iter->event, EIO, RANDOMFAIL_ERROR); + ret = -1; + } + return ret; +} + +const struct fs fs_class_randomfail = { + .name = "randomfail", + .v = { + fs_randomfail_alloc, + fs_randomfail_init, + NULL, + fs_randomfail_free, + fs_randomfail_get_properties, + fs_randomfail_file_alloc, + fs_randomfail_file_init, + fs_randomfail_file_deinit, + fs_wrapper_file_close, + fs_wrapper_file_get_path, + fs_wrapper_set_async_callback, + fs_wrapper_wait_async, + fs_wrapper_set_metadata, + fs_randomfail_get_metadata, + fs_randomfail_prefetch, + fs_randomfail_read, + fs_randomfail_read_stream, + fs_randomfail_write, + fs_randomfail_write_stream, + fs_randomfail_write_stream_finish, + fs_randomfail_lock, + fs_randomfail_unlock, + fs_randomfail_exists, + fs_randomfail_stat, + fs_randomfail_copy, + fs_randomfail_rename, + fs_randomfail_delete, + fs_randomfail_iter_alloc, + fs_randomfail_iter_init, + fs_randomfail_iter_next, + fs_randomfail_iter_deinit, + NULL, + fs_randomfail_get_nlinks, + } +}; diff --git a/src/lib-fs/fs-sis-common.c b/src/lib-fs/fs-sis-common.c new file mode 100644 index 0000000..654c44e --- /dev/null +++ b/src/lib-fs/fs-sis-common.c @@ -0,0 +1,59 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "fs-sis-common.h" + +#include <sys/stat.h> + +int fs_sis_path_parse(struct fs_file *file, const char *path, + const char **dir_r, const char **hash_r) +{ + const char *fname, *p; + + fname = strrchr(path, '/'); + if (fname == NULL) { + *dir_r = "."; + fname = path; + } else { + *dir_r = t_strdup_until(path, fname); + fname++; + } + + /* assume filename begins with "<hash>-" */ + p = strchr(fname, '-'); + if (p == NULL) { + fs_set_error(file->event, EINVAL, "open(%s) failed: " + "Filenames must begin with '<hash>-'", path); + return -1; + } + *hash_r = t_strdup_until(fname, p); + return 0; +} + +void fs_sis_try_unlink_hash_file(struct fs_file *sis_file, + struct fs_file *super_file) +{ + struct fs_file *hash_file; + struct stat st1, st2; + const char *dir, *hash, *hash_path; + + if (fs_sis_path_parse(sis_file, super_file->path, &dir, &hash) == 0 && + fs_stat(super_file, &st1) == 0 && st1.st_nlink == 2) { + /* this may be the last link. if hashes/ file is the same, + delete it. */ + hash_path = t_strdup_printf("%s/"HASH_DIR_NAME"/%s", dir, hash); + hash_file = fs_file_init_with_event(super_file->fs, + super_file->event, hash_path, + FS_OPEN_MODE_READONLY); + if (fs_stat(hash_file, &st2) == 0 && + st1.st_ino == st2.st_ino && + CMP_DEV_T(st1.st_dev, st2.st_dev)) { + if (fs_delete(hash_file) < 0) { + e_error(hash_file->event, "%s", + fs_file_last_error(hash_file)); + } + } + fs_file_deinit(&hash_file); + } +} + diff --git a/src/lib-fs/fs-sis-common.h b/src/lib-fs/fs-sis-common.h new file mode 100644 index 0000000..5156ca6 --- /dev/null +++ b/src/lib-fs/fs-sis-common.h @@ -0,0 +1,14 @@ +#ifndef FS_SIS_COMMON_H +#define FS_SIS_COMMON_H + +#include "fs-api-private.h" + +#define HASH_DIR_NAME "hashes" + +int fs_sis_path_parse(struct fs_file *file, const char *path, + const char **dir_r, const char **hash_r); +void fs_sis_try_unlink_hash_file(struct fs_file *sis_file, + struct fs_file *super_file); + +#endif + diff --git a/src/lib-fs/fs-sis-queue.c b/src/lib-fs/fs-sis-queue.c new file mode 100644 index 0000000..8ad7d98 --- /dev/null +++ b/src/lib-fs/fs-sis-queue.c @@ -0,0 +1,210 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "fs-sis-common.h" + +#define QUEUE_DIR_NAME "queue" + +struct sis_queue_fs { + struct fs fs; + char *queue_dir; +}; + +struct sis_queue_fs_file { + struct fs_file file; + struct sis_queue_fs *fs; +}; + +#define SISQUEUE_FS(ptr) container_of((ptr), struct sis_queue_fs, fs) +#define SISQUEUE_FILE(ptr) container_of((ptr), struct sis_queue_fs_file, file) + +static struct fs *fs_sis_queue_alloc(void) +{ + struct sis_queue_fs *fs; + + fs = i_new(struct sis_queue_fs, 1); + fs->fs = fs_class_sis_queue; + return &fs->fs; +} + +static int +fs_sis_queue_init(struct fs *_fs, const char *args, + const struct fs_settings *set, const char **error_r) +{ + struct sis_queue_fs *fs = SISQUEUE_FS(_fs); + const char *p, *parent_name, *parent_args; + + /* <queue_dir>:<parent fs>[:<args>] */ + + p = strchr(args, ':'); + if (p == NULL || p[1] == '\0') { + *error_r = "Parent filesystem not given as parameter"; + return -1; + } + + fs->queue_dir = i_strdup_until(args, p); + parent_name = p + 1; + + parent_args = strchr(parent_name, ':'); + if (parent_args == NULL) + parent_args = ""; + else + parent_name = t_strdup_until(parent_name, parent_args++); + if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0) + return -1; + return 0; +} + +static void fs_sis_queue_free(struct fs *_fs) +{ + struct sis_queue_fs *fs = SISQUEUE_FS(_fs); + + i_free(fs->queue_dir); + i_free(fs); +} + +static struct fs_file *fs_sis_queue_file_alloc(void) +{ + struct sis_queue_fs_file *file = i_new(struct sis_queue_fs_file, 1); + return &file->file; +} + +static void +fs_sis_queue_file_init(struct fs_file *_file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct sis_queue_fs_file *file = SISQUEUE_FILE(_file); + struct sis_queue_fs *fs = SISQUEUE_FS(_file->fs); + + file->file.path = i_strdup(path); + file->fs = fs; + + if (mode == FS_OPEN_MODE_APPEND) + fs_set_error(_file->event, ENOTSUP, "APPEND mode not supported"); + else + file->file.parent = fs_file_init_parent(_file, path, mode, flags); +} + +static void fs_sis_queue_file_deinit(struct fs_file *_file) +{ + struct sis_queue_fs_file *file = SISQUEUE_FILE(_file); + + fs_file_free(_file); + i_free(file->file.path); + i_free(file); +} + +static void fs_sis_queue_add(struct sis_queue_fs_file *file) +{ + struct sis_queue_fs *fs = SISQUEUE_FS(file->file.fs); + struct fs_file *queue_file; + const char *fname, *path, *queue_path; + + path = fs_file_path(&file->file); + fname = strrchr(path, '/'); + if (fname != NULL) + fname++; + else + fname = path; + + queue_path = t_strdup_printf("%s/%s", fs->queue_dir, fname); + queue_file = fs_file_init_parent(&file->file, queue_path, FS_OPEN_MODE_CREATE, 0); + if (fs_write(queue_file, "", 0) < 0 && errno != EEXIST) + e_error(file->file.event, "%s", fs_file_last_error(queue_file)); + fs_file_deinit(&queue_file); +} + +static int fs_sis_queue_write(struct fs_file *_file, const void *data, size_t size) +{ + struct sis_queue_fs_file *file = SISQUEUE_FILE(_file); + + if (_file->parent == NULL) + return -1; + if (fs_write(_file->parent, data, size) < 0) + return -1; + T_BEGIN { + fs_sis_queue_add(file); + } T_END; + return 0; +} + +static void fs_sis_queue_write_stream(struct fs_file *_file) +{ + i_assert(_file->output == NULL); + + if (_file->parent == NULL) { + _file->output = o_stream_create_error_str(EINVAL, "%s", + fs_file_last_error(_file)); + } else { + _file->output = fs_write_stream(_file->parent); + } + o_stream_set_name(_file->output, _file->path); +} + +static int fs_sis_queue_write_stream_finish(struct fs_file *_file, bool success) +{ + struct sis_queue_fs_file *file = SISQUEUE_FILE(_file); + + if (!success) { + if (_file->parent != NULL) + fs_write_stream_abort_parent(_file, &_file->output); + return -1; + } + + if (fs_write_stream_finish(_file->parent, &_file->output) < 0) + return -1; + T_BEGIN { + fs_sis_queue_add(file); + } T_END; + return 1; +} + +static int fs_sis_queue_delete(struct fs_file *_file) +{ + T_BEGIN { + fs_sis_try_unlink_hash_file(_file, _file->parent); + } T_END; + return fs_delete(_file->parent); +} + +const struct fs fs_class_sis_queue = { + .name = "sis-queue", + .v = { + fs_sis_queue_alloc, + fs_sis_queue_init, + NULL, + fs_sis_queue_free, + fs_wrapper_get_properties, + fs_sis_queue_file_alloc, + fs_sis_queue_file_init, + fs_sis_queue_file_deinit, + fs_wrapper_file_close, + fs_wrapper_file_get_path, + fs_wrapper_set_async_callback, + fs_wrapper_wait_async, + fs_wrapper_set_metadata, + fs_wrapper_get_metadata, + fs_wrapper_prefetch, + fs_wrapper_read, + fs_wrapper_read_stream, + fs_sis_queue_write, + fs_sis_queue_write_stream, + fs_sis_queue_write_stream_finish, + fs_wrapper_lock, + fs_wrapper_unlock, + fs_wrapper_exists, + fs_wrapper_stat, + fs_wrapper_copy, + fs_wrapper_rename, + fs_sis_queue_delete, + fs_wrapper_iter_alloc, + fs_wrapper_iter_init, + NULL, + NULL, + NULL, + fs_wrapper_get_nlinks, + } +}; diff --git a/src/lib-fs/fs-sis.c b/src/lib-fs/fs-sis.c new file mode 100644 index 0000000..6a020bc --- /dev/null +++ b/src/lib-fs/fs-sis.c @@ -0,0 +1,369 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "ostream-cmp.h" +#include "fs-sis-common.h" + +#define FS_SIS_REQUIRED_PROPS \ + (FS_PROPERTY_FASTCOPY | FS_PROPERTY_STAT) + +struct sis_fs { + struct fs fs; +}; + +struct sis_fs_file { + struct fs_file file; + struct sis_fs *fs; + enum fs_open_mode open_mode; + + struct fs_file *hash_file; + struct istream *hash_input; + struct ostream *fs_output; + + char *hash, *hash_path; + bool opened; +}; + +#define SIS_FS(ptr) container_of((ptr), struct sis_fs, fs) +#define SIS_FILE(ptr) container_of((ptr), struct sis_fs_file, file) + +static struct fs *fs_sis_alloc(void) +{ + struct sis_fs *fs; + + fs = i_new(struct sis_fs, 1); + fs->fs = fs_class_sis; + return &fs->fs; +} + +static int +fs_sis_init(struct fs *_fs, const char *args, const struct fs_settings *set, + const char **error_r) +{ + enum fs_properties props; + const char *parent_name, *parent_args; + + if (*args == '\0') { + *error_r = "Parent filesystem not given as parameter"; + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; + } + if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0) + return -1; + props = fs_get_properties(_fs->parent); + if ((props & FS_SIS_REQUIRED_PROPS) != FS_SIS_REQUIRED_PROPS) { + *error_r = t_strdup_printf("%s backend can't be used with SIS", + parent_name); + return -1; + } + return 0; +} + +static void fs_sis_free(struct fs *_fs) +{ + struct sis_fs *fs = SIS_FS(_fs); + + i_free(fs); +} + +static struct fs_file *fs_sis_file_alloc(void) +{ + struct sis_fs_file *file = i_new(struct sis_fs_file, 1); + return &file->file; +} + +static void +fs_sis_file_init(struct fs_file *_file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct sis_fs_file *file = SIS_FILE(_file); + struct sis_fs *fs = SIS_FS(_file->fs); + const char *dir, *hash; + + file->file.path = i_strdup(path); + file->fs = fs; + file->open_mode = mode; + if (mode == FS_OPEN_MODE_APPEND) { + fs_set_error(_file->event, ENOTSUP, "APPEND mode not supported"); + return; + } + + if (fs_sis_path_parse(_file, path, &dir, &hash) < 0) + return; + + /* if hashes/<hash> already exists, open it */ + file->hash_path = i_strdup_printf("%s/"HASH_DIR_NAME"/%s", dir, hash); + file->hash_file = fs_file_init_parent(_file, file->hash_path, + FS_OPEN_MODE_READONLY, 0); + + file->hash_input = fs_read_stream(file->hash_file, IO_BLOCK_SIZE); + if (i_stream_read(file->hash_input) == -1) { + /* doesn't exist */ + if (errno != ENOENT) { + e_error(file->file.event, "Couldn't read hash file %s: %m", + file->hash_path); + } + i_stream_destroy(&file->hash_input); + } + + file->file.parent = fs_file_init_parent(_file, path, mode, flags); +} + +static void fs_sis_file_deinit(struct fs_file *_file) +{ + struct sis_fs_file *file = SIS_FILE(_file); + + fs_file_deinit(&file->hash_file); + fs_file_free(_file); + i_free(file->hash); + i_free(file->hash_path); + i_free(file->file.path); + i_free(file); +} + +static void fs_sis_file_close(struct fs_file *_file) +{ + struct sis_fs_file *file = SIS_FILE(_file); + + i_stream_unref(&file->hash_input); + fs_file_close(file->hash_file); + fs_file_close(_file->parent); +} + +static bool fs_sis_try_link(struct sis_fs_file *file) +{ + const struct stat *st; + struct stat st2; + + if (i_stream_stat(file->hash_input, FALSE, &st) < 0) + return FALSE; + + /* we can use the existing file */ + if (fs_copy(file->hash_file, file->file.parent) < 0) { + if (errno != ENOENT && errno != EMLINK) { + e_error(file->file.event, "%s", + fs_file_last_error(file->hash_file)); + } + /* failed to use link(), continue as if it hadn't been equal */ + return FALSE; + } + if (fs_stat(file->file.parent, &st2) < 0) { + e_error(file->file.event, "%s", + fs_file_last_error(file->file.parent)); + if (fs_delete(file->file.parent) < 0) { + e_error(file->file.event, "%s", + fs_file_last_error(file->file.parent)); + } + return FALSE; + } + if (st->st_ino != st2.st_ino) { + /* the hashes/ file was already replaced with something else */ + if (fs_delete(file->file.parent) < 0) { + e_error(file->file.event, "%s", + fs_file_last_error(file->file.parent)); + } + return FALSE; + } + return TRUE; +} + +static void fs_sis_replace_hash_file(struct sis_fs_file *file) +{ + struct fs *super_fs = file->file.parent->fs; + struct fs_file *temp_file; + const char *hash_fname; + string_t *temp_path; + int ret; + + if (file->hash_input == NULL) { + /* hash file didn't exist previously. we should be able to + create it with link() */ + if (fs_copy(file->file.parent, file->hash_file) < 0) { + if (errno == EEXIST) { + /* the file was just created. it's probably + a duplicate, but it's too much trouble + trying to deduplicate it anymore */ + } else { + e_error(file->file.event, "%s", + fs_file_last_error(file->hash_file)); + } + } + return; + } + + temp_path = t_str_new(256); + hash_fname = strrchr(file->hash_path, '/'); + if (hash_fname == NULL) + hash_fname = file->hash_path; + else { + str_append_data(temp_path, file->hash_path, + (hash_fname-file->hash_path) + 1); + hash_fname++; + } + str_printfa(temp_path, "%s%s.tmp", + super_fs->set.temp_file_prefix, hash_fname); + + /* replace existing hash file atomically */ + temp_file = fs_file_init_parent(&file->file, str_c(temp_path), + FS_OPEN_MODE_READONLY, 0); + ret = fs_copy(file->file.parent, temp_file); + if (ret < 0 && errno == EEXIST) { + /* either someone's racing us or it's a stale file. + try to continue. */ + if (fs_delete(temp_file) < 0 && + errno != ENOENT) + e_error(file->file.event, "%s", fs_file_last_error(temp_file)); + ret = fs_copy(file->file.parent, temp_file); + } + if (ret < 0) { + e_error(file->file.event, "%s", fs_file_last_error(temp_file)); + fs_file_deinit(&temp_file); + return; + } + + if (fs_rename(temp_file, file->hash_file) < 0) { + if (errno == ENOENT) { + /* apparently someone else just renamed it. ignore. */ + } else { + e_error(file->file.event, "%s", + fs_file_last_error(file->hash_file)); + } + (void)fs_delete(temp_file); + } + fs_file_deinit(&temp_file); +} + +static int fs_sis_write(struct fs_file *_file, const void *data, size_t size) +{ + struct sis_fs_file *file = SIS_FILE(_file); + + if (_file->parent == NULL) + return -1; + + if (file->hash_input != NULL && + stream_cmp_block(file->hash_input, data, size) && + i_stream_read_eof(file->hash_input)) { + /* try to use existing file */ + if (fs_sis_try_link(file)) + return 0; + } + + if (fs_write(_file->parent, data, size) < 0) + return -1; + T_BEGIN { + fs_sis_replace_hash_file(file); + } T_END; + return 0; +} + +static void fs_sis_write_stream(struct fs_file *_file) +{ + struct sis_fs_file *file = SIS_FILE(_file); + + i_assert(_file->output == NULL); + + if (_file->parent == NULL) { + _file->output = o_stream_create_error_str(EINVAL, "%s", + fs_file_last_error(_file)); + } else { + file->fs_output = fs_write_stream(_file->parent); + if (file->hash_input == NULL) { + _file->output = file->fs_output; + o_stream_ref(_file->output); + } else { + /* compare if files are equal */ + _file->output = o_stream_create_cmp(file->fs_output, + file->hash_input); + } + } + o_stream_set_name(_file->output, _file->path); +} + +static int fs_sis_write_stream_finish(struct fs_file *_file, bool success) +{ + struct sis_fs_file *file = SIS_FILE(_file); + + if (!success) { + if (_file->parent != NULL) + fs_write_stream_abort_parent(_file, &file->fs_output); + o_stream_unref(&_file->output); + return -1; + } + + if (file->hash_input != NULL && + o_stream_cmp_equals(_file->output) && + i_stream_read_eof(file->hash_input)) { + o_stream_unref(&_file->output); + if (fs_sis_try_link(file)) { + fs_write_stream_abort_parent(_file, &file->fs_output); + return 1; + } + } + if (_file->output != NULL) + o_stream_unref(&_file->output); + + if (fs_write_stream_finish(_file->parent, &file->fs_output) < 0) + return -1; + T_BEGIN { + fs_sis_replace_hash_file(file); + } T_END; + return 1; +} + +static int fs_sis_delete(struct fs_file *_file) +{ + T_BEGIN { + fs_sis_try_unlink_hash_file(_file, _file->parent); + } T_END; + return fs_delete(_file->parent); +} + +const struct fs fs_class_sis = { + .name = "sis", + .v = { + fs_sis_alloc, + fs_sis_init, + NULL, + fs_sis_free, + fs_wrapper_get_properties, + fs_sis_file_alloc, + fs_sis_file_init, + fs_sis_file_deinit, + fs_sis_file_close, + fs_wrapper_file_get_path, + fs_wrapper_set_async_callback, + fs_wrapper_wait_async, + fs_wrapper_set_metadata, + fs_wrapper_get_metadata, + fs_wrapper_prefetch, + fs_wrapper_read, + fs_wrapper_read_stream, + fs_sis_write, + fs_sis_write_stream, + fs_sis_write_stream_finish, + fs_wrapper_lock, + fs_wrapper_unlock, + fs_wrapper_exists, + fs_wrapper_stat, + fs_wrapper_copy, + fs_wrapper_rename, + fs_sis_delete, + fs_wrapper_iter_alloc, + fs_wrapper_iter_init, + NULL, + NULL, + NULL, + fs_wrapper_get_nlinks, + } +}; diff --git a/src/lib-fs/fs-test-async.c b/src/lib-fs/fs-test-async.c new file mode 100644 index 0000000..a8cd3f5 --- /dev/null +++ b/src/lib-fs/fs-test-async.c @@ -0,0 +1,103 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "ostream.h" +#include "fs-test.h" +#include "test-common.h" + +static void test_fs_async_write(const char *test_name, struct fs *fs) +{ + struct fs_file *file; + struct test_fs_file *test_file; + struct ostream *output; + unsigned int i; + int ret; + + test_begin(t_strdup_printf("%s: async write", test_name)); + for (i = 0; i < 3; i++) { + file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE | + FS_OPEN_FLAG_ASYNC); + output = fs_write_stream(file); + + o_stream_nsend_str(output, "12345"); + if (i < 2) { + test_assert(fs_write_stream_finish(file, &output) == 0); + test_assert(output == NULL); + test_assert(fs_write_stream_finish_async(file) == 0); + } + + test_file = test_fs_file_get(fs, "foo"); + test_file->wait_async = FALSE; + + switch (i) { + case 0: + while ((ret = fs_write_stream_finish_async(file)) == 0) + fs_wait_async(fs); + test_assert(ret > 0); + test_assert(test_file->contents->used > 0); + break; + case 1: + test_file->io_failure = TRUE; + test_assert(fs_write_stream_finish_async(file) < 0); + test_assert(test_file->contents->used == 0); + break; + case 2: + fs_write_stream_abort_error(file, &output, "test"); + test_assert(test_file->contents->used == 0); + break; + } + fs_file_deinit(&file); + } + test_end(); +} + +static void test_fs_async_copy(const char *test_name, struct fs *fs) +{ + struct fs_file *src, *dest; + struct test_fs_file *test_file; + int ret; + + test_begin(t_strdup_printf("%s: async copy", test_name)); + + src = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE); + test_assert(fs_write(src, "source", 6) == 0); + + dest = fs_file_init(fs, "bar", FS_OPEN_MODE_REPLACE | + FS_OPEN_FLAG_ASYNC); + + test_assert(fs_copy(src, dest) == -1 && errno == EAGAIN); + + test_file = test_fs_file_get(fs, "bar"); + test_file->wait_async = FALSE; + + while ((ret = fs_copy_finish_async(dest)) < 0 && errno == EAGAIN) + fs_wait_async(fs); + test_assert(ret == 0); + test_assert(test_file->contents->used > 0); + fs_file_deinit(&dest); + + fs_file_deinit(&src); + test_end(); +} + +void test_fs_async(const char *test_name, enum fs_properties properties, + const char *driver, const char *args) +{ + struct fs_settings fs_set; + struct fs *fs; + struct test_fs *test_fs; + const char *error; + + i_zero(&fs_set); + if (fs_init(driver, args, &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + + test_fs = test_fs_get(fs); + test_fs->properties = properties; + + test_fs_async_write(test_name, fs); + test_fs_async_copy(test_name, fs); + + fs_deinit(&fs); +} diff --git a/src/lib-fs/fs-test.c b/src/lib-fs/fs-test.c new file mode 100644 index 0000000..d352718 --- /dev/null +++ b/src/lib-fs/fs-test.c @@ -0,0 +1,443 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "ostream.h" +#include "test-common.h" +#include "fs-test.h" + +static struct fs *fs_test_alloc(void) +{ + struct test_fs *fs; + + fs = i_new(struct test_fs, 1); + fs->fs = fs_class_test; + i_array_init(&fs->iter_files, 32); + return &fs->fs; +} + +static int +fs_test_init(struct fs *_fs ATTR_UNUSED, const char *args ATTR_UNUSED, + const struct fs_settings *set ATTR_UNUSED, + const char **error_r ATTR_UNUSED) +{ + return 0; +} + +static void fs_test_free(struct fs *_fs) +{ + struct test_fs *fs = (struct test_fs *)_fs; + + array_free(&fs->iter_files); + i_free(fs); +} + +static enum fs_properties fs_test_get_properties(struct fs *_fs) +{ + struct test_fs *fs = (struct test_fs *)_fs; + + return fs->properties; +} + +static struct fs_file *fs_test_file_alloc(void) +{ + struct test_fs_file *file = i_new(struct test_fs_file, 1); + return &file->file; +} + +static void +fs_test_file_init(struct fs_file *_file, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + file->file.path = i_strdup(path); + file->file.flags = flags; + file->mode = mode; + file->contents = buffer_create_dynamic(default_pool, 1024); + file->exists = TRUE; + file->seekable = TRUE; + file->wait_async = (flags & FS_OPEN_FLAG_ASYNC) != 0; +} + +static void fs_test_file_deinit(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + fs_file_free(_file); + buffer_free(&file->contents); + i_free(file->file.path); + i_free(file); +} + +static void fs_test_file_close(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + file->closed = TRUE; +} + +static const char *fs_test_file_get_path(struct fs_file *_file) +{ + return _file->path; +} + +static void +fs_test_set_async_callback(struct fs_file *_file, + fs_file_async_callback_t *callback, + void *context) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + file->async_callback = callback; + file->async_context = context; +} + +static void fs_test_wait_async(struct fs *_fs ATTR_UNUSED) +{ +} + +static void +fs_test_set_metadata(struct fs_file *_file, const char *key, + const char *value) +{ + if (strcmp(key, FS_METADATA_WRITE_FNAME) == 0) { + i_free(_file->path); + _file->path = i_strdup(value); + } else { + fs_default_set_metadata(_file, key, value); + } +} + +static int +fs_test_get_metadata(struct fs_file *_file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if ((flags & FS_GET_METADATA_FLAG_LOADED_ONLY) != 0) { + *metadata_r = &_file->metadata; + return 0; + } + + if (file->wait_async) { + fs_file_set_error_async(_file); + return -1; + } + if (file->io_failure) { + errno = EIO; + return -1; + } + fs_metadata_init(_file); + *metadata_r = &_file->metadata; + return 0; +} + +static bool fs_test_prefetch(struct fs_file *_file ATTR_UNUSED, + uoff_t length ATTR_UNUSED) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + file->prefetched = TRUE; + return TRUE; +} + +static void fs_test_stream_destroyed(struct test_fs_file *file) +{ + i_assert(file->input != NULL); + file->input = NULL; +} + +static struct istream * +fs_test_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + struct istream *input; + + i_assert(file->input == NULL); + + if (!file->exists) + return i_stream_create_error(ENOENT); + if (file->io_failure) + return i_stream_create_error(EIO); + input = test_istream_create_data(file->contents->data, + file->contents->used); + i_stream_add_destroy_callback(input, fs_test_stream_destroyed, file); + if (!file->seekable) + input->seekable = FALSE; + file->input = input; + return input; +} + +static void fs_test_write_stream(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + i_assert(_file->output == NULL); + + buffer_set_used_size(file->contents, 0); + _file->output = o_stream_create_buffer(file->contents); +} + +static int fs_test_write_stream_finish(struct fs_file *_file, bool success) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + o_stream_destroy(&_file->output); + if (file->wait_async) { + fs_file_set_error_async(_file); + return 0; + } + if (file->io_failure) + success = FALSE; + if (!success) + buffer_set_used_size(file->contents, 0); + return success ? 1 : -1; +} + +static int +fs_test_lock(struct fs_file *_file, unsigned int secs ATTR_UNUSED, + struct fs_lock **lock_r) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->locked) + return 0; + file->locked = TRUE; + *lock_r = i_new(struct fs_lock, 1); + (*lock_r)->file = _file; + return 1; +} + +static void fs_test_unlock(struct fs_lock *lock) +{ + struct test_fs_file *file = (struct test_fs_file *)lock->file; + + file->locked = FALSE; + i_free(lock); +} + +static int fs_test_exists(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->wait_async) { + fs_file_set_error_async(_file); + return -1; + } + if (file->io_failure) { + errno = EIO; + return -1; + } + return file->exists ? 1 : 0; +} + +static int fs_test_stat(struct fs_file *_file, struct stat *st_r) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->wait_async) { + fs_file_set_error_async(_file); + return -1; + } + if (file->io_failure) { + errno = EIO; + return -1; + } + if (!file->exists) { + errno = ENOENT; + return -1; + } + i_zero(st_r); + st_r->st_size = file->contents->used; + return 0; +} + +static int fs_test_copy(struct fs_file *_src, struct fs_file *_dest) +{ + struct test_fs_file *src; + struct test_fs_file *dest = (struct test_fs_file *)_dest; + + if (_src != NULL) + dest->copy_src = test_fs_file_get(_src->fs, fs_file_path(_src)); + src = dest->copy_src; + if (dest->wait_async) { + fs_file_set_error_async(_dest); + return -1; + } + dest->copy_src = NULL; + + if (dest->io_failure) { + errno = EIO; + return -1; + } + if (!src->exists) { + errno = ENOENT; + return -1; + } + buffer_set_used_size(dest->contents, 0); + buffer_append_buf(dest->contents, src->contents, 0, SIZE_MAX); + dest->exists = TRUE; + return 0; +} + +static int fs_test_rename(struct fs_file *_src, struct fs_file *_dest) +{ + struct test_fs_file *src = (struct test_fs_file *)_src; + struct test_fs_file *dest = (struct test_fs_file *)_dest; + + if (src->wait_async || dest->wait_async) { + fs_file_set_error_async(_dest); + return -1; + } + + if (fs_test_copy(_src, _dest) < 0) + return -1; + src->exists = FALSE; + return 0; +} + +static int fs_test_delete(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->wait_async) { + fs_file_set_error_async(_file); + return -1; + } + + if (!file->exists) { + errno = ENOENT; + return -1; + } + return 0; +} + +static struct fs_iter *fs_test_iter_alloc(void) +{ + struct test_fs_iter *iter = i_new(struct test_fs_iter, 1); + return &iter->iter; +} + +static void +fs_test_iter_init(struct fs_iter *_iter, const char *path, + enum fs_iter_flags flags ATTR_UNUSED) +{ + struct test_fs_iter *iter = (struct test_fs_iter *)_iter; + struct test_fs *fs = (struct test_fs *)_iter->fs; + + iter->prefix = i_strdup(path); + iter->prefix_len = strlen(iter->prefix); + iter->prev_dir = i_strdup(""); + array_sort(&fs->iter_files, i_strcmp_p); +} + +static const char *fs_test_iter_next(struct fs_iter *_iter) +{ + struct test_fs_iter *iter = (struct test_fs_iter *)_iter; + struct test_fs *fs = (struct test_fs *)_iter->fs; + const char *const *files, *p; + unsigned int count; + size_t len, prev_dir_len = strlen(iter->prev_dir); + + files = array_get(&fs->iter_files, &count); + for (; iter->idx < count; iter->idx++) { + const char *fname = files[iter->idx]; + + if (strncmp(fname, iter->prefix, iter->prefix_len) != 0) + continue; + p = strrchr(fname, '/'); + if ((_iter->flags & FS_ITER_FLAG_DIRS) == 0) { + if (p == NULL) + return fname; + if (p[1] == '\0') + continue; /* dir/ */ + return p+1; + } + + if (p == NULL) + continue; + len = p - fname; + if (len == 0) + continue; + if (len == prev_dir_len && + strncmp(fname, iter->prev_dir, len) == 0) + continue; + i_free(iter->prev_dir); + iter->prev_dir = i_strndup(fname, len); + return iter->prev_dir; + } + return NULL; +} + +static int fs_test_iter_deinit(struct fs_iter *_iter) +{ + struct test_fs_iter *iter = (struct test_fs_iter *)_iter; + int ret = iter->failed ? -1 : 0; + + i_free(iter->prefix); + return ret; +} + +struct test_fs *test_fs_get(struct fs *fs) +{ + while (strcmp(fs->name, "test") != 0) { + i_assert(fs->parent != NULL); + fs = fs->parent; + } + return (struct test_fs *)fs; +} + +struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path) +{ + struct fs_file *file; + + fs = &test_fs_get(fs)->fs; + + for (file = fs->files;; file = file->next) { + i_assert(file != NULL); + if (strcmp(fs_file_path(file), path) == 0) + break; + } + return (struct test_fs_file *)file; +} + +const struct fs fs_class_test = { + .name = "test", + .v = { + fs_test_alloc, + fs_test_init, + NULL, + fs_test_free, + fs_test_get_properties, + fs_test_file_alloc, + fs_test_file_init, + fs_test_file_deinit, + fs_test_file_close, + fs_test_file_get_path, + fs_test_set_async_callback, + fs_test_wait_async, + fs_test_set_metadata, + fs_test_get_metadata, + fs_test_prefetch, + NULL, + fs_test_read_stream, + NULL, + fs_test_write_stream, + fs_test_write_stream_finish, + fs_test_lock, + fs_test_unlock, + fs_test_exists, + fs_test_stat, + fs_test_copy, + fs_test_rename, + fs_test_delete, + fs_test_iter_alloc, + fs_test_iter_init, + fs_test_iter_next, + fs_test_iter_deinit, + NULL, + NULL, + } +}; diff --git a/src/lib-fs/fs-test.h b/src/lib-fs/fs-test.h new file mode 100644 index 0000000..5a2ed14 --- /dev/null +++ b/src/lib-fs/fs-test.h @@ -0,0 +1,45 @@ +#ifndef FS_TEST_H +#define FS_TEST_H + +#include "fs-api-private.h" + +struct test_fs { + struct fs fs; + enum fs_properties properties; + ARRAY_TYPE(const_string) iter_files; +}; + +struct test_fs_file { + struct fs_file file; + enum fs_open_mode mode; + + fs_file_async_callback_t *async_callback; + void *async_context; + + buffer_t *contents; + struct istream *input; + struct test_fs_file *copy_src; + + bool prefetched; + bool locked; + bool exists; + bool seekable; + bool closed; + bool io_failure; + bool wait_async; +}; + +struct test_fs_iter { + struct fs_iter iter; + char *prefix, *prev_dir; + unsigned int prefix_len, idx; + bool failed; +}; + +struct test_fs *test_fs_get(struct fs *fs); +struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path); + +void test_fs_async(const char *test_name, enum fs_properties properties, + const char *driver, const char *args); + +#endif diff --git a/src/lib-fs/fs-wrapper.c b/src/lib-fs/fs-wrapper.c new file mode 100644 index 0000000..81d746e --- /dev/null +++ b/src/lib-fs/fs-wrapper.c @@ -0,0 +1,172 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "fs-api-private.h" +#include "ostream.h" + +struct wrapper_fs_iter { + struct fs_iter iter; + struct fs_iter *parent; +}; + +enum fs_properties fs_wrapper_get_properties(struct fs *fs) +{ + return fs_get_properties(fs->parent); +} + +void fs_wrapper_file_close(struct fs_file *file) +{ + fs_file_close(file->parent); +} + +const char *fs_wrapper_file_get_path(struct fs_file *file) +{ + return fs_file_path(file->parent); +} + +void fs_wrapper_set_async_callback(struct fs_file *file, + fs_file_async_callback_t *callback, + void *context) +{ + fs_file_set_async_callback(file->parent, *callback, context); +} + +void fs_wrapper_wait_async(struct fs *fs) +{ + fs_wait_async(fs->parent); +} + +void fs_wrapper_set_metadata(struct fs_file *file, const char *key, + const char *value) +{ + fs_set_metadata(file->parent, key, value); +} + +int fs_wrapper_get_metadata(struct fs_file *file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + return fs_get_metadata_full(file->parent, flags, metadata_r); +} + +bool fs_wrapper_prefetch(struct fs_file *file, uoff_t length) +{ + return fs_prefetch(file->parent, length); +} + +ssize_t fs_wrapper_read(struct fs_file *file, void *buf, size_t size) +{ + return fs_read(file->parent, buf, size); +} + +struct istream * +fs_wrapper_read_stream(struct fs_file *file, size_t max_buffer_size) +{ + return fs_read_stream(file->parent, max_buffer_size); +} + +int fs_wrapper_write(struct fs_file *file, const void *data, size_t size) +{ + return fs_write(file->parent, data, size); +} + +void fs_wrapper_write_stream(struct fs_file *file) +{ + i_assert(file->output == NULL); + + file->output = fs_write_stream(file->parent); +} + +int fs_wrapper_write_stream_finish(struct fs_file *file, bool success) +{ + if (file->output == NULL) + return fs_write_stream_finish_async(file->parent); + + if (!success) { + fs_write_stream_abort_parent(file, &file->output); + return -1; + } + return fs_write_stream_finish(file->parent, &file->output); +} + +int fs_wrapper_lock(struct fs_file *file, unsigned int secs, + struct fs_lock **lock_r) +{ + return fs_lock(file->parent, secs, lock_r); +} + +void fs_wrapper_unlock(struct fs_lock *_lock ATTR_UNUSED) +{ + i_unreached(); +} + +int fs_wrapper_exists(struct fs_file *file) +{ + return fs_exists(file->parent); +} + +int fs_wrapper_stat(struct fs_file *file, struct stat *st_r) +{ + return fs_stat(file->parent, st_r); +} + +int fs_wrapper_get_nlinks(struct fs_file *file, nlink_t *nlinks_r) +{ + return fs_get_nlinks(file->parent, nlinks_r); +} + +int fs_wrapper_copy(struct fs_file *src, struct fs_file *dest) +{ + if (src != NULL) + return fs_copy(src->parent, dest->parent); + else + return fs_copy_finish_async(dest->parent); +} + +int fs_wrapper_rename(struct fs_file *src, struct fs_file *dest) +{ + return fs_rename(src->parent, dest->parent); +} + +int fs_wrapper_delete(struct fs_file *file) +{ + return fs_delete(file->parent); +} + +struct fs_iter *fs_wrapper_iter_alloc(void) +{ + struct wrapper_fs_iter *iter = i_new(struct wrapper_fs_iter, 1); + return &iter->iter; +} + +void fs_wrapper_iter_init(struct fs_iter *_iter, const char *path, + enum fs_iter_flags flags) +{ + struct wrapper_fs_iter *iter = (struct wrapper_fs_iter *)_iter; + + iter->parent = fs_iter_init_parent(_iter, path, flags); +} + +const char *fs_wrapper_iter_next(struct fs_iter *_iter) +{ + struct wrapper_fs_iter *iter = (struct wrapper_fs_iter *)_iter; + const char *fname; + + iter->parent->async_callback = _iter->async_callback; + iter->parent->async_context = _iter->async_context; + + fname = fs_iter_next(iter->parent); + _iter->async_have_more = iter->parent->async_have_more; + return fname; +} + +int fs_wrapper_iter_deinit(struct fs_iter *_iter) +{ + struct wrapper_fs_iter *iter = (struct wrapper_fs_iter *)_iter; + const char *error; + int ret; + + if ((ret = fs_iter_deinit(&iter->parent, &error)) < 0) + fs_set_error_errno(_iter->event, "%s", error); + return ret; +} diff --git a/src/lib-fs/fs-wrapper.h b/src/lib-fs/fs-wrapper.h new file mode 100644 index 0000000..ebfd35d --- /dev/null +++ b/src/lib-fs/fs-wrapper.h @@ -0,0 +1,40 @@ +#ifndef FS_WRAPPER_H +#define FS_WRAPPER_H + +enum fs_get_metadata_flags; + +enum fs_properties fs_wrapper_get_properties(struct fs *fs); +void fs_wrapper_file_close(struct fs_file *file); +const char *fs_wrapper_file_get_path(struct fs_file *file); +void fs_wrapper_set_async_callback(struct fs_file *file, + fs_file_async_callback_t *callback, + void *context); +void fs_wrapper_wait_async(struct fs *fs); +void fs_wrapper_set_metadata(struct fs_file *file, const char *key, + const char *value); +int fs_wrapper_get_metadata(struct fs_file *file, + enum fs_get_metadata_flags flags, + const ARRAY_TYPE(fs_metadata) **metadata_r); +bool fs_wrapper_prefetch(struct fs_file *file, uoff_t length); +ssize_t fs_wrapper_read(struct fs_file *file, void *buf, size_t size); +struct istream * +fs_wrapper_read_stream(struct fs_file *file, size_t max_buffer_size); +int fs_wrapper_write(struct fs_file *file, const void *data, size_t size); +void fs_wrapper_write_stream(struct fs_file *file); +int fs_wrapper_write_stream_finish(struct fs_file *file, bool success); +int fs_wrapper_lock(struct fs_file *file, unsigned int secs, + struct fs_lock **lock_r); +void fs_wrapper_unlock(struct fs_lock *_lock); +int fs_wrapper_exists(struct fs_file *file); +int fs_wrapper_stat(struct fs_file *file, struct stat *st_r); +int fs_wrapper_get_nlinks(struct fs_file *file, nlink_t *nlinks_r); +int fs_wrapper_copy(struct fs_file *src, struct fs_file *dest); +int fs_wrapper_rename(struct fs_file *src, struct fs_file *dest); +int fs_wrapper_delete(struct fs_file *file); +struct fs_iter *fs_wrapper_iter_alloc(void); +void fs_wrapper_iter_init(struct fs_iter *iter, const char *path, + enum fs_iter_flags flags); +const char *fs_wrapper_iter_next(struct fs_iter *iter); +int fs_wrapper_iter_deinit(struct fs_iter *iter); + +#endif diff --git a/src/lib-fs/istream-fs-file.c b/src/lib-fs/istream-fs-file.c new file mode 100644 index 0000000..4f61cfe --- /dev/null +++ b/src/lib-fs/istream-fs-file.c @@ -0,0 +1,61 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-private.h" +#include "fs-api-private.h" +#include "istream-fs-file.h" + +struct fs_file_istream { + struct istream_private istream; + struct fs_file *file; +}; + +static void i_stream_fs_file_close(struct iostream_private *stream, + bool close_parent ATTR_UNUSED) +{ + struct fs_file_istream *fstream = (struct fs_file_istream *)stream; + + i_stream_destroy(&fstream->istream.parent); + fs_file_deinit(&fstream->file); +} + +static ssize_t i_stream_fs_file_read(struct istream_private *stream) +{ + struct fs_file_istream *fstream = (struct fs_file_istream *)stream; + struct istream *input; + + if (fstream->istream.parent == NULL) { + input = fs_read_stream(fstream->file, + i_stream_get_max_buffer_size(&stream->istream)); + i_stream_init_parent(stream, input); + i_stream_unref(&input); + } + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + return i_stream_read_copy_from_parent(&stream->istream); +} + +struct istream * +i_stream_create_fs_file(struct fs_file **file, size_t max_buffer_size) +{ + struct fs_file_istream *fstream; + struct istream *input; + + fstream = i_new(struct fs_file_istream, 1); + fstream->file = *file; + fstream->istream.iostream.close = i_stream_fs_file_close; + fstream->istream.max_buffer_size = max_buffer_size; + fstream->istream.read = i_stream_fs_file_read; + fstream->istream.stream_size_passthrough = TRUE; + + fstream->istream.istream.blocking = + ((*file)->flags & FS_OPEN_FLAG_ASYNC) == 0; + fstream->istream.istream.seekable = + ((*file)->flags & FS_OPEN_FLAG_SEEKABLE) != 0; + + input = i_stream_create(&fstream->istream, NULL, -1, 0); + i_stream_set_name(input, fs_file_path(*file)); + *file = NULL; + return input; +} diff --git a/src/lib-fs/istream-fs-file.h b/src/lib-fs/istream-fs-file.h new file mode 100644 index 0000000..2821c0e --- /dev/null +++ b/src/lib-fs/istream-fs-file.h @@ -0,0 +1,13 @@ +#ifndef ISTREAM_FS_FILE_H +#define ISTREAM_FS_FILE_H + +struct fs_file; + +/* Open the given file only when something is actually tried to be read from + the stream. The file is automatically deinitialized when the stream is + destroyed (which is why it's also set to NULL so it's not accidentally + double-freed). */ +struct istream * +i_stream_create_fs_file(struct fs_file **file, size_t max_buffer_size); + +#endif diff --git a/src/lib-fs/istream-fs-stats.c b/src/lib-fs/istream-fs-stats.c new file mode 100644 index 0000000..3f77430 --- /dev/null +++ b/src/lib-fs/istream-fs-stats.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "fs-api-private.h" +#include "istream-private.h" +#include "istream-fs-stats.h" + +struct fs_stats_istream { + struct istream_private istream; + struct fs_file *file; +}; + +static ssize_t +i_stream_fs_stats_read(struct istream_private *stream) +{ + struct fs_stats_istream *sstream = (struct fs_stats_istream *)stream; + ssize_t ret; + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + + ret = i_stream_read_copy_from_parent(&stream->istream); + if (ret > 0) { + /* count the first returned bytes as the finish time, since + we don't want to count the time caller spends on processing + this stream. (only the first fs_file_timing_end() call + actually does anything - the others are ignored.) */ + fs_file_timing_end(sstream->file, FS_OP_READ); + } + return ret; +} + +struct istream * +i_stream_create_fs_stats(struct istream *input, struct fs_file *file) +{ + struct fs_stats_istream *sstream; + + sstream = i_new(struct fs_stats_istream, 1); + sstream->file = file; + sstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + sstream->istream.stream_size_passthrough = TRUE; + sstream->istream.read = i_stream_fs_stats_read; + sstream->istream.istream.blocking = input->blocking; + sstream->istream.istream.seekable = input->seekable; + return i_stream_create(&sstream->istream, input, + i_stream_get_fd(input), 0); +} diff --git a/src/lib-fs/istream-fs-stats.h b/src/lib-fs/istream-fs-stats.h new file mode 100644 index 0000000..aca74b6 --- /dev/null +++ b/src/lib-fs/istream-fs-stats.h @@ -0,0 +1,9 @@ +#ifndef ISTREAM_FS_STATS_H +#define ISTREAM_FS_STATS_H + +struct fs_file; + +struct istream * +i_stream_create_fs_stats(struct istream *input, struct fs_file *file); + +#endif diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c new file mode 100644 index 0000000..ac2e8fb --- /dev/null +++ b/src/lib-fs/istream-metawrap.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-private.h" +#include "istream-metawrap.h" + +#define METAWRAP_MAX_METADATA_LINE_LEN 8192 + +struct metawrap_istream { + struct istream_private istream; + metawrap_callback_t *callback; + void *context; + + uoff_t start_offset, pending_seek; + bool in_metadata; +}; + +static int metadata_header_read(struct metawrap_istream *mstream) +{ + char *line, *p; + + while ((line = i_stream_read_next_line(mstream->istream.parent)) != NULL) { + if (*line == '\0') { + mstream->callback(NULL, NULL, mstream->context); + return 1; + } + p = strchr(line, ':'); + if (p == NULL) { + io_stream_set_error(&mstream->istream.iostream, + "Metadata header line is missing ':' at offset %"PRIuUOFF_T, + mstream->istream.istream.v_offset); + mstream->istream.istream.stream_errno = EINVAL; + return -1; + } + *p++ = '\0'; + mstream->callback(line, p, mstream->context); + } + if (mstream->istream.parent->eof) { + if (mstream->istream.parent->stream_errno != 0) { + mstream->istream.istream.stream_errno = + mstream->istream.parent->stream_errno; + } else { + io_stream_set_error(&mstream->istream.iostream, + "Metadata header is missing ending line at offset %"PRIuUOFF_T, + mstream->istream.istream.v_offset); + mstream->istream.istream.stream_errno = EPIPE; + return -1; + } + mstream->istream.istream.eof = TRUE; + return -1; + } + i_assert(!mstream->istream.parent->blocking); + return 0; +} + +static ssize_t i_stream_metawrap_read(struct istream_private *stream) +{ + struct metawrap_istream *mstream = (struct metawrap_istream *)stream; + int ret; + + i_stream_seek(stream->parent, mstream->start_offset + + stream->istream.v_offset); + + if (mstream->in_metadata) { + size_t prev_max_size = i_stream_get_max_buffer_size(stream->parent); + + i_stream_set_max_buffer_size(stream->parent, METAWRAP_MAX_METADATA_LINE_LEN); + ret = metadata_header_read(mstream); + i_stream_set_max_buffer_size(stream->parent, prev_max_size); + + i_assert(stream->istream.v_offset == 0); + mstream->start_offset = stream->parent->v_offset; + if (ret <= 0) + return ret; + /* this stream is kind of silently skipping over the metadata */ + stream->start_offset += mstream->start_offset; + mstream->in_metadata = FALSE; + if (mstream->pending_seek != 0) { + i_stream_seek(&stream->istream, mstream->pending_seek); + return i_stream_read_memarea(&stream->istream); + } + } + /* after metadata header it's all just passthrough */ + return i_stream_read_copy_from_parent(&stream->istream); +} + +static void +i_stream_metawrap_seek(struct istream_private *stream, + uoff_t v_offset, bool mark ATTR_UNUSED) +{ + struct metawrap_istream *mstream = (struct metawrap_istream *)stream; + + if (!mstream->in_metadata) { + /* already read through metadata. we can skip directly. */ + stream->istream.v_offset = v_offset; + mstream->pending_seek = 0; + } else { + /* we need to read through the metadata first */ + mstream->pending_seek = v_offset; + stream->istream.v_offset = 0; + } + stream->skip = stream->pos = 0; +} + +static int i_stream_metawrap_stat(struct istream_private *stream, bool exact) +{ + struct metawrap_istream *mstream = (struct metawrap_istream *)stream; + const struct stat *st; + int ret; + + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; + return -1; + } + stream->statbuf = *st; + + if (mstream->in_metadata) { + ret = i_stream_read_memarea(&stream->istream); + if (ret < 0 && stream->istream.stream_errno != 0) + return -1; + if (ret == 0) { + stream->statbuf.st_size = -1; + return 0; + } + } + i_assert((uoff_t)stream->statbuf.st_size >= mstream->start_offset); + stream->statbuf.st_size -= mstream->start_offset; + return 0; +} + +struct istream * +i_stream_create_metawrap(struct istream *input, + metawrap_callback_t *callback, void *context) +{ + struct metawrap_istream *mstream; + + mstream = i_new(struct metawrap_istream, 1); + mstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + + mstream->istream.read = i_stream_metawrap_read; + mstream->istream.seek = i_stream_metawrap_seek; + mstream->istream.stat = input->seekable ? i_stream_metawrap_stat : NULL; + + mstream->istream.istream.readable_fd = input->readable_fd; + mstream->istream.istream.blocking = input->blocking; + mstream->istream.istream.seekable = input->seekable; + mstream->in_metadata = TRUE; + mstream->callback = callback; + mstream->context = context; + return i_stream_create(&mstream->istream, input, + i_stream_get_fd(input), 0); +} diff --git a/src/lib-fs/istream-metawrap.h b/src/lib-fs/istream-metawrap.h new file mode 100644 index 0000000..90ca2d3 --- /dev/null +++ b/src/lib-fs/istream-metawrap.h @@ -0,0 +1,14 @@ +#ifndef ISTREAM_METAWRAP_H +#define ISTREAM_METAWRAP_H + +typedef void +metawrap_callback_t(const char *key, const char *value, void *context); + +/* Input stream is in format "key:value\nkey2:value2\n...\n\ncontents. + The given callback is called for each key/value metadata pair, and the + returned stream will skip over the metadata and return only the contents. */ +struct istream * +i_stream_create_metawrap(struct istream *input, + metawrap_callback_t *callback, void *context); + +#endif diff --git a/src/lib-fs/ostream-cmp.c b/src/lib-fs/ostream-cmp.c new file mode 100644 index 0000000..8599799 --- /dev/null +++ b/src/lib-fs/ostream-cmp.c @@ -0,0 +1,95 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "ostream-private.h" +#include "ostream-cmp.h" + +struct cmp_ostream { + struct ostream_private ostream; + + struct istream *input; + bool equals; +}; + +static void o_stream_cmp_close(struct iostream_private *stream, + bool close_parent) +{ + struct cmp_ostream *cstream = (struct cmp_ostream *)stream; + + if (cstream->input == NULL) + return; + + i_stream_unref(&cstream->input); + if (close_parent) + o_stream_close(cstream->ostream.parent); +} + +bool stream_cmp_block(struct istream *input, + const unsigned char *data, size_t size) +{ + const unsigned char *indata; + size_t insize, max; + + while (size > 0) { + (void)i_stream_read_bytes(input, &indata, &insize, size); + max = I_MIN(insize, size); + if (insize == 0 || memcmp(data, indata, max) != 0) + return FALSE; + data += max; + size -= max; + i_stream_skip(input, max); + } + return TRUE; +} + +static ssize_t +o_stream_cmp_sendv(struct ostream_private *stream, + const struct const_iovec *iov, unsigned int iov_count) +{ + struct cmp_ostream *cstream = (struct cmp_ostream *)stream; + unsigned int i; + ssize_t ret; + + if (cstream->equals) { + for (i = 0; i < iov_count; i++) { + if (!stream_cmp_block(cstream->input, iov[i].iov_base, + iov[i].iov_len)) { + cstream->equals = FALSE; + break; + } + } + } + + if ((ret = o_stream_sendv(stream->parent, iov, iov_count)) < 0) { + o_stream_copy_error_from_parent(stream); + return -1; + } + + stream->ostream.offset += ret; + return ret; +} + +struct ostream * +o_stream_create_cmp(struct ostream *output, struct istream *input) +{ + struct cmp_ostream *cstream; + + cstream = i_new(struct cmp_ostream, 1); + cstream->ostream.sendv = o_stream_cmp_sendv; + cstream->ostream.iostream.close = o_stream_cmp_close; + cstream->input = input; + cstream->equals = TRUE; + i_stream_ref(input); + + return o_stream_create(&cstream->ostream, output, + o_stream_get_fd(output)); +} + +bool o_stream_cmp_equals(struct ostream *_output) +{ + struct cmp_ostream *cstream = + (struct cmp_ostream *)_output->real_stream; + + return cstream->equals; +} diff --git a/src/lib-fs/ostream-cmp.h b/src/lib-fs/ostream-cmp.h new file mode 100644 index 0000000..cd6d567 --- /dev/null +++ b/src/lib-fs/ostream-cmp.h @@ -0,0 +1,15 @@ +#ifndef OSTREAM_CMP_H +#define OSTREAM_CMP_H + +/* Compare given input stream to output being written to output stream. */ +struct ostream * +o_stream_create_cmp(struct ostream *output, struct istream *input); +/* Returns TRUE if input and output are equal so far. If the caller needs to + know if the files are entirely equal, it should check also if input stream + is at EOF. */ +bool o_stream_cmp_equals(struct ostream *output); + +bool stream_cmp_block(struct istream *input, + const unsigned char *data, size_t size); + +#endif diff --git a/src/lib-fs/ostream-metawrap.c b/src/lib-fs/ostream-metawrap.c new file mode 100644 index 0000000..3359bd3 --- /dev/null +++ b/src/lib-fs/ostream-metawrap.c @@ -0,0 +1,71 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "ostream-private.h" +#include "ostream-metawrap.h" + +struct metawrap_ostream { + struct ostream_private ostream; + void (*write_callback)(void *); + void *context; +}; + +static void o_stream_metawrap_call_callback(struct metawrap_ostream *mstream) +{ + void (*write_callback)(void *) = mstream->write_callback; + + if (write_callback != NULL) { + mstream->write_callback = NULL; + write_callback(mstream->context); + /* metadata headers aren't counted as part of the offset */ + mstream->ostream.ostream.offset = 0; + } +} + +static ssize_t +o_stream_metawrap_sendv(struct ostream_private *stream, + const struct const_iovec *iov, unsigned int iov_count) +{ + struct metawrap_ostream *mstream = (struct metawrap_ostream *)stream; + ssize_t ret; + + o_stream_metawrap_call_callback(mstream); + if ((ret = o_stream_sendv(stream->parent, iov, iov_count)) < 0) + o_stream_copy_error_from_parent(stream); + else + stream->ostream.offset += ret; + return ret; +} + +static enum ostream_send_istream_result +o_stream_metawrap_send_istream(struct ostream_private *_outstream, + struct istream *instream) +{ + struct metawrap_ostream *outstream = + (struct metawrap_ostream *)_outstream; + uoff_t orig_instream_offset = instream->v_offset; + enum ostream_send_istream_result res; + + o_stream_metawrap_call_callback(outstream); + if ((res = o_stream_send_istream(_outstream->parent, instream)) == OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT) + o_stream_copy_error_from_parent(_outstream); + _outstream->ostream.offset += instream->v_offset - orig_instream_offset; + return res; +} + +struct ostream * +o_stream_create_metawrap(struct ostream *output, + void (*write_callback)(void *), void *context) +{ + struct metawrap_ostream *mstream; + + mstream = i_new(struct metawrap_ostream, 1); + mstream->ostream.sendv = o_stream_metawrap_sendv; + mstream->ostream.send_istream = o_stream_metawrap_send_istream; + mstream->write_callback = write_callback; + mstream->context = context; + + return o_stream_create(&mstream->ostream, output, + o_stream_get_fd(output)); +} diff --git a/src/lib-fs/ostream-metawrap.h b/src/lib-fs/ostream-metawrap.h new file mode 100644 index 0000000..09a172c --- /dev/null +++ b/src/lib-fs/ostream-metawrap.h @@ -0,0 +1,8 @@ +#ifndef OSTREAM_METAWRAP_H +#define OSTREAM_METAWRAP_H + +struct ostream * +o_stream_create_metawrap(struct ostream *output, + void (*write_callback)(void *), void *context); + +#endif diff --git a/src/lib-fs/test-fs-metawrap.c b/src/lib-fs/test-fs-metawrap.c new file mode 100644 index 0000000..161f1c5 --- /dev/null +++ b/src/lib-fs/test-fs-metawrap.c @@ -0,0 +1,103 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "fs-test.h" +#include "test-common.h" + +static const struct fs_settings fs_set; + +static void test_fs_metawrap_stat(void) +{ + struct fs *fs; + struct fs_file *file; + struct test_fs_file *test_file; + struct istream *input; + struct stat st; + const char *error; + unsigned int i; + + test_begin("fs metawrap stat"); + + if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + + for (i = 0; i < 2; i++) { + file = fs_file_init(fs, "foo", FS_OPEN_MODE_READONLY); + + test_file = test_fs_file_get(fs, "foo"); + str_append(test_file->contents, "key:value\n\n12345678901234567890"); + + if (i == 0) { + input = fs_read_stream(file, 2); + test_istream_set_max_buffer_size(test_file->input, 2); + } else { + input = NULL; + } + + test_assert_idx(fs_stat(file, &st) == 0 && st.st_size == 20, i); + + i_stream_unref(&input); + fs_file_deinit(&file); + } + fs_deinit(&fs); + test_end(); +} + +static void test_fs_metawrap_async(void) +{ + test_fs_async("metawrap", FS_PROPERTY_METADATA, "metawrap", "test"); + test_fs_async("metawrap passthrough", 0, "metawrap", "test"); + test_fs_async("double-metawrap", FS_PROPERTY_METADATA, "metawrap", "metawrap:test"); +} + +static void test_fs_metawrap_write_empty(void) +{ + struct fs *fs; + struct stat st; + const char *error; + + test_begin("fs metawrap write empty file"); + if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + struct fs_file *file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE); + struct ostream *output = fs_write_stream(file); + test_assert(fs_write_stream_finish(file, &output) > 0); + test_assert(fs_stat(file, &st) == 0 && st.st_size == 0); + fs_file_deinit(&file); + fs_deinit(&fs); + test_end(); +} + +static void test_fs_metawrap_write_fname_rename(void) +{ + struct fs *fs; + const char *error; + + test_begin("fs metawrap write fname rename"); + if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + struct fs_file *file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE); + struct ostream *output = fs_write_stream(file); + o_stream_nsend_str(output, "test"); + fs_set_metadata(file, FS_METADATA_WRITE_FNAME, "renamed"); + test_assert(fs_write_stream_finish(file, &output) > 0); + test_assert(strcmp(fs_file_path(file), "renamed") == 0); + fs_file_deinit(&file); + fs_deinit(&fs); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_fs_metawrap_stat, + test_fs_metawrap_async, + test_fs_metawrap_write_empty, + test_fs_metawrap_write_fname_rename, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-fs/test-fs-posix.c b/src/lib-fs/test-fs-posix.c new file mode 100644 index 0000000..d6fea11 --- /dev/null +++ b/src/lib-fs/test-fs-posix.c @@ -0,0 +1,144 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "ostream.h" +#include "fs-api.h" +#include "safe-mkdir.h" +#include "safe-mkstemp.h" +#include "test-common.h" +#include "unlink-directory.h" +#include <sys/stat.h> +#include <unistd.h> + +static void test_fs_posix(void) +{ + const char testdir[] = ".test-fs-posix"; + const char *unlink_err; + + if (unlink_directory(testdir, UNLINK_DIRECTORY_FLAG_RMDIR, &unlink_err) < 0) { + i_error("Couldn't prepare test directory (%s): %s", testdir, unlink_err); + goto error_no_testdir; + } + if (safe_mkdir(testdir, 0700, (uid_t)-1, (gid_t)-1) != 1) { + /* Something just raced us to create this directory, bail. */ + goto error_no_testdir; + } + + int ret; + const char *error; + struct fs *fs; + struct fs_settings fs_set; + + test_begin("test-fs-posix filesystem"); + i_zero(&fs_set); + ret = fs_init("posix", t_strdup_printf("prefix=%s/", testdir), &fs_set, &fs, &error); + test_out_quiet("fs_init() failed", ret >= 0); + if (ret < 0) { + test_end(); + goto error_no_fs; + } + + struct fs *ref = fs; + fs_ref(ref); + fs_unref(&ref); + test_assert(ref == NULL); + test_assert(fs != NULL); + + test_assert(fs_get_parent(fs) == NULL); + test_assert_strcmp(fs_get_driver(fs), "posix"); + test_end(); + + struct fs_file *file; + char buf[10]; + ssize_t count; + test_begin("test-fs-posix bad file read"); + file = fs_file_init(fs, "fail_1", FS_OPEN_MODE_READONLY); + test_assert(fs_exists(file) == 0); + count = fs_read(file, buf, 1); + test_assert(count == -1 && errno == ENOENT && + strstr(fs_file_last_error(file), "No such file or directory") != NULL); + fs_file_deinit(&file); + test_end(); + + test_begin("test-fs-posix good file write"); + file = fs_file_init(fs, "good1", FS_OPEN_MODE_REPLACE); + test_assert(file != NULL); + test_assert(fs_exists(file) == 0); /* file not created until data is written */ + test_assert(fs_write(file, "X", 1) == 0); + test_assert(fs_exists(file) == 1); + fs_file_deinit(&file); + test_end(); + + test_begin("test-fs-posix good file read"); + file = fs_file_init(fs, "good1", FS_OPEN_MODE_READONLY); + test_assert(fs_exists(file) == 1); + errno = 0; + count = fs_read(file, buf, 2); + test_assert(count == 1 && errno == 0); + fs_file_deinit(&file); + test_end(); + + struct fs_iter *iter = fs_iter_init(fs, "/", 0); + const char *filename; + test_begin("test-fs-posix iterator"); + filename = fs_iter_next(iter); + test_assert_strcmp(filename, "good1"); + test_assert(fs_iter_next(iter) == NULL); + fs_iter_deinit(&iter, &error); + test_end(); + + struct stat st; + test_begin("test-fs-posix file stat and delete"); + file = fs_file_init(fs, "good1", FS_OPEN_MODE_READONLY); + test_assert(fs_stat(file, &st) == 0); + test_assert(st.st_size == 1); + test_assert(fs_delete(file) == 0); + fs_file_deinit(&file); + test_end(); + + test_begin("test-fs-posix file write fname rename"); + file = fs_file_init(fs, "subdir/badfname", FS_OPEN_MODE_REPLACE); + struct ostream *output = fs_write_stream(file); + o_stream_nsend_str(output, "hello"); + fs_set_metadata(file, FS_METADATA_WRITE_FNAME, "subdir/rename1"); + test_assert(fs_write_stream_finish(file, &output) == 1); + test_assert(strcmp(fs_file_path(file), "subdir/rename1") == 0); + fs_file_deinit(&file); + file = fs_file_init(fs, "subdir/rename1", FS_OPEN_MODE_READONLY); + test_assert(fs_stat(file, &st) == 0); + test_assert(st.st_size == 5); + fs_file_deinit(&file); + test_end(); + + test_begin("test-fs-posix file copy fname rename"); + struct fs_file *src = fs_file_init(fs, "subdir/rename1", FS_OPEN_MODE_READONLY); + struct fs_file *dest = fs_file_init(fs, "subdir/badfname", FS_OPEN_MODE_REPLACE); + fs_set_metadata(dest, FS_METADATA_WRITE_FNAME, "subdir/rename2"); + test_assert(fs_copy(src, dest) == 0); + test_assert(strcmp(fs_file_path(dest), "subdir/rename2") == 0); + fs_file_deinit(&src); + fs_file_deinit(&dest); + file = fs_file_init(fs, "subdir/rename2", FS_OPEN_MODE_READONLY); + test_assert(fs_stat(file, &st) == 0); + test_assert(st.st_size == 5); + fs_file_deinit(&file); + test_end(); + + fs_deinit(&fs); + +error_no_fs: + if (unlink_directory(testdir, UNLINK_DIRECTORY_FLAG_RMDIR, &unlink_err) < 0) + i_error("Couldn't clean up test directory (%s): %s", testdir, unlink_err); +error_no_testdir: + return; +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_fs_posix, + NULL + }; + return test_run(test_functions); +} |